provided hash-based index generator

GitOrigin-RevId: 6680b87504403c0bd3039fdacc7c4577c0b8239e
This commit is contained in:
Dmitry Batkovich
2019-08-06 11:14:12 +03:00
committed by intellij-monorepo-bot
parent 988352f3d1
commit caabd81c55
6 changed files with 516 additions and 0 deletions
@@ -1922,6 +1922,8 @@
<jreProvider implementation="com.intellij.execution.ui.BundledJreProvider"/>
<registryKey key="ide.java.show.bundled.runtime" defaultValue="false" description="Enables using bundled JetBrains Runtime for execution"/>
<codeInsight.inlayProvider language="JAVA" implementationClass="com.intellij.codeInsight.daemon.impl.JavaLensProvider"/>
<fileBasedIndex.providedLocator implementation="com.intellij.index.BasicProvidedExtensionLocator"/>
</extensions>
<extensions defaultExtensionNs="org.jetbrains">
@@ -0,0 +1,111 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.internal;
import com.intellij.codeInspection.bytecodeAnalysis.BytecodeAnalysisIndex;
import com.intellij.find.ngrams.TrigramIndex;
import com.intellij.openapi.actionSystem.AnAction;
import com.intellij.openapi.actionSystem.AnActionEvent;
import com.intellij.openapi.fileChooser.FileChooser;
import com.intellij.openapi.fileChooser.FileChooserDescriptor;
import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory;
import com.intellij.openapi.module.ModuleManager;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.roots.*;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.vfs.VfsUtilCore;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileUrlChangeAdapter;
import com.intellij.psi.impl.cache.impl.id.IdIndex;
import com.intellij.psi.impl.java.stubs.index.JavaAutoModuleNameIndex;
import com.intellij.psi.stubs.StubIndexKey;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.FileBasedIndexExtension;
import com.intellij.util.indexing.ID;
import com.intellij.util.indexing.IndexableSetContributor;
import com.intellij.util.indexing.hash.HashBasedIndexGenerator;
import one.util.streamex.StreamEx;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.index.stubs.StubHashBasedIndexGenerator;
import java.io.File;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class DumpIndexAction extends AnAction {
@Override
public void actionPerformed(@NotNull AnActionEvent event) {
Project project = event.getProject();
if (project == null) return;
Stream<VirtualFile> libRoots = Arrays
.stream(ModuleManager.getInstance(project).getModules())
.flatMap(m -> Arrays.stream(ModuleRootManager.getInstance(m).getOrderEntries()))
.filter(e -> e instanceof LibraryOrSdkOrderEntry)
.flatMap(e -> Stream.concat(Arrays.stream(e.getFiles(OrderRootType.CLASSES)), Arrays.stream(e.getFiles(OrderRootType.SOURCES))));
Stream<VirtualFile> additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(),
IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream()));
Set<VirtualFile> roots = Stream.concat(libRoots, additionalRoots).collect(Collectors.toSet());
FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor();
descriptor.withTitle("Select Index Dump Directory");
VirtualFile file = FileChooser.chooseFile(descriptor, project, null);
if (file == null) return;
File out = VfsUtilCore.virtualToIoFile(file);
FileUtil.delete(out);
exportIndices(roots, out);
}
public static void exportIndices(@NotNull Set<VirtualFile> roots, @NotNull File out) {
StubHashBasedIndexGenerator generator = new StubHashBasedIndexGenerator(out);
generator.generate(roots);
getExportableIndices().forEach(ex -> {
HashBasedIndexGenerator indexGenerator = new HashBasedIndexGenerator(ex, out);
indexGenerator.generate(roots);
});
}
@NotNull
private static Stream<FileBasedIndexExtension> getExportableIndices() {
//kt
Stream<FileBasedIndexExtension> ktIndices =
FileBasedIndexExtension
.EXTENSION_POINT_NAME
.extensions()
.filter(id -> id.getName().getName().contains("kotlin"));
Set<ID<Object, Object>> xmlIndexIds = ContainerUtil.set(ID.findByName("XmlTagNames"),
ID.findByName("XmlNamespaces"),
ID.findByName("SchemaTypeInheritance"),
ID.findByName("DomFileIndex"),
ID.findByName("xmlProperties"));
//xml
Stream<FileBasedIndexExtension> xmlIndices =
FileBasedIndexExtension
.EXTENSION_POINT_NAME
.extensions()
.filter(id -> xmlIndexIds.contains(id.getName()));
//base
Stream<FileBasedIndexExtension> coreIndices =
FileBasedIndexExtension
.EXTENSION_POINT_NAME
.extensions()
.filter(ex -> ex.getName().equals(TrigramIndex.INDEX_ID) ||
ex.getName().equals(IdIndex.NAME));
//java
Stream<FileBasedIndexExtension> javaIndices =
FileBasedIndexExtension
.EXTENSION_POINT_NAME
.extensions()
.filter(ex -> ex instanceof BytecodeAnalysisIndex || ex instanceof JavaAutoModuleNameIndex);
return Stream.concat(Stream.concat(Stream.concat(coreIndices, javaIndices), xmlIndices), ktIndices);
}
}
@@ -0,0 +1,82 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.index;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.stubs.StubUpdatingIndex;
import com.intellij.psi.stubs.provided.StubProvidedIndexExtension;
import com.intellij.util.indexing.FileBasedIndexExtension;
import com.intellij.util.indexing.ID;
import com.intellij.util.indexing.provided.ProvidedIndexExtension;
import com.intellij.util.indexing.provided.ProvidedIndexExtensionLocator;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.KeyDescriptor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
public class BasicProvidedExtensionLocator implements ProvidedIndexExtensionLocator {
private static final String PREBUILT_INDEX_PATH_PROP = "prebuilt.hash.index.dir";
@Nullable
@Override
public <K, V> ProvidedIndexExtension<K, V> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
File root = getPrebuiltIndexPath();
if (root == null || !new File(root, StringUtil.toLowerCase(originalExtension.getName().getName())).exists()) return null;
return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID)
? (ProvidedIndexExtension<K, V>)new StubProvidedIndexExtension(root)
: new ProvidedIndexExtensionImpl<>(root, originalExtension);
}
@Nullable
private static File getPrebuiltIndexPath() {
String path = System.getProperty(PREBUILT_INDEX_PATH_PROP);
if (path == null) return null;
File file = new File(path);
return file.exists() ? file : null;
}
private static class ProvidedIndexExtensionImpl<K, V> implements ProvidedIndexExtension<K, V> {
@NotNull
private final File myIndexFile;
@NotNull
private final ID<K, V> myIndexId;
@NotNull
private final KeyDescriptor<K> myKeyDescriptor;
@NotNull
private final DataExternalizer<V> myValueExternalizer;
private ProvidedIndexExtensionImpl(@NotNull File file,
@NotNull FileBasedIndexExtension<K, V> originalExtension) {
myIndexFile = file;
myIndexId = originalExtension.getName();
myKeyDescriptor = originalExtension.getKeyDescriptor();
myValueExternalizer = originalExtension.getValueExternalizer();
}
@NotNull
@Override
public File getIndexPath() {
return myIndexFile;
}
@NotNull
@Override
public ID<K, V> getIndexId() {
return myIndexId;
}
@NotNull
@Override
public KeyDescriptor<K> createKeyDescriptor() {
return myKeyDescriptor;
}
@NotNull
@Override
public DataExternalizer<V> createValueExternalizer() {
return myValueExternalizer;
}
}
}
@@ -0,0 +1,230 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util.indexing.hash;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.VfsUtilCore;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileVisitor;
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
import com.intellij.util.indexing.*;
import com.intellij.util.indexing.impl.InputData;
import com.intellij.util.indexing.impl.MapIndexStorage;
import com.intellij.util.indexing.impl.MapReduceIndex;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.KeyDescriptor;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
public class HashBasedIndexGenerator<K, V> {
@NotNull private final File myOut;
@NotNull private final File myHashOut;
@NotNull private final FakeIndexExtension<K, V> myExtension;
@NotNull private final FileBasedIndex.InputFilter myInputFilter;
protected ContentHashesUtil.HashEnumerator myHashEnumerator;
private InvertedIndex<K, V, FileContent> myIndex;
public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension<K, V> indexExtension, @NotNull File out) {
this(indexExtension.getKeyDescriptor(),
indexExtension.getValueExternalizer(),
indexExtension,
out,
out);
}
public HashBasedIndexGenerator(@NotNull KeyDescriptor<K> keyDescriptor,
@NotNull DataExternalizer<V> valueExternalizer,
@NotNull FileBasedIndexExtension<K, V> originalExtension,
@NotNull File out,
@NotNull File hashOut) {
myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension);
myOut = out;
myHashOut = hashOut;
FileBasedIndex.InputFilter filter = originalExtension.getInputFilter();
if (filter instanceof FileBasedIndex.FileTypeSpecificInputFilter) {
Set<FileType> fileTypes = new HashSet<>();
((FileBasedIndex.FileTypeSpecificInputFilter)filter).registerFileTypesUsedForIndexing(fileTypes::add);
myInputFilter = file -> fileTypes.contains(file.getFileType()) && filter.acceptInput(file);
} else {
myInputFilter = filter;
}
}
@NotNull
public File getOut() {
return myOut;
}
public void openIndex() throws IOException {
myHashEnumerator = getHashEnumerator();
String indexName = myExtension.getName().getName();
myIndex = new MapReduceIndex<K, V, FileContent>(myExtension, new MapIndexStorage<K, V>(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName),
myExtension.getKeyDescriptor(),
myExtension.getValueExternalizer(),
myExtension.getCacheSize(),
myExtension.keyIsUniqueForIndexedFile()) {
@Override
protected void checkCanceled() {
//ignore
}
}, null, null) {
@Override
protected void updateForwardIndex(int inputId, @NotNull InputData<K, V> data) throws IOException {
super.updateForwardIndex(inputId, data);
try {
visitInputData(inputId, data);
}
catch (StorageException e) {
throw new IOException(e);
}
}
@Override
public void checkCanceled() {
//ignore
}
@Override
protected void requestRebuild(@NotNull Throwable e) {
throw new RuntimeException(e);
}
};
}
@NotNull
protected ContentHashesUtil.HashEnumerator getHashEnumerator() throws IOException {
return new ContentHashesUtil.HashEnumerator(new File(myHashOut, "hashes"));
}
protected void visitInputData(int hashId, @NotNull InputData<K, V> data) throws StorageException {
}
public void closeIndex() throws IOException {
if (myIndex != null) myIndex.dispose();
if (myHashEnumerator != null) myHashEnumerator.close();
}
public final void generate(@NotNull Collection<VirtualFile> roots) {
try {
openIndex();
for (VirtualFile root : roots) {
VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor<Boolean>() {
@Override
public boolean visitFile(@NotNull VirtualFile file) {
if (!file.isDirectory() && myInputFilter.acceptInput(file)) {
indexFile(file);
}
return true;
}
});
}
}
catch (IOException e) {
throw new RuntimeException(e);
}
finally {
try {
closeIndex();
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
}
protected void indexFile(@NotNull VirtualFile f) {
try {
FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray());
byte[] hash = FileBasedIndexImpl.calculateHash(fc.getContent(), fc.getCharset(), f.getFileType(), fc.getFileType());
int hashId = myHashEnumerator.enumerate(hash);
if (!myIndex.update(hashId, fc).compute()) {
throw new RuntimeException();
}
}
catch (IOException e) {
throw new RuntimeException("cant index " + f.getPath() + " for " + myExtension.getName().getName(), e);
}
}
public InvertedIndex<K, V, FileContent> getIndex() {
return myIndex;
}
private static final class FakeIndexExtension<K, V> extends FileBasedIndexExtension<K, V> {
@NotNull
private final KeyDescriptor<K> myKeyDescriptor;
@NotNull
private final DataExternalizer<V> myValueExternalizer;
@NotNull
private final FileBasedIndexExtension<K, V> myOriginalExtension;
private FakeIndexExtension(@NotNull KeyDescriptor<K> descriptor,
@NotNull DataExternalizer<V> externalizer,
@NotNull FileBasedIndexExtension<K, V> originalExtension) {
myKeyDescriptor = descriptor;
myValueExternalizer = externalizer;
myOriginalExtension = originalExtension;
}
@NotNull
@Override
public ID<K, V> getName() {
return myOriginalExtension.getName();
}
@NotNull
@Override
public FileBasedIndex.InputFilter getInputFilter() {
return myOriginalExtension.getInputFilter();
}
@Override
public boolean dependsOnFileContent() {
return true;
}
@Override
public int getCacheSize() {
return myOriginalExtension.getCacheSize();
}
@Override
public boolean keyIsUniqueForIndexedFile() {
return myOriginalExtension.keyIsUniqueForIndexedFile();
}
@NotNull
@Override
public DataIndexer<K, V, FileContent> getIndexer() {
return myOriginalExtension.getIndexer();
}
@NotNull
@Override
public KeyDescriptor<K> getKeyDescriptor() {
return myKeyDescriptor;
}
@NotNull
@Override
public DataExternalizer<V> getValueExternalizer() {
return myValueExternalizer;
}
@Override
public int getVersion() {
return myOriginalExtension.getVersion();
}
}
}
@@ -0,0 +1,88 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.index.stubs;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
import com.intellij.psi.stubs.*;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.FileBasedIndexExtension;
import com.intellij.util.indexing.StorageException;
import com.intellij.util.indexing.hash.HashBasedIndexGenerator;
import com.intellij.util.indexing.impl.EmptyInputDataDiffBuilder;
import com.intellij.util.indexing.impl.InputData;
import com.intellij.util.indexing.impl.MapReduceIndex;
import com.intellij.util.indexing.impl.UpdateData;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.util.*;
public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer, SerializedStubTree> {
private final Map<StubIndexKey, HashBasedIndexGenerator> myStubIndexesGeneratorMap = new HashMap<>();
private final Set<StubIndexKey> myUsedKeys = new HashSet<>();
public StubHashBasedIndexGenerator(@NotNull File out) {
super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(false, null), getExtension(), out, out);
for (StubIndexExtension<?, ?> stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) {
myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(stubIndexExtension.getKeyDescriptor(),
StubIndexImpl.StubIdExternalizer.INSTANCE,
StubIndexImpl
.wrapStubIndexExtension(stubIndexExtension),
new File(out, getStubsDir()),
out) {
@NotNull
@Override
protected ContentHashesUtil.HashEnumerator getHashEnumerator() {
return StubHashBasedIndexGenerator.this.myHashEnumerator;
}
});
}
}
@NotNull
private static String getStubsDir() {
return StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName());
}
@Override
protected void visitInputData(int hashId, @NotNull InputData<Integer, SerializedStubTree> data) throws StorageException {
super.visitInputData(hashId, data);
SerializedStubTree tree = ContainerUtil.getFirstItem(data.getKeyValues().values());
if (tree == null) return;
Map<StubIndexKey, Map<Object, StubIdList>> map = tree.getStubIndicesValueMap();
for (Map.Entry<StubIndexKey, Map<Object, StubIdList>> entry : map.entrySet()) {
StubIndexKey key = entry.getKey();
Map<Object, StubIdList> value = entry.getValue();
myUsedKeys.add(key);
MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex();
index.updateWithMap(new UpdateData(hashId, value, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null));
}
}
@Override
public void openIndex() throws IOException {
super.openIndex();
for (HashBasedIndexGenerator generator : myStubIndexesGeneratorMap.values()) {
generator.openIndex();
}
}
@Override
public void closeIndex() throws IOException {
super.closeIndex();
for (Map.Entry<StubIndexKey, HashBasedIndexGenerator> entry : myStubIndexesGeneratorMap.entrySet()) {
HashBasedIndexGenerator generator = entry.getValue();
generator.closeIndex();
}
((SerializationManagerEx)SerializationManager.getInstance()).flushNameStorage();
FileUtil.copyDir(PathManager.getIndexRoot(), new File(getOut(), getStubsDir()), f -> f.getName().startsWith("rep.names"));
}
private static StubUpdatingIndex getExtension() {
return (StubUpdatingIndex)FileBasedIndexExtension.EXTENSION_POINT_NAME.extensions().filter(ex -> ex instanceof StubUpdatingIndex).findAny().get();
}
}
+3
View File
@@ -464,6 +464,9 @@
<action id="StaticIconFields" internal="true" class="com.intellij.internal.StaticIconFieldsAction"
text="List Static Field Initializers"/>
<action id="DumpIndex" internal="true" class="com.intellij.internal.DumpIndexAction"
text="Dump Index"/>
<add-to-group group-id="Internal"/>
</group>