diff --git a/java/java-impl/src/META-INF/JavaPlugin.xml b/java/java-impl/src/META-INF/JavaPlugin.xml index a218d100f9ea..815dca967245 100644 --- a/java/java-impl/src/META-INF/JavaPlugin.xml +++ b/java/java-impl/src/META-INF/JavaPlugin.xml @@ -1922,6 +1922,8 @@ + + diff --git a/java/java-impl/src/com/intellij/internal/DumpIndexAction.java b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java new file mode 100644 index 000000000000..68e4ef8fd9b3 --- /dev/null +++ b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java @@ -0,0 +1,111 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal; + +import com.intellij.codeInspection.bytecodeAnalysis.BytecodeAnalysisIndex; +import com.intellij.find.ngrams.TrigramIndex; +import com.intellij.openapi.actionSystem.AnAction; +import com.intellij.openapi.actionSystem.AnActionEvent; +import com.intellij.openapi.fileChooser.FileChooser; +import com.intellij.openapi.fileChooser.FileChooserDescriptor; +import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory; +import com.intellij.openapi.module.ModuleManager; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.roots.*; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.vfs.VfsUtilCore; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileUrlChangeAdapter; +import com.intellij.psi.impl.cache.impl.id.IdIndex; +import com.intellij.psi.impl.java.stubs.index.JavaAutoModuleNameIndex; +import com.intellij.psi.stubs.StubIndexKey; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.FileBasedIndexExtension; +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.IndexableSetContributor; +import com.intellij.util.indexing.hash.HashBasedIndexGenerator; +import one.util.streamex.StreamEx; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.index.stubs.StubHashBasedIndexGenerator; + +import java.io.File; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class DumpIndexAction extends AnAction { + + @Override + public void actionPerformed(@NotNull AnActionEvent event) { + Project project = event.getProject(); + if (project == null) return; + Stream libRoots = Arrays + .stream(ModuleManager.getInstance(project).getModules()) + .flatMap(m -> Arrays.stream(ModuleRootManager.getInstance(m).getOrderEntries())) + .filter(e -> e instanceof LibraryOrSdkOrderEntry) + .flatMap(e -> Stream.concat(Arrays.stream(e.getFiles(OrderRootType.CLASSES)), Arrays.stream(e.getFiles(OrderRootType.SOURCES)))); + + Stream additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(), + IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())); + Set roots = Stream.concat(libRoots, additionalRoots).collect(Collectors.toSet()); + + FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor(); + descriptor.withTitle("Select Index Dump Directory"); + VirtualFile file = FileChooser.chooseFile(descriptor, project, null); + if (file == null) return; + File out = VfsUtilCore.virtualToIoFile(file); + FileUtil.delete(out); + exportIndices(roots, out); + } + + public static void exportIndices(@NotNull Set roots, @NotNull File out) { + StubHashBasedIndexGenerator generator = new StubHashBasedIndexGenerator(out); + generator.generate(roots); + + getExportableIndices().forEach(ex -> { + HashBasedIndexGenerator indexGenerator = new HashBasedIndexGenerator(ex, out); + indexGenerator.generate(roots); + }); + } + + @NotNull + private static Stream getExportableIndices() { + //kt + Stream ktIndices = + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(id -> id.getName().getName().contains("kotlin")); + + Set> xmlIndexIds = ContainerUtil.set(ID.findByName("XmlTagNames"), + ID.findByName("XmlNamespaces"), + ID.findByName("SchemaTypeInheritance"), + ID.findByName("DomFileIndex"), + ID.findByName("xmlProperties")); + //xml + Stream xmlIndices = + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(id -> xmlIndexIds.contains(id.getName())); + + //base + Stream coreIndices = + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(ex -> ex.getName().equals(TrigramIndex.INDEX_ID) || + ex.getName().equals(IdIndex.NAME)); + + //java + Stream javaIndices = + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(ex -> ex instanceof BytecodeAnalysisIndex || ex instanceof JavaAutoModuleNameIndex); + + return Stream.concat(Stream.concat(Stream.concat(coreIndices, javaIndices), xmlIndices), ktIndices); + } +} diff --git a/platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java b/platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java new file mode 100644 index 000000000000..9e8ceefbf022 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java @@ -0,0 +1,82 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.index; + +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.psi.stubs.StubUpdatingIndex; +import com.intellij.psi.stubs.provided.StubProvidedIndexExtension; +import com.intellij.util.indexing.FileBasedIndexExtension; +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.provided.ProvidedIndexExtension; +import com.intellij.util.indexing.provided.ProvidedIndexExtensionLocator; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.KeyDescriptor; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; + +public class BasicProvidedExtensionLocator implements ProvidedIndexExtensionLocator { + private static final String PREBUILT_INDEX_PATH_PROP = "prebuilt.hash.index.dir"; + + @Nullable + @Override + public ProvidedIndexExtension findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension) { + File root = getPrebuiltIndexPath(); + if (root == null || !new File(root, StringUtil.toLowerCase(originalExtension.getName().getName())).exists()) return null; + + return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID) + ? (ProvidedIndexExtension)new StubProvidedIndexExtension(root) + : new ProvidedIndexExtensionImpl<>(root, originalExtension); + } + + @Nullable + private static File getPrebuiltIndexPath() { + String path = System.getProperty(PREBUILT_INDEX_PATH_PROP); + if (path == null) return null; + File file = new File(path); + return file.exists() ? file : null; + } + + private static class ProvidedIndexExtensionImpl implements ProvidedIndexExtension { + @NotNull + private final File myIndexFile; + @NotNull + private final ID myIndexId; + @NotNull + private final KeyDescriptor myKeyDescriptor; + @NotNull + private final DataExternalizer myValueExternalizer; + + private ProvidedIndexExtensionImpl(@NotNull File file, + @NotNull FileBasedIndexExtension originalExtension) { + myIndexFile = file; + myIndexId = originalExtension.getName(); + myKeyDescriptor = originalExtension.getKeyDescriptor(); + myValueExternalizer = originalExtension.getValueExternalizer(); + } + + @NotNull + @Override + public File getIndexPath() { + return myIndexFile; + } + + @NotNull + @Override + public ID getIndexId() { + return myIndexId; + } + + @NotNull + @Override + public KeyDescriptor createKeyDescriptor() { + return myKeyDescriptor; + } + + @NotNull + @Override + public DataExternalizer createValueExternalizer() { + return myValueExternalizer; + } + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java new file mode 100644 index 000000000000..9cbf963a8259 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java @@ -0,0 +1,230 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.hash; + +import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.VfsUtilCore; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileVisitor; +import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; +import com.intellij.util.indexing.*; +import com.intellij.util.indexing.impl.InputData; +import com.intellij.util.indexing.impl.MapIndexStorage; +import com.intellij.util.indexing.impl.MapReduceIndex; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.KeyDescriptor; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +public class HashBasedIndexGenerator { + @NotNull private final File myOut; + @NotNull private final File myHashOut; + @NotNull private final FakeIndexExtension myExtension; + @NotNull private final FileBasedIndex.InputFilter myInputFilter; + + protected ContentHashesUtil.HashEnumerator myHashEnumerator; + private InvertedIndex myIndex; + + public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension indexExtension, @NotNull File out) { + this(indexExtension.getKeyDescriptor(), + indexExtension.getValueExternalizer(), + indexExtension, + out, + out); + } + + public HashBasedIndexGenerator(@NotNull KeyDescriptor keyDescriptor, + @NotNull DataExternalizer valueExternalizer, + @NotNull FileBasedIndexExtension originalExtension, + @NotNull File out, + @NotNull File hashOut) { + myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension); + myOut = out; + myHashOut = hashOut; + + FileBasedIndex.InputFilter filter = originalExtension.getInputFilter(); + + if (filter instanceof FileBasedIndex.FileTypeSpecificInputFilter) { + Set fileTypes = new HashSet<>(); + ((FileBasedIndex.FileTypeSpecificInputFilter)filter).registerFileTypesUsedForIndexing(fileTypes::add); + myInputFilter = file -> fileTypes.contains(file.getFileType()) && filter.acceptInput(file); + } else { + myInputFilter = filter; + } + } + + @NotNull + public File getOut() { + return myOut; + } + + public void openIndex() throws IOException { + myHashEnumerator = getHashEnumerator(); + String indexName = myExtension.getName().getName(); + myIndex = new MapReduceIndex(myExtension, new MapIndexStorage(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName), + myExtension.getKeyDescriptor(), + myExtension.getValueExternalizer(), + myExtension.getCacheSize(), + myExtension.keyIsUniqueForIndexedFile()) { + @Override + protected void checkCanceled() { + //ignore + } + }, null, null) { + @Override + protected void updateForwardIndex(int inputId, @NotNull InputData data) throws IOException { + super.updateForwardIndex(inputId, data); + try { + visitInputData(inputId, data); + } + catch (StorageException e) { + throw new IOException(e); + } + } + + @Override + public void checkCanceled() { + //ignore + } + + @Override + protected void requestRebuild(@NotNull Throwable e) { + throw new RuntimeException(e); + } + }; + } + + @NotNull + protected ContentHashesUtil.HashEnumerator getHashEnumerator() throws IOException { + return new ContentHashesUtil.HashEnumerator(new File(myHashOut, "hashes")); + } + + protected void visitInputData(int hashId, @NotNull InputData data) throws StorageException { + + } + + public void closeIndex() throws IOException { + if (myIndex != null) myIndex.dispose(); + if (myHashEnumerator != null) myHashEnumerator.close(); + } + + + public final void generate(@NotNull Collection roots) { + try { + openIndex(); + + for (VirtualFile root : roots) { + VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor() { + @Override + public boolean visitFile(@NotNull VirtualFile file) { + if (!file.isDirectory() && myInputFilter.acceptInput(file)) { + indexFile(file); + } + return true; + } + }); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + finally { + try { + closeIndex(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + protected void indexFile(@NotNull VirtualFile f) { + try { + FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray()); + byte[] hash = FileBasedIndexImpl.calculateHash(fc.getContent(), fc.getCharset(), f.getFileType(), fc.getFileType()); + int hashId = myHashEnumerator.enumerate(hash); + if (!myIndex.update(hashId, fc).compute()) { + throw new RuntimeException(); + } + } + catch (IOException e) { + throw new RuntimeException("cant index " + f.getPath() + " for " + myExtension.getName().getName(), e); + } + } + + public InvertedIndex getIndex() { + return myIndex; + } + + private static final class FakeIndexExtension extends FileBasedIndexExtension { + @NotNull + private final KeyDescriptor myKeyDescriptor; + @NotNull + private final DataExternalizer myValueExternalizer; + @NotNull + private final FileBasedIndexExtension myOriginalExtension; + + private FakeIndexExtension(@NotNull KeyDescriptor descriptor, + @NotNull DataExternalizer externalizer, + @NotNull FileBasedIndexExtension originalExtension) { + myKeyDescriptor = descriptor; + myValueExternalizer = externalizer; + myOriginalExtension = originalExtension; + } + + @NotNull + @Override + public ID getName() { + return myOriginalExtension.getName(); + } + + @NotNull + @Override + public FileBasedIndex.InputFilter getInputFilter() { + return myOriginalExtension.getInputFilter(); + } + + @Override + public boolean dependsOnFileContent() { + return true; + } + + @Override + public int getCacheSize() { + return myOriginalExtension.getCacheSize(); + } + + @Override + public boolean keyIsUniqueForIndexedFile() { + return myOriginalExtension.keyIsUniqueForIndexedFile(); + } + + @NotNull + @Override + public DataIndexer getIndexer() { + return myOriginalExtension.getIndexer(); + } + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return myKeyDescriptor; + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return myValueExternalizer; + } + + @Override + public int getVersion() { + return myOriginalExtension.getVersion(); + } + } +} diff --git a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java new file mode 100644 index 000000000000..74ceb672c0cb --- /dev/null +++ b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java @@ -0,0 +1,88 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.index.stubs; + +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; +import com.intellij.psi.stubs.*; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.FileBasedIndexExtension; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.indexing.hash.HashBasedIndexGenerator; +import com.intellij.util.indexing.impl.EmptyInputDataDiffBuilder; +import com.intellij.util.indexing.impl.InputData; +import com.intellij.util.indexing.impl.MapReduceIndex; +import com.intellij.util.indexing.impl.UpdateData; +import com.intellij.util.io.EnumeratorIntegerDescriptor; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator { + private final Map myStubIndexesGeneratorMap = new HashMap<>(); + private final Set myUsedKeys = new HashSet<>(); + + public StubHashBasedIndexGenerator(@NotNull File out) { + super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(false, null), getExtension(), out, out); + for (StubIndexExtension stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) { + myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(stubIndexExtension.getKeyDescriptor(), + StubIndexImpl.StubIdExternalizer.INSTANCE, + StubIndexImpl + .wrapStubIndexExtension(stubIndexExtension), + new File(out, getStubsDir()), + out) { + @NotNull + @Override + protected ContentHashesUtil.HashEnumerator getHashEnumerator() { + return StubHashBasedIndexGenerator.this.myHashEnumerator; + } + }); + } + } + + @NotNull + private static String getStubsDir() { + return StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName()); + } + + @Override + protected void visitInputData(int hashId, @NotNull InputData data) throws StorageException { + super.visitInputData(hashId, data); + SerializedStubTree tree = ContainerUtil.getFirstItem(data.getKeyValues().values()); + if (tree == null) return; + Map> map = tree.getStubIndicesValueMap(); + for (Map.Entry> entry : map.entrySet()) { + StubIndexKey key = entry.getKey(); + Map value = entry.getValue(); + myUsedKeys.add(key); + MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex(); + index.updateWithMap(new UpdateData(hashId, value, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null)); + } + } + + @Override + public void openIndex() throws IOException { + super.openIndex(); + for (HashBasedIndexGenerator generator : myStubIndexesGeneratorMap.values()) { + generator.openIndex(); + } + } + + @Override + public void closeIndex() throws IOException { + super.closeIndex(); + for (Map.Entry entry : myStubIndexesGeneratorMap.entrySet()) { + HashBasedIndexGenerator generator = entry.getValue(); + generator.closeIndex(); + } + ((SerializationManagerEx)SerializationManager.getInstance()).flushNameStorage(); + FileUtil.copyDir(PathManager.getIndexRoot(), new File(getOut(), getStubsDir()), f -> f.getName().startsWith("rep.names")); + } + + private static StubUpdatingIndex getExtension() { + return (StubUpdatingIndex)FileBasedIndexExtension.EXTENSION_POINT_NAME.extensions().filter(ex -> ex instanceof StubUpdatingIndex).findAny().get(); + } +} diff --git a/resources/src/idea/JavaActions.xml b/resources/src/idea/JavaActions.xml index f9be6a8d2d76..c2f4df2f676e 100644 --- a/resources/src/idea/JavaActions.xml +++ b/resources/src/idea/JavaActions.xml @@ -464,6 +464,9 @@ + +