mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-09-27 10:03:11 +07:00
Shared indices: add "dumpIndex" command used to generate indices for JDKs.
GitOrigin-RevId: 4049b6c92024d8cec2390ce6fe4b5f493a80d388
This commit is contained in:
Sergey Patrikeev
committed by
intellij-monorepo-bot
intellij-monorepo-bot
parent
76bacd6626
commit
403bc56a58
@@ -0,0 +1,119 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.internal
|
||||
|
||||
import com.intellij.jdkDownloader.JdkInstallRequest
|
||||
import com.intellij.jdkDownloader.JdkInstaller
|
||||
import com.intellij.jdkDownloader.JdkListDownloader
|
||||
import com.intellij.openapi.application.ApplicationStarter
|
||||
import com.intellij.openapi.diagnostic.Logger
|
||||
import com.intellij.openapi.progress.EmptyProgressIndicator
|
||||
import com.intellij.openapi.project.ProjectManager
|
||||
import com.intellij.openapi.projectRoots.JavaSdk
|
||||
import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil
|
||||
import com.intellij.openapi.roots.OrderRootType
|
||||
import com.intellij.util.io.createDirectories
|
||||
import com.intellij.util.io.delete
|
||||
import com.intellij.util.io.exists
|
||||
import com.intellij.util.io.isDirectory
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import kotlin.system.exitProcess
|
||||
|
||||
class DumpIndexStarter : ApplicationStarter {
|
||||
override fun getCommandName() = "dumpIndex"
|
||||
|
||||
private companion object {
|
||||
|
||||
private val LOG = Logger.getInstance("dumpIndex")
|
||||
|
||||
fun getRequiredProperty(name: String): String =
|
||||
System.getProperty(name) ?: throw IllegalArgumentException("'$name' is not specified")
|
||||
|
||||
val outputPath: Path by lazy {
|
||||
val path = Paths.get(getRequiredProperty("dump.index.output.path"))
|
||||
path.createDirectories()
|
||||
}
|
||||
|
||||
val removeDownloadedJdks: Boolean by lazy {
|
||||
System.getProperty("dump.index.remove.downloaded.jdks", "false")!!.toBoolean()
|
||||
}
|
||||
|
||||
fun info(message: String) {
|
||||
println(message)
|
||||
}
|
||||
}
|
||||
|
||||
private fun dumpOptions() {
|
||||
info("DumpIndex options")
|
||||
info("Output path: $outputPath")
|
||||
}
|
||||
|
||||
override fun main(args: Array<out String>) {
|
||||
dumpOptions()
|
||||
try {
|
||||
start()
|
||||
info("DumpIndex finished")
|
||||
exitProcess(0)
|
||||
}
|
||||
catch (e: Throwable) {
|
||||
info("DumpIndex failed")
|
||||
e.printStackTrace()
|
||||
exitProcess(-1)
|
||||
}
|
||||
}
|
||||
|
||||
private fun start() {
|
||||
val tempPath = outputPath.resolve("temp")
|
||||
val projectPath = tempPath.resolve("project").createDirectories()
|
||||
val jdksPath = tempPath.resolve("jdks").createDirectories()
|
||||
|
||||
info("Downloading list of available JDKs")
|
||||
val jdkItems = JdkListDownloader.downloadModelForJdkInstaller(EmptyProgressIndicator())
|
||||
info("Available JDKs: " + jdkItems.joinToString { it.fullPresentationText })
|
||||
|
||||
val project = ProjectManager.getInstance().createProject("tempProject", projectPath.toAbsolutePath().toString())
|
||||
if (project == null) {
|
||||
LOG.error("Failed to create temp project")
|
||||
return
|
||||
}
|
||||
|
||||
val jdkType = JavaSdk.getInstance()
|
||||
for (jdkItem in jdkItems) {
|
||||
val jdkPath = jdksPath.resolve(jdkItem.installFolderName)
|
||||
|
||||
if (jdkPath.isDirectory()) {
|
||||
info("JDK ${jdkItem.fullPresentationText} is already downloaded")
|
||||
}
|
||||
else {
|
||||
info("Downloading ${jdkItem.fullPresentationText} to $jdkPath")
|
||||
val jdkInstallRequest = JdkInstallRequest(jdkItem, jdkPath.toFile())
|
||||
JdkInstaller.installJdk(jdkInstallRequest, EmptyProgressIndicator())
|
||||
}
|
||||
|
||||
info("Creating SDK from ${jdkItem.fullPresentationText}")
|
||||
val sdk = SdkConfigurationUtil.createAndAddSDK(jdkPath.toAbsolutePath().toString(), jdkType)
|
||||
if (sdk == null) {
|
||||
LOG.warn("Failed to create JDK for ${jdkItem.fullPresentationText}")
|
||||
continue
|
||||
}
|
||||
|
||||
val jdkIndicesPath = outputPath.resolve(jdkItem.installFolderName)
|
||||
if (jdkIndicesPath.exists()) {
|
||||
info("Delete previous indices in $jdkIndicesPath")
|
||||
}
|
||||
|
||||
info("Building indices for ${jdkItem.fullPresentationText} to $jdkIndicesPath")
|
||||
try {
|
||||
val roots = (sdk.rootProvider.getFiles(OrderRootType.CLASSES) + sdk.rootProvider.getFiles(OrderRootType.SOURCES)).toSet()
|
||||
val indexChunk = DumpIndexAction.IndexChunk(roots, jdkItem.installFolderName)
|
||||
DumpIndexAction.exportSingleIndexChunk(project, indexChunk, jdkIndicesPath, EmptyProgressIndicator())
|
||||
}
|
||||
finally {
|
||||
SdkConfigurationUtil.removeSdk(sdk)
|
||||
if (removeDownloadedJdks) {
|
||||
jdkPath.delete()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1972,6 +1972,8 @@
|
||||
<commandLineInspectionProjectConfigurator implementation="com.intellij.codeInspection.JavaCommandLineInspectionProjectConfigurator"/>
|
||||
|
||||
<compiler.task implementation="com.intellij.execution.scratch.JavaScratchCompilationSupport"/>
|
||||
|
||||
<appStarter implementation="com.intellij.internal.DumpIndexStarter"/>
|
||||
</extensions>
|
||||
|
||||
<extensions defaultExtensionNs="org.jetbrains">
|
||||
|
||||
@@ -22,6 +22,9 @@ import com.intellij.openapi.util.io.FileUtil;
|
||||
import com.intellij.openapi.util.text.StringUtil;
|
||||
import com.intellij.openapi.vfs.VfsUtilCore;
|
||||
import com.intellij.openapi.vfs.VirtualFile;
|
||||
import com.intellij.openapi.vfs.VirtualFileVisitor;
|
||||
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
|
||||
import com.intellij.psi.SingleRootFileViewProvider;
|
||||
import com.intellij.psi.impl.JavaSimplePropertyIndex;
|
||||
import com.intellij.psi.impl.cache.impl.id.IdIndex;
|
||||
import com.intellij.psi.impl.cache.impl.todo.TodoIndex;
|
||||
@@ -35,6 +38,7 @@ import com.intellij.util.indexing.FileBasedIndexExtension;
|
||||
import com.intellij.util.indexing.ID;
|
||||
import com.intellij.util.indexing.IndexableSetContributor;
|
||||
import com.intellij.util.indexing.hash.HashBasedIndexGenerator;
|
||||
import com.intellij.util.io.PathKt;
|
||||
import com.intellij.util.io.zip.JBZipEntry;
|
||||
import com.intellij.util.io.zip.JBZipFile;
|
||||
import gnu.trove.THashSet;
|
||||
@@ -58,6 +62,36 @@ public class DumpIndexAction extends AnAction {
|
||||
public void actionPerformed(@NotNull AnActionEvent event) {
|
||||
Project project = event.getProject();
|
||||
if (project == null) return;
|
||||
|
||||
FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor();
|
||||
descriptor.withTitle("Select Index Dump Directory");
|
||||
VirtualFile file = FileChooser.chooseFile(descriptor, project, null);
|
||||
if (file != null) {
|
||||
ProgressManager.getInstance().run(new Task.Modal(project, "Exporting Indexes..." , true) {
|
||||
@Override
|
||||
public void run(@NotNull ProgressIndicator indicator) {
|
||||
File out = VfsUtilCore.virtualToIoFile(file);
|
||||
FileUtil.delete(out);
|
||||
exportIndices(project, out.toPath(), indicator);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public static void exportIndices(@NotNull Project project, @NotNull Path out, @NotNull ProgressIndicator indicator) {
|
||||
List<IndexChunk> chunks = ReadAction.compute(() -> buildChunks(project));
|
||||
exportIndices(project, chunks, out, indicator);
|
||||
}
|
||||
|
||||
public static void exportSingleIndexChunk(@NotNull Project project,
|
||||
@NotNull IndexChunk chunk,
|
||||
@NotNull Path out,
|
||||
@NotNull ProgressIndicator indicator) {
|
||||
exportIndices(project, Collections.singletonList(chunk), out, indicator);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static List<IndexChunk> buildChunks(Project project) {
|
||||
Collection<IndexChunk> projectChunks = Arrays
|
||||
.stream(ModuleManager.getInstance(project).getModules())
|
||||
.flatMap(m -> IndexChunk.generate(m))
|
||||
@@ -65,8 +99,9 @@ public class DumpIndexAction extends AnAction {
|
||||
.values();
|
||||
|
||||
Set<VirtualFile>
|
||||
additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(),
|
||||
IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())).collect(
|
||||
additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream
|
||||
.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(),
|
||||
IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())).collect(
|
||||
Collectors.toSet());
|
||||
|
||||
Set<VirtualFile> synthRoots = new THashSet<>();
|
||||
@@ -81,41 +116,35 @@ public class DumpIndexAction extends AnAction {
|
||||
}
|
||||
}
|
||||
|
||||
List<IndexChunk> chunks = Stream.concat(projectChunks.stream(),
|
||||
return Stream.concat(projectChunks.stream(),
|
||||
Stream.of(new IndexChunk(additionalRoots, "ADDITIONAL"),
|
||||
new IndexChunk(synthRoots, "SYNTH"))).collect(Collectors.toList());
|
||||
|
||||
//IndexChunk chunk = chunks.stream().reduce((c1, c2) -> IndexChunk.mergeUnsafe(c1, c2)).get();
|
||||
|
||||
FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor();
|
||||
descriptor.withTitle("Select Index Dump Directory");
|
||||
VirtualFile file = FileChooser.chooseFile(descriptor, project, null);
|
||||
if (file != null) {
|
||||
ProgressManager.getInstance().run(new Task.Modal(project, "Exporting Indexes..." , true) {
|
||||
@Override
|
||||
public void run(@NotNull ProgressIndicator indicator) {
|
||||
File out = VfsUtilCore.virtualToIoFile(file);
|
||||
FileUtil.delete(out);
|
||||
exportIndices(chunks, out, indicator, project);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public static void exportIndices(@NotNull List<IndexChunk> chunks,
|
||||
@NotNull File out,
|
||||
@NotNull ProgressIndicator indicator,
|
||||
@NotNull Project project) {
|
||||
private static void exportIndices(@NotNull Project project,
|
||||
@NotNull List<IndexChunk> chunks,
|
||||
@NotNull Path out,
|
||||
@NotNull ProgressIndicator indicator) {
|
||||
indicator.setIndeterminate(false);
|
||||
AtomicInteger idx = new AtomicInteger();
|
||||
if (!JobLauncher.getInstance().invokeConcurrentlyUnderProgress(chunks, indicator, chunk -> {
|
||||
indicator.setText("Indexing '" + chunk.getName() + "' chunk");
|
||||
File chunkOut = new File(out, chunk.getName());
|
||||
indicator.setText("Indexing chunk " + chunk.getName());
|
||||
Path chunkRoot = out.resolve(chunk.getName());
|
||||
ReadAction.run(() -> {
|
||||
Stream<HashBasedIndexGenerator<?, ?>> fbIndexes = getExportableIndices(true).map(ex -> new HashBasedIndexGenerator(ex, chunkOut));
|
||||
Stream<HashBasedIndexGenerator<?, ?>> stubIndex = Stream.of(new StubHashBasedIndexGenerator(chunkOut));
|
||||
List<HashBasedIndexGenerator<?, ?>> indexes = Stream.concat(fbIndexes, stubIndex).collect(Collectors.toList());
|
||||
HashBasedIndexGenerator.generate(chunk.getRoots(),indexes, project, chunkOut);
|
||||
List<HashBasedIndexGenerator<?, ?>> fileBasedGenerators = getExportableIndices(true)
|
||||
.map(extension -> getGenerator(chunkRoot, extension))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
StubHashBasedIndexGenerator stubGenerator = new StubHashBasedIndexGenerator(chunkRoot);
|
||||
|
||||
List<HashBasedIndexGenerator<?, ?>> allGenerators = new ArrayList<>(fileBasedGenerators);
|
||||
allGenerators.add(stubGenerator);
|
||||
|
||||
generate(chunk, project, allGenerators, chunkRoot);
|
||||
|
||||
printStatistics(chunk, fileBasedGenerators, stubGenerator);
|
||||
deleteEmptyIndices(fileBasedGenerators, chunkRoot.resolve("empty-indices.txt"));
|
||||
deleteEmptyIndices(stubGenerator.getStubGenerators(), chunkRoot.resolve("empty-stub-indices.txt"));
|
||||
});
|
||||
indicator.setFraction(((double) idx.incrementAndGet()) / chunks.size());
|
||||
return true;
|
||||
@@ -123,16 +152,47 @@ public class DumpIndexAction extends AnAction {
|
||||
throw new AssertionError();
|
||||
}
|
||||
|
||||
zipIndexOut(out, indicator);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static <K, V> HashBasedIndexGenerator<K, V> getGenerator(Path chunkRoot, FileBasedIndexExtension<K, V> extension) {
|
||||
return new HashBasedIndexGenerator<>(extension, chunkRoot);
|
||||
}
|
||||
|
||||
private static void deleteEmptyIndices(@NotNull List<HashBasedIndexGenerator<?, ?>> generators,
|
||||
@NotNull Path dumpEmptyIndicesNamesFile) {
|
||||
Set<String> emptyIndices = new HashSet<>();
|
||||
for (HashBasedIndexGenerator<?, ?> generator : generators) {
|
||||
if (generator.isEmpty()) {
|
||||
emptyIndices.add(generator.getExtension().getName().getName());
|
||||
Path indexRoot = generator.getIndexRoot();
|
||||
PathKt.delete(indexRoot);
|
||||
}
|
||||
}
|
||||
if (!emptyIndices.isEmpty()) {
|
||||
String emptyIndicesText = String.join("\n", emptyIndices);
|
||||
try {
|
||||
PathKt.write(dumpEmptyIndicesNamesFile, emptyIndicesText);
|
||||
}
|
||||
catch (IOException e) {
|
||||
LOG.error(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void zipIndexOut(@NotNull Path out, @NotNull ProgressIndicator indicator) {
|
||||
indicator.setIndeterminate(true);
|
||||
indicator.setText("Zipping index pack");
|
||||
|
||||
File zipFile = new File(out.getAbsolutePath() + ".zip");
|
||||
FileUtil.delete(zipFile);
|
||||
File zipFile = new File(out.toFile().getAbsolutePath() + ".zip");
|
||||
if (zipFile.exists()) {
|
||||
FileUtil.delete(zipFile);
|
||||
}
|
||||
try (JBZipFile file = new JBZipFile(zipFile)) {
|
||||
Path outPath = out.toPath();
|
||||
Files.walk(outPath).forEach(p -> {
|
||||
Files.walk(out).forEach(p -> {
|
||||
if (Files.isDirectory(p)) return;
|
||||
String relativePath = outPath.relativize(p).toString();
|
||||
String relativePath = out.relativize(p).toString();
|
||||
try {
|
||||
JBZipEntry entry = file.getOrCreateEntry(relativePath);
|
||||
entry.setMethod(ZipEntry.STORED);
|
||||
@@ -148,8 +208,90 @@ public class DumpIndexAction extends AnAction {
|
||||
}
|
||||
}
|
||||
|
||||
private static void generate(@NotNull IndexChunk chunk,
|
||||
@NotNull Project project,
|
||||
@NotNull Collection<HashBasedIndexGenerator<?, ?>> generators,
|
||||
@NotNull Path chunkOut) {
|
||||
try {
|
||||
ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(chunkOut.resolve("hashes"));
|
||||
try {
|
||||
for (HashBasedIndexGenerator<?, ?> generator : generators) {
|
||||
generator.openIndex();
|
||||
}
|
||||
|
||||
for (VirtualFile root : chunk.getRoots()) {
|
||||
VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor<Boolean>() {
|
||||
@Override
|
||||
public boolean visitFile(@NotNull VirtualFile file) {
|
||||
|
||||
if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) {
|
||||
for (HashBasedIndexGenerator<?, ?> generator : generators) {
|
||||
generator.indexFile(file, project, hashEnumerator);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
finally {
|
||||
hashEnumerator.close();
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
finally {
|
||||
try {
|
||||
for (HashBasedIndexGenerator<?, ?> generator : generators) {
|
||||
generator.closeIndex();
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void printStatistics(@NotNull IndexChunk chunk,
|
||||
@NotNull List<HashBasedIndexGenerator<?, ?>> fileBasedGenerators,
|
||||
@NotNull StubHashBasedIndexGenerator stubGenerator) {
|
||||
StringBuilder stats = new StringBuilder();
|
||||
stats.append("Statistics for index chunk ").append(chunk.getName()).append("\n");
|
||||
|
||||
stats.append("File based indices (").append(fileBasedGenerators.size()).append(")");
|
||||
for (HashBasedIndexGenerator<?, ?> generator : fileBasedGenerators) {
|
||||
appendGeneratorStatistics(stats, generator);
|
||||
}
|
||||
|
||||
Collection<HashBasedIndexGenerator<?, ?>> stubGenerators = stubGenerator.getStubGenerators();
|
||||
stats.append("Stub indices (").append(stubGenerators.size()).append(")");
|
||||
for (HashBasedIndexGenerator<?, ?> generator : stubGenerators) {
|
||||
appendGeneratorStatistics(stats, generator);
|
||||
}
|
||||
|
||||
System.out.println(stats.toString());
|
||||
}
|
||||
|
||||
private static void appendGeneratorStatistics(@NotNull StringBuilder stringBuilder,
|
||||
@NotNull HashBasedIndexGenerator<?, ?> generator) {
|
||||
stringBuilder
|
||||
.append(" ")
|
||||
.append("Generator for ")
|
||||
.append(generator.getExtension().getName().getName())
|
||||
.append(" indexed ")
|
||||
.append(generator.getIndexedFilesNumber())
|
||||
.append(" files");
|
||||
|
||||
if (generator.isEmpty()) {
|
||||
stringBuilder.append( "(empty result)");
|
||||
}
|
||||
stringBuilder.append("\n");
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static Stream<FileBasedIndexExtension> getExportableIndices(boolean all) {
|
||||
private static Stream<FileBasedIndexExtension<?, ?>> getExportableIndices(boolean all) {
|
||||
if (all) {
|
||||
return FileBasedIndexExtension
|
||||
.EXTENSION_POINT_NAME
|
||||
@@ -159,7 +301,7 @@ public class DumpIndexAction extends AnAction {
|
||||
}
|
||||
|
||||
//kt
|
||||
Stream<FileBasedIndexExtension> ktIndices =
|
||||
Stream<FileBasedIndexExtension<?, ?>> ktIndices =
|
||||
FileBasedIndexExtension
|
||||
.EXTENSION_POINT_NAME
|
||||
.extensions()
|
||||
@@ -171,14 +313,14 @@ public class DumpIndexAction extends AnAction {
|
||||
ID.findByName("DomFileIndex"),
|
||||
ID.findByName("xmlProperties"));
|
||||
//xml
|
||||
Stream<FileBasedIndexExtension> xmlIndices =
|
||||
Stream<FileBasedIndexExtension<?, ?>> xmlIndices =
|
||||
FileBasedIndexExtension
|
||||
.EXTENSION_POINT_NAME
|
||||
.extensions()
|
||||
.filter(id -> xmlIndexIds.contains(id.getName()));
|
||||
|
||||
//base
|
||||
Stream<FileBasedIndexExtension> coreIndices =
|
||||
Stream<FileBasedIndexExtension<?, ?>> coreIndices =
|
||||
FileBasedIndexExtension
|
||||
.EXTENSION_POINT_NAME
|
||||
.extensions()
|
||||
@@ -188,7 +330,7 @@ public class DumpIndexAction extends AnAction {
|
||||
ex.getName().getName().equals("HashFragmentIndex"));
|
||||
|
||||
//java
|
||||
Stream<FileBasedIndexExtension> javaIndices =
|
||||
Stream<FileBasedIndexExtension<?, ?>> javaIndices =
|
||||
FileBasedIndexExtension
|
||||
.EXTENSION_POINT_NAME
|
||||
.extensions()
|
||||
@@ -202,11 +344,11 @@ public class DumpIndexAction extends AnAction {
|
||||
return Stream.concat(Stream.concat(Stream.concat(coreIndices, javaIndices), xmlIndices), ktIndices);
|
||||
}
|
||||
|
||||
private static final class IndexChunk {
|
||||
public static final class IndexChunk {
|
||||
private final Set<VirtualFile> myRoots;
|
||||
private final String myName;
|
||||
|
||||
IndexChunk(Set<VirtualFile> roots, String name) {
|
||||
public IndexChunk(Set<VirtualFile> roots, String name) {
|
||||
myRoots = roots;
|
||||
myName = name;
|
||||
}
|
||||
|
||||
+69
-92
@@ -4,12 +4,10 @@ package com.intellij.util.indexing.hash;
|
||||
import com.intellij.openapi.fileTypes.FileType;
|
||||
import com.intellij.openapi.project.Project;
|
||||
import com.intellij.openapi.util.text.StringUtil;
|
||||
import com.intellij.openapi.vfs.VfsUtilCore;
|
||||
import com.intellij.openapi.vfs.VirtualFile;
|
||||
import com.intellij.openapi.vfs.VirtualFileVisitor;
|
||||
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
|
||||
import com.intellij.psi.SingleRootFileViewProvider;
|
||||
import com.intellij.util.indexing.*;
|
||||
import com.intellij.util.indexing.impl.IndexStorage;
|
||||
import com.intellij.util.indexing.impl.InputData;
|
||||
import com.intellij.util.indexing.impl.MapIndexStorage;
|
||||
import com.intellij.util.indexing.impl.MapReduceIndex;
|
||||
@@ -18,35 +16,41 @@ import com.intellij.util.io.DataExternalizer;
|
||||
import com.intellij.util.io.KeyDescriptor;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class HashBasedIndexGenerator<K, V> {
|
||||
@NotNull
|
||||
private final File myOut;
|
||||
@NotNull
|
||||
private final FakeIndexExtension<K, V> myExtension;
|
||||
@NotNull
|
||||
private final FileBasedIndex.InputFilter myInputFilter;
|
||||
|
||||
private final Path myStorageFile;
|
||||
|
||||
private final AtomicInteger myIndexedFilesNumber = new AtomicInteger();
|
||||
|
||||
private final AtomicBoolean myIsEmpty = new AtomicBoolean(true);
|
||||
|
||||
private InvertedIndex<K, V, FileContent> myIndex;
|
||||
|
||||
public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension<K, V> indexExtension, @NotNull File out) {
|
||||
this(indexExtension.getKeyDescriptor(),
|
||||
indexExtension.getValueExternalizer(),
|
||||
indexExtension,
|
||||
out
|
||||
);
|
||||
public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension<K, V> indexExtension, @NotNull Path outRoot) {
|
||||
this(indexExtension.getKeyDescriptor(), indexExtension.getValueExternalizer(), indexExtension, outRoot);
|
||||
}
|
||||
|
||||
public HashBasedIndexGenerator(@NotNull KeyDescriptor<K> keyDescriptor,
|
||||
@NotNull DataExternalizer<V> valueExternalizer,
|
||||
@NotNull FileBasedIndexExtension<K, V> originalExtension,
|
||||
@NotNull File out) {
|
||||
@NotNull Path outRoot) {
|
||||
myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension);
|
||||
myOut = out;
|
||||
|
||||
String indexName = originalExtension.getName().getName();
|
||||
myStorageFile = outRoot.resolve(StringUtil.toLowerCase(indexName)).resolve(indexName);
|
||||
|
||||
FileBasedIndex.InputFilter filter = originalExtension.getInputFilter();
|
||||
|
||||
@@ -54,33 +58,41 @@ public class HashBasedIndexGenerator<K, V> {
|
||||
Set<FileType> fileTypes = new HashSet<>();
|
||||
((FileBasedIndex.FileTypeSpecificInputFilter)filter).registerFileTypesUsedForIndexing(fileTypes::add);
|
||||
myInputFilter = file -> fileTypes.contains(file.getFileType()) && filter.acceptInput(file);
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
myInputFilter = filter;
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public File getOut() {
|
||||
return myOut;
|
||||
}
|
||||
|
||||
public void openIndex() throws IOException {
|
||||
String indexName = myExtension.getName().getName();
|
||||
boolean singleEntry = myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension;
|
||||
myIndex = new MapReduceIndex<K, V, FileContent>(myExtension, new MapIndexStorage<K, V>(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName).toPath(),
|
||||
myExtension.getKeyDescriptor(),
|
||||
myExtension.getValueExternalizer(),
|
||||
myExtension.getCacheSize(),
|
||||
myExtension.keyIsUniqueForIndexedFile()) {
|
||||
IndexStorage<K, V> indexStorage = new MapIndexStorage<K, V>(myStorageFile,
|
||||
myExtension.getKeyDescriptor(),
|
||||
myExtension.getValueExternalizer(),
|
||||
myExtension.getCacheSize(),
|
||||
myExtension.keyIsUniqueForIndexedFile()) {
|
||||
@Override
|
||||
protected void checkCanceled() {
|
||||
//ignore
|
||||
}
|
||||
}, null, null) {
|
||||
|
||||
@Override
|
||||
public void addValue(K k, int inputId, V v) throws StorageException {
|
||||
super.addValue(k, inputId, v);
|
||||
myIsEmpty.set(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeAllValues(@NotNull K k, int inputId) {
|
||||
throw new AssertionError("Must not happen in index generator");
|
||||
}
|
||||
};
|
||||
|
||||
myIndex = new MapReduceIndex<K, V, FileContent>(myExtension, indexStorage, null, null) {
|
||||
@NotNull
|
||||
@Override
|
||||
protected Map<K, V> mapByIndexer(int inputId, @NotNull FileContent content) {
|
||||
Map<K, V> data = super.mapByIndexer(inputId, content);
|
||||
boolean singleEntry = HashBasedIndexGenerator.this.myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension;
|
||||
if (singleEntry && !data.isEmpty()) {
|
||||
data = Collections.singletonMap((K)(Integer)inputId, data.values().iterator().next());
|
||||
}
|
||||
@@ -105,7 +117,7 @@ public class HashBasedIndexGenerator<K, V> {
|
||||
|
||||
@Override
|
||||
protected void requestRebuild(@NotNull Throwable e) {
|
||||
throw new RuntimeException("error while processing " + indexName, e);
|
||||
throw new RuntimeException("Error while processing " + myExtension.getName().getName(), e);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -115,82 +127,47 @@ public class HashBasedIndexGenerator<K, V> {
|
||||
}
|
||||
|
||||
public void closeIndex() throws IOException {
|
||||
if (myIndex != null) myIndex.dispose();
|
||||
if (myIndex != null) {
|
||||
myIndex.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public static void generate(@NotNull Collection<VirtualFile> roots,
|
||||
@NotNull Collection<HashBasedIndexGenerator<?, ?>> generators,
|
||||
@NotNull Project project,
|
||||
@NotNull File hashOut) {
|
||||
|
||||
LongAdder l = new LongAdder();
|
||||
try {
|
||||
ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(new File(hashOut, "hashes").toPath());
|
||||
|
||||
for (HashBasedIndexGenerator<?, ?> generator : generators) {
|
||||
generator.openIndex();
|
||||
}
|
||||
|
||||
for (VirtualFile root : roots) {
|
||||
VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor<Boolean>() {
|
||||
@Override
|
||||
public boolean visitFile(@NotNull VirtualFile file) {
|
||||
|
||||
if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) {
|
||||
for (HashBasedIndexGenerator<?, ?> generator : generators) {
|
||||
if (generator.myInputFilter.acceptInput(file)) {
|
||||
l.increment();
|
||||
generator.indexFile(file, project, hashEnumerator);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
synchronized (hashEnumerator) {
|
||||
hashEnumerator.close();
|
||||
}
|
||||
public void indexFile(@NotNull VirtualFile f, @NotNull Project project, @NotNull ContentHashesUtil.HashEnumerator hashEnumerator) {
|
||||
if (!myInputFilter.acceptInput(f)) {
|
||||
return;
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
finally {
|
||||
try {
|
||||
for (HashBasedIndexGenerator<?, ?> generator : generators) {
|
||||
generator.closeIndex();
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println("Indexed " + l.sum() + " files to " + hashOut.getPath());
|
||||
}
|
||||
|
||||
protected void indexFile(@NotNull VirtualFile f,
|
||||
@NotNull Project project,
|
||||
@NotNull ContentHashesUtil.HashEnumerator hashEnumerator) {
|
||||
myIndexedFilesNumber.incrementAndGet();
|
||||
try {
|
||||
FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray());
|
||||
byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(fc, false);
|
||||
int hashId;
|
||||
synchronized (hashEnumerator) {
|
||||
hashId = Math.abs(hashEnumerator.enumerate(hash));
|
||||
}
|
||||
int hashId = Math.abs(hashEnumerator.enumerate(hash));
|
||||
fc.putUserData(IndexingDataKeys.PROJECT, project);
|
||||
if (!myIndex.update(hashId, fc).compute()) {
|
||||
throw new RuntimeException();
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException("cant index " + f.getPath() + " for " + myExtension.getName().getName(), e);
|
||||
throw new RuntimeException("Can't index " + f.getPath() + " for " + myExtension.getName().getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public Path getIndexRoot() {
|
||||
return myStorageFile.getParent();
|
||||
}
|
||||
|
||||
public int getIndexedFilesNumber() {
|
||||
return myIndexedFilesNumber.get();
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
return myIsEmpty.get();
|
||||
}
|
||||
|
||||
public FileBasedIndexExtension<K, V> getExtension() {
|
||||
return myExtension;
|
||||
}
|
||||
|
||||
public InvertedIndex<K, V, FileContent> getIndex() {
|
||||
return myIndex;
|
||||
}
|
||||
|
||||
@@ -17,35 +17,44 @@ import com.intellij.util.indexing.impl.UpdateData;
|
||||
import com.intellij.util.io.EnumeratorIntegerDescriptor;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer, SerializedStubTree> {
|
||||
private final Map<StubIndexKey, HashBasedIndexGenerator> myStubIndexesGeneratorMap = new HashMap<>();
|
||||
private final Set<StubIndexKey> myUsedKeys = new HashSet<>();
|
||||
|
||||
public StubHashBasedIndexGenerator(@NotNull File out) {
|
||||
super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(
|
||||
true,
|
||||
null,
|
||||
StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), getExtension(), out);
|
||||
private final Map<StubIndexKey<?, ?>, HashBasedIndexGenerator<?, ?>> myStubIndexesGeneratorMap = new HashMap<>();
|
||||
|
||||
private final Path myStubIndicesRoot;
|
||||
|
||||
public StubHashBasedIndexGenerator(@NotNull Path outRoot) {
|
||||
super(
|
||||
EnumeratorIntegerDescriptor.INSTANCE,
|
||||
new SerializedStubTreeDataExternalizer(true, null, StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE),
|
||||
new StubUpdatingIndex(StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE),
|
||||
outRoot
|
||||
);
|
||||
|
||||
myStubIndicesRoot = outRoot.resolve(StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName()));
|
||||
|
||||
for (StubIndexExtension<?, ?> stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) {
|
||||
FileBasedIndexExtension<?, Void> ex = StubIndexImpl
|
||||
.wrapStubIndexExtension(stubIndexExtension);
|
||||
myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(ex.getKeyDescriptor(),
|
||||
ex.getValueExternalizer(),
|
||||
ex,
|
||||
new File(out, getStubsDir())
|
||||
) {
|
||||
|
||||
});
|
||||
FileBasedIndexExtension<?, Void> extension = StubIndexImpl.wrapStubIndexExtension(stubIndexExtension);
|
||||
HashBasedIndexGenerator<?, Void> hashBasedIndexGenerator = createGenerator(extension);
|
||||
myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), hashBasedIndexGenerator);
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static String getStubsDir() {
|
||||
return StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName());
|
||||
private <K, V> HashBasedIndexGenerator<K, V> createGenerator(FileBasedIndexExtension<K, V> extension) {
|
||||
return new HashBasedIndexGenerator<>(extension.getKeyDescriptor(), extension.getValueExternalizer(), extension, myStubIndicesRoot);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public List<HashBasedIndexGenerator<?, ?>> getStubGenerators() {
|
||||
return new ArrayList<>(myStubIndexesGeneratorMap.values());
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -57,17 +66,23 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
|
||||
for (Map.Entry<StubIndexKey, Map<Object, StubIdList>> entry : map.entrySet()) {
|
||||
StubIndexKey key = entry.getKey();
|
||||
Map<Object, StubIdList> value = entry.getValue();
|
||||
myUsedKeys.add(key);
|
||||
MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex();
|
||||
Map<Object, Object> reducedValue = Maps.asMap(value.keySet(), k -> null);
|
||||
index.updateWithMap(new UpdateData(hashId, reducedValue, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null));
|
||||
UpdateData<?, ?> updateData = new UpdateData(
|
||||
hashId,
|
||||
reducedValue,
|
||||
() -> new EmptyInputDataDiffBuilder(hashId),
|
||||
index.getExtension().getName(),
|
||||
null
|
||||
);
|
||||
index.updateWithMap(updateData);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void openIndex() throws IOException {
|
||||
super.openIndex();
|
||||
for (HashBasedIndexGenerator generator : myStubIndexesGeneratorMap.values()) {
|
||||
for (HashBasedIndexGenerator<?, ?> generator : myStubIndexesGeneratorMap.values()) {
|
||||
generator.openIndex();
|
||||
}
|
||||
}
|
||||
@@ -75,15 +90,11 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
|
||||
@Override
|
||||
public void closeIndex() throws IOException {
|
||||
super.closeIndex();
|
||||
for (Map.Entry<StubIndexKey, HashBasedIndexGenerator> entry : myStubIndexesGeneratorMap.entrySet()) {
|
||||
HashBasedIndexGenerator generator = entry.getValue();
|
||||
generator.closeIndex();
|
||||
for (Map.Entry<StubIndexKey<?, ?>, HashBasedIndexGenerator<?, ?>> entry : myStubIndexesGeneratorMap.entrySet()) {
|
||||
entry.getValue().closeIndex();
|
||||
}
|
||||
((SerializationManagerEx)SerializationManager.getInstance()).flushNameStorage();
|
||||
FileUtil.copyDir(PathManager.getIndexRoot(), new File(getOut(), getStubsDir()), f -> f.getName().startsWith("rep.names"));
|
||||
}
|
||||
|
||||
private static StubUpdatingIndex getExtension() {
|
||||
return (new StubUpdatingIndex(StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE));
|
||||
FileUtil.copyDir(PathManager.getIndexRoot(), myStubIndicesRoot.toFile(), f -> f.getName().startsWith("rep.names"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user