From 403bc56a58f2536c8dbc0b3c5dc780a534cdf6b0 Mon Sep 17 00:00:00 2001 From: Sergey Patrikeev Date: Fri, 27 Dec 2019 16:42:18 +0300 Subject: [PATCH] Shared indices: add "dumpIndex" command used to generate indices for JDKs. GitOrigin-RevId: 4049b6c92024d8cec2390ce6fe4b5f493a80d388 --- .../com/intellij/internal/DumpIndexStarter.kt | 119 ++++++++++ java/java-impl/src/META-INF/JavaPlugin.xml | 2 + .../intellij/internal/DumpIndexAction.java | 224 ++++++++++++++---- .../hash/HashBasedIndexGenerator.java | 161 ++++++------- .../stubs/StubHashBasedIndexGenerator.java | 71 +++--- 5 files changed, 414 insertions(+), 163 deletions(-) create mode 100644 java/idea-ui/src/com/intellij/internal/DumpIndexStarter.kt diff --git a/java/idea-ui/src/com/intellij/internal/DumpIndexStarter.kt b/java/idea-ui/src/com/intellij/internal/DumpIndexStarter.kt new file mode 100644 index 000000000000..fa8f4ee07e31 --- /dev/null +++ b/java/idea-ui/src/com/intellij/internal/DumpIndexStarter.kt @@ -0,0 +1,119 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal + +import com.intellij.jdkDownloader.JdkInstallRequest +import com.intellij.jdkDownloader.JdkInstaller +import com.intellij.jdkDownloader.JdkListDownloader +import com.intellij.openapi.application.ApplicationStarter +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.progress.EmptyProgressIndicator +import com.intellij.openapi.project.ProjectManager +import com.intellij.openapi.projectRoots.JavaSdk +import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil +import com.intellij.openapi.roots.OrderRootType +import com.intellij.util.io.createDirectories +import com.intellij.util.io.delete +import com.intellij.util.io.exists +import com.intellij.util.io.isDirectory +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.system.exitProcess + +class DumpIndexStarter : ApplicationStarter { + override fun getCommandName() = "dumpIndex" + + private companion object { + + private val LOG = Logger.getInstance("dumpIndex") + + fun getRequiredProperty(name: String): String = + System.getProperty(name) ?: throw IllegalArgumentException("'$name' is not specified") + + val outputPath: Path by lazy { + val path = Paths.get(getRequiredProperty("dump.index.output.path")) + path.createDirectories() + } + + val removeDownloadedJdks: Boolean by lazy { + System.getProperty("dump.index.remove.downloaded.jdks", "false")!!.toBoolean() + } + + fun info(message: String) { + println(message) + } + } + + private fun dumpOptions() { + info("DumpIndex options") + info("Output path: $outputPath") + } + + override fun main(args: Array) { + dumpOptions() + try { + start() + info("DumpIndex finished") + exitProcess(0) + } + catch (e: Throwable) { + info("DumpIndex failed") + e.printStackTrace() + exitProcess(-1) + } + } + + private fun start() { + val tempPath = outputPath.resolve("temp") + val projectPath = tempPath.resolve("project").createDirectories() + val jdksPath = tempPath.resolve("jdks").createDirectories() + + info("Downloading list of available JDKs") + val jdkItems = JdkListDownloader.downloadModelForJdkInstaller(EmptyProgressIndicator()) + info("Available JDKs: " + jdkItems.joinToString { it.fullPresentationText }) + + val project = ProjectManager.getInstance().createProject("tempProject", projectPath.toAbsolutePath().toString()) + if (project == null) { + LOG.error("Failed to create temp project") + return + } + + val jdkType = JavaSdk.getInstance() + for (jdkItem in jdkItems) { + val jdkPath = jdksPath.resolve(jdkItem.installFolderName) + + if (jdkPath.isDirectory()) { + info("JDK ${jdkItem.fullPresentationText} is already downloaded") + } + else { + info("Downloading ${jdkItem.fullPresentationText} to $jdkPath") + val jdkInstallRequest = JdkInstallRequest(jdkItem, jdkPath.toFile()) + JdkInstaller.installJdk(jdkInstallRequest, EmptyProgressIndicator()) + } + + info("Creating SDK from ${jdkItem.fullPresentationText}") + val sdk = SdkConfigurationUtil.createAndAddSDK(jdkPath.toAbsolutePath().toString(), jdkType) + if (sdk == null) { + LOG.warn("Failed to create JDK for ${jdkItem.fullPresentationText}") + continue + } + + val jdkIndicesPath = outputPath.resolve(jdkItem.installFolderName) + if (jdkIndicesPath.exists()) { + info("Delete previous indices in $jdkIndicesPath") + } + + info("Building indices for ${jdkItem.fullPresentationText} to $jdkIndicesPath") + try { + val roots = (sdk.rootProvider.getFiles(OrderRootType.CLASSES) + sdk.rootProvider.getFiles(OrderRootType.SOURCES)).toSet() + val indexChunk = DumpIndexAction.IndexChunk(roots, jdkItem.installFolderName) + DumpIndexAction.exportSingleIndexChunk(project, indexChunk, jdkIndicesPath, EmptyProgressIndicator()) + } + finally { + SdkConfigurationUtil.removeSdk(sdk) + if (removeDownloadedJdks) { + jdkPath.delete() + } + } + } + } +} \ No newline at end of file diff --git a/java/java-impl/src/META-INF/JavaPlugin.xml b/java/java-impl/src/META-INF/JavaPlugin.xml index a09d18a9cccf..4b17c0247dcd 100644 --- a/java/java-impl/src/META-INF/JavaPlugin.xml +++ b/java/java-impl/src/META-INF/JavaPlugin.xml @@ -1972,6 +1972,8 @@ + + diff --git a/java/java-impl/src/com/intellij/internal/DumpIndexAction.java b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java index ce6033066211..a494863e65ca 100644 --- a/java/java-impl/src/com/intellij/internal/DumpIndexAction.java +++ b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java @@ -22,6 +22,9 @@ import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileVisitor; +import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; +import com.intellij.psi.SingleRootFileViewProvider; import com.intellij.psi.impl.JavaSimplePropertyIndex; import com.intellij.psi.impl.cache.impl.id.IdIndex; import com.intellij.psi.impl.cache.impl.todo.TodoIndex; @@ -35,6 +38,7 @@ import com.intellij.util.indexing.FileBasedIndexExtension; import com.intellij.util.indexing.ID; import com.intellij.util.indexing.IndexableSetContributor; import com.intellij.util.indexing.hash.HashBasedIndexGenerator; +import com.intellij.util.io.PathKt; import com.intellij.util.io.zip.JBZipEntry; import com.intellij.util.io.zip.JBZipFile; import gnu.trove.THashSet; @@ -58,6 +62,36 @@ public class DumpIndexAction extends AnAction { public void actionPerformed(@NotNull AnActionEvent event) { Project project = event.getProject(); if (project == null) return; + + FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor(); + descriptor.withTitle("Select Index Dump Directory"); + VirtualFile file = FileChooser.chooseFile(descriptor, project, null); + if (file != null) { + ProgressManager.getInstance().run(new Task.Modal(project, "Exporting Indexes..." , true) { + @Override + public void run(@NotNull ProgressIndicator indicator) { + File out = VfsUtilCore.virtualToIoFile(file); + FileUtil.delete(out); + exportIndices(project, out.toPath(), indicator); + } + }); + } + } + + public static void exportIndices(@NotNull Project project, @NotNull Path out, @NotNull ProgressIndicator indicator) { + List chunks = ReadAction.compute(() -> buildChunks(project)); + exportIndices(project, chunks, out, indicator); + } + + public static void exportSingleIndexChunk(@NotNull Project project, + @NotNull IndexChunk chunk, + @NotNull Path out, + @NotNull ProgressIndicator indicator) { + exportIndices(project, Collections.singletonList(chunk), out, indicator); + } + + @NotNull + private static List buildChunks(Project project) { Collection projectChunks = Arrays .stream(ModuleManager.getInstance(project).getModules()) .flatMap(m -> IndexChunk.generate(m)) @@ -65,8 +99,9 @@ public class DumpIndexAction extends AnAction { .values(); Set - additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(), - IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())).collect( + additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream + .concat(IndexableSetContributor.getRootsToIndex(contributor).stream(), + IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())).collect( Collectors.toSet()); Set synthRoots = new THashSet<>(); @@ -81,41 +116,35 @@ public class DumpIndexAction extends AnAction { } } - List chunks = Stream.concat(projectChunks.stream(), + return Stream.concat(projectChunks.stream(), Stream.of(new IndexChunk(additionalRoots, "ADDITIONAL"), new IndexChunk(synthRoots, "SYNTH"))).collect(Collectors.toList()); - - //IndexChunk chunk = chunks.stream().reduce((c1, c2) -> IndexChunk.mergeUnsafe(c1, c2)).get(); - - FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor(); - descriptor.withTitle("Select Index Dump Directory"); - VirtualFile file = FileChooser.chooseFile(descriptor, project, null); - if (file != null) { - ProgressManager.getInstance().run(new Task.Modal(project, "Exporting Indexes..." , true) { - @Override - public void run(@NotNull ProgressIndicator indicator) { - File out = VfsUtilCore.virtualToIoFile(file); - FileUtil.delete(out); - exportIndices(chunks, out, indicator, project); - } - }); - } } - public static void exportIndices(@NotNull List chunks, - @NotNull File out, - @NotNull ProgressIndicator indicator, - @NotNull Project project) { + private static void exportIndices(@NotNull Project project, + @NotNull List chunks, + @NotNull Path out, + @NotNull ProgressIndicator indicator) { indicator.setIndeterminate(false); AtomicInteger idx = new AtomicInteger(); if (!JobLauncher.getInstance().invokeConcurrentlyUnderProgress(chunks, indicator, chunk -> { - indicator.setText("Indexing '" + chunk.getName() + "' chunk"); - File chunkOut = new File(out, chunk.getName()); + indicator.setText("Indexing chunk " + chunk.getName()); + Path chunkRoot = out.resolve(chunk.getName()); ReadAction.run(() -> { - Stream> fbIndexes = getExportableIndices(true).map(ex -> new HashBasedIndexGenerator(ex, chunkOut)); - Stream> stubIndex = Stream.of(new StubHashBasedIndexGenerator(chunkOut)); - List> indexes = Stream.concat(fbIndexes, stubIndex).collect(Collectors.toList()); - HashBasedIndexGenerator.generate(chunk.getRoots(),indexes, project, chunkOut); + List> fileBasedGenerators = getExportableIndices(true) + .map(extension -> getGenerator(chunkRoot, extension)) + .collect(Collectors.toList()); + + StubHashBasedIndexGenerator stubGenerator = new StubHashBasedIndexGenerator(chunkRoot); + + List> allGenerators = new ArrayList<>(fileBasedGenerators); + allGenerators.add(stubGenerator); + + generate(chunk, project, allGenerators, chunkRoot); + + printStatistics(chunk, fileBasedGenerators, stubGenerator); + deleteEmptyIndices(fileBasedGenerators, chunkRoot.resolve("empty-indices.txt")); + deleteEmptyIndices(stubGenerator.getStubGenerators(), chunkRoot.resolve("empty-stub-indices.txt")); }); indicator.setFraction(((double) idx.incrementAndGet()) / chunks.size()); return true; @@ -123,16 +152,47 @@ public class DumpIndexAction extends AnAction { throw new AssertionError(); } + zipIndexOut(out, indicator); + } + + @NotNull + private static HashBasedIndexGenerator getGenerator(Path chunkRoot, FileBasedIndexExtension extension) { + return new HashBasedIndexGenerator<>(extension, chunkRoot); + } + + private static void deleteEmptyIndices(@NotNull List> generators, + @NotNull Path dumpEmptyIndicesNamesFile) { + Set emptyIndices = new HashSet<>(); + for (HashBasedIndexGenerator generator : generators) { + if (generator.isEmpty()) { + emptyIndices.add(generator.getExtension().getName().getName()); + Path indexRoot = generator.getIndexRoot(); + PathKt.delete(indexRoot); + } + } + if (!emptyIndices.isEmpty()) { + String emptyIndicesText = String.join("\n", emptyIndices); + try { + PathKt.write(dumpEmptyIndicesNamesFile, emptyIndicesText); + } + catch (IOException e) { + LOG.error(e); + } + } + } + + private static void zipIndexOut(@NotNull Path out, @NotNull ProgressIndicator indicator) { indicator.setIndeterminate(true); indicator.setText("Zipping index pack"); - File zipFile = new File(out.getAbsolutePath() + ".zip"); - FileUtil.delete(zipFile); + File zipFile = new File(out.toFile().getAbsolutePath() + ".zip"); + if (zipFile.exists()) { + FileUtil.delete(zipFile); + } try (JBZipFile file = new JBZipFile(zipFile)) { - Path outPath = out.toPath(); - Files.walk(outPath).forEach(p -> { + Files.walk(out).forEach(p -> { if (Files.isDirectory(p)) return; - String relativePath = outPath.relativize(p).toString(); + String relativePath = out.relativize(p).toString(); try { JBZipEntry entry = file.getOrCreateEntry(relativePath); entry.setMethod(ZipEntry.STORED); @@ -148,8 +208,90 @@ public class DumpIndexAction extends AnAction { } } + private static void generate(@NotNull IndexChunk chunk, + @NotNull Project project, + @NotNull Collection> generators, + @NotNull Path chunkOut) { + try { + ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(chunkOut.resolve("hashes")); + try { + for (HashBasedIndexGenerator generator : generators) { + generator.openIndex(); + } + + for (VirtualFile root : chunk.getRoots()) { + VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor() { + @Override + public boolean visitFile(@NotNull VirtualFile file) { + + if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) { + for (HashBasedIndexGenerator generator : generators) { + generator.indexFile(file, project, hashEnumerator); + } + } + + return true; + } + }); + } + } + finally { + hashEnumerator.close(); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + finally { + try { + for (HashBasedIndexGenerator generator : generators) { + generator.closeIndex(); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private static void printStatistics(@NotNull IndexChunk chunk, + @NotNull List> fileBasedGenerators, + @NotNull StubHashBasedIndexGenerator stubGenerator) { + StringBuilder stats = new StringBuilder(); + stats.append("Statistics for index chunk ").append(chunk.getName()).append("\n"); + + stats.append("File based indices (").append(fileBasedGenerators.size()).append(")"); + for (HashBasedIndexGenerator generator : fileBasedGenerators) { + appendGeneratorStatistics(stats, generator); + } + + Collection> stubGenerators = stubGenerator.getStubGenerators(); + stats.append("Stub indices (").append(stubGenerators.size()).append(")"); + for (HashBasedIndexGenerator generator : stubGenerators) { + appendGeneratorStatistics(stats, generator); + } + + System.out.println(stats.toString()); + } + + private static void appendGeneratorStatistics(@NotNull StringBuilder stringBuilder, + @NotNull HashBasedIndexGenerator generator) { + stringBuilder + .append(" ") + .append("Generator for ") + .append(generator.getExtension().getName().getName()) + .append(" indexed ") + .append(generator.getIndexedFilesNumber()) + .append(" files"); + + if (generator.isEmpty()) { + stringBuilder.append( "(empty result)"); + } + stringBuilder.append("\n"); + } + @NotNull - private static Stream getExportableIndices(boolean all) { + private static Stream> getExportableIndices(boolean all) { if (all) { return FileBasedIndexExtension .EXTENSION_POINT_NAME @@ -159,7 +301,7 @@ public class DumpIndexAction extends AnAction { } //kt - Stream ktIndices = + Stream> ktIndices = FileBasedIndexExtension .EXTENSION_POINT_NAME .extensions() @@ -171,14 +313,14 @@ public class DumpIndexAction extends AnAction { ID.findByName("DomFileIndex"), ID.findByName("xmlProperties")); //xml - Stream xmlIndices = + Stream> xmlIndices = FileBasedIndexExtension .EXTENSION_POINT_NAME .extensions() .filter(id -> xmlIndexIds.contains(id.getName())); //base - Stream coreIndices = + Stream> coreIndices = FileBasedIndexExtension .EXTENSION_POINT_NAME .extensions() @@ -188,7 +330,7 @@ public class DumpIndexAction extends AnAction { ex.getName().getName().equals("HashFragmentIndex")); //java - Stream javaIndices = + Stream> javaIndices = FileBasedIndexExtension .EXTENSION_POINT_NAME .extensions() @@ -202,11 +344,11 @@ public class DumpIndexAction extends AnAction { return Stream.concat(Stream.concat(Stream.concat(coreIndices, javaIndices), xmlIndices), ktIndices); } - private static final class IndexChunk { + public static final class IndexChunk { private final Set myRoots; private final String myName; - IndexChunk(Set roots, String name) { + public IndexChunk(Set roots, String name) { myRoots = roots; myName = name; } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java index df95e9b7fd0a..548f7d84ccf8 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java @@ -4,12 +4,10 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.openapi.vfs.VirtualFileVisitor; import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; -import com.intellij.psi.SingleRootFileViewProvider; import com.intellij.util.indexing.*; +import com.intellij.util.indexing.impl.IndexStorage; import com.intellij.util.indexing.impl.InputData; import com.intellij.util.indexing.impl.MapIndexStorage; import com.intellij.util.indexing.impl.MapReduceIndex; @@ -18,35 +16,41 @@ import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.KeyDescriptor; import org.jetbrains.annotations.NotNull; -import java.io.File; import java.io.IOException; -import java.util.*; -import java.util.concurrent.atomic.LongAdder; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; public class HashBasedIndexGenerator { - @NotNull - private final File myOut; @NotNull private final FakeIndexExtension myExtension; @NotNull private final FileBasedIndex.InputFilter myInputFilter; + private final Path myStorageFile; + + private final AtomicInteger myIndexedFilesNumber = new AtomicInteger(); + + private final AtomicBoolean myIsEmpty = new AtomicBoolean(true); + private InvertedIndex myIndex; - public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension indexExtension, @NotNull File out) { - this(indexExtension.getKeyDescriptor(), - indexExtension.getValueExternalizer(), - indexExtension, - out - ); + public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension indexExtension, @NotNull Path outRoot) { + this(indexExtension.getKeyDescriptor(), indexExtension.getValueExternalizer(), indexExtension, outRoot); } public HashBasedIndexGenerator(@NotNull KeyDescriptor keyDescriptor, @NotNull DataExternalizer valueExternalizer, @NotNull FileBasedIndexExtension originalExtension, - @NotNull File out) { + @NotNull Path outRoot) { myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension); - myOut = out; + + String indexName = originalExtension.getName().getName(); + myStorageFile = outRoot.resolve(StringUtil.toLowerCase(indexName)).resolve(indexName); FileBasedIndex.InputFilter filter = originalExtension.getInputFilter(); @@ -54,33 +58,41 @@ public class HashBasedIndexGenerator { Set fileTypes = new HashSet<>(); ((FileBasedIndex.FileTypeSpecificInputFilter)filter).registerFileTypesUsedForIndexing(fileTypes::add); myInputFilter = file -> fileTypes.contains(file.getFileType()) && filter.acceptInput(file); - } else { + } + else { myInputFilter = filter; } } - @NotNull - public File getOut() { - return myOut; - } - public void openIndex() throws IOException { - String indexName = myExtension.getName().getName(); - boolean singleEntry = myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension; - myIndex = new MapReduceIndex(myExtension, new MapIndexStorage(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName).toPath(), - myExtension.getKeyDescriptor(), - myExtension.getValueExternalizer(), - myExtension.getCacheSize(), - myExtension.keyIsUniqueForIndexedFile()) { + IndexStorage indexStorage = new MapIndexStorage(myStorageFile, + myExtension.getKeyDescriptor(), + myExtension.getValueExternalizer(), + myExtension.getCacheSize(), + myExtension.keyIsUniqueForIndexedFile()) { @Override protected void checkCanceled() { //ignore } - }, null, null) { + + @Override + public void addValue(K k, int inputId, V v) throws StorageException { + super.addValue(k, inputId, v); + myIsEmpty.set(false); + } + + @Override + public void removeAllValues(@NotNull K k, int inputId) { + throw new AssertionError("Must not happen in index generator"); + } + }; + + myIndex = new MapReduceIndex(myExtension, indexStorage, null, null) { @NotNull @Override protected Map mapByIndexer(int inputId, @NotNull FileContent content) { Map data = super.mapByIndexer(inputId, content); + boolean singleEntry = HashBasedIndexGenerator.this.myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension; if (singleEntry && !data.isEmpty()) { data = Collections.singletonMap((K)(Integer)inputId, data.values().iterator().next()); } @@ -105,7 +117,7 @@ public class HashBasedIndexGenerator { @Override protected void requestRebuild(@NotNull Throwable e) { - throw new RuntimeException("error while processing " + indexName, e); + throw new RuntimeException("Error while processing " + myExtension.getName().getName(), e); } }; } @@ -115,82 +127,47 @@ public class HashBasedIndexGenerator { } public void closeIndex() throws IOException { - if (myIndex != null) myIndex.dispose(); + if (myIndex != null) { + myIndex.dispose(); + } } - public static void generate(@NotNull Collection roots, - @NotNull Collection> generators, - @NotNull Project project, - @NotNull File hashOut) { - - LongAdder l = new LongAdder(); - try { - ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(new File(hashOut, "hashes").toPath()); - - for (HashBasedIndexGenerator generator : generators) { - generator.openIndex(); - } - - for (VirtualFile root : roots) { - VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor() { - @Override - public boolean visitFile(@NotNull VirtualFile file) { - - if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) { - for (HashBasedIndexGenerator generator : generators) { - if (generator.myInputFilter.acceptInput(file)) { - l.increment(); - generator.indexFile(file, project, hashEnumerator); - } - } - } - - return true; - } - }); - } - - synchronized (hashEnumerator) { - hashEnumerator.close(); - } + public void indexFile(@NotNull VirtualFile f, @NotNull Project project, @NotNull ContentHashesUtil.HashEnumerator hashEnumerator) { + if (!myInputFilter.acceptInput(f)) { + return; } - catch (IOException e) { - throw new RuntimeException(e); - } - finally { - try { - for (HashBasedIndexGenerator generator : generators) { - generator.closeIndex(); - } - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - - System.out.println("Indexed " + l.sum() + " files to " + hashOut.getPath()); - } - - protected void indexFile(@NotNull VirtualFile f, - @NotNull Project project, - @NotNull ContentHashesUtil.HashEnumerator hashEnumerator) { + myIndexedFilesNumber.incrementAndGet(); try { FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray()); byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(fc, false); - int hashId; - synchronized (hashEnumerator) { - hashId = Math.abs(hashEnumerator.enumerate(hash)); - } + int hashId = Math.abs(hashEnumerator.enumerate(hash)); fc.putUserData(IndexingDataKeys.PROJECT, project); if (!myIndex.update(hashId, fc).compute()) { throw new RuntimeException(); } } catch (IOException e) { - throw new RuntimeException("cant index " + f.getPath() + " for " + myExtension.getName().getName(), e); + throw new RuntimeException("Can't index " + f.getPath() + " for " + myExtension.getName().getName(), e); } } + @NotNull + public Path getIndexRoot() { + return myStorageFile.getParent(); + } + + public int getIndexedFilesNumber() { + return myIndexedFilesNumber.get(); + } + + public boolean isEmpty() { + return myIsEmpty.get(); + } + + public FileBasedIndexExtension getExtension() { + return myExtension; + } + public InvertedIndex getIndex() { return myIndex; } diff --git a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java index ce242349b4c0..9244daf5a54c 100644 --- a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java +++ b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java @@ -17,35 +17,44 @@ import com.intellij.util.indexing.impl.UpdateData; import com.intellij.util.io.EnumeratorIntegerDescriptor; import org.jetbrains.annotations.NotNull; -import java.io.File; import java.io.IOException; -import java.util.*; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator { - private final Map myStubIndexesGeneratorMap = new HashMap<>(); - private final Set myUsedKeys = new HashSet<>(); - public StubHashBasedIndexGenerator(@NotNull File out) { - super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer( - true, - null, - StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), getExtension(), out); + private final Map, HashBasedIndexGenerator> myStubIndexesGeneratorMap = new HashMap<>(); + + private final Path myStubIndicesRoot; + + public StubHashBasedIndexGenerator(@NotNull Path outRoot) { + super( + EnumeratorIntegerDescriptor.INSTANCE, + new SerializedStubTreeDataExternalizer(true, null, StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), + new StubUpdatingIndex(StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), + outRoot + ); + + myStubIndicesRoot = outRoot.resolve(StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName())); + for (StubIndexExtension stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) { - FileBasedIndexExtension ex = StubIndexImpl - .wrapStubIndexExtension(stubIndexExtension); - myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(ex.getKeyDescriptor(), - ex.getValueExternalizer(), - ex, - new File(out, getStubsDir()) - ) { - - }); + FileBasedIndexExtension extension = StubIndexImpl.wrapStubIndexExtension(stubIndexExtension); + HashBasedIndexGenerator hashBasedIndexGenerator = createGenerator(extension); + myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), hashBasedIndexGenerator); } } @NotNull - private static String getStubsDir() { - return StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName()); + private HashBasedIndexGenerator createGenerator(FileBasedIndexExtension extension) { + return new HashBasedIndexGenerator<>(extension.getKeyDescriptor(), extension.getValueExternalizer(), extension, myStubIndicesRoot); + } + + @NotNull + public List> getStubGenerators() { + return new ArrayList<>(myStubIndexesGeneratorMap.values()); } @Override @@ -57,17 +66,23 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator> entry : map.entrySet()) { StubIndexKey key = entry.getKey(); Map value = entry.getValue(); - myUsedKeys.add(key); MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex(); Map reducedValue = Maps.asMap(value.keySet(), k -> null); - index.updateWithMap(new UpdateData(hashId, reducedValue, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null)); + UpdateData updateData = new UpdateData( + hashId, + reducedValue, + () -> new EmptyInputDataDiffBuilder(hashId), + index.getExtension().getName(), + null + ); + index.updateWithMap(updateData); } } @Override public void openIndex() throws IOException { super.openIndex(); - for (HashBasedIndexGenerator generator : myStubIndexesGeneratorMap.values()) { + for (HashBasedIndexGenerator generator : myStubIndexesGeneratorMap.values()) { generator.openIndex(); } } @@ -75,15 +90,11 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator entry : myStubIndexesGeneratorMap.entrySet()) { - HashBasedIndexGenerator generator = entry.getValue(); - generator.closeIndex(); + for (Map.Entry, HashBasedIndexGenerator> entry : myStubIndexesGeneratorMap.entrySet()) { + entry.getValue().closeIndex(); } ((SerializationManagerEx)SerializationManager.getInstance()).flushNameStorage(); - FileUtil.copyDir(PathManager.getIndexRoot(), new File(getOut(), getStubsDir()), f -> f.getName().startsWith("rep.names")); - } - private static StubUpdatingIndex getExtension() { - return (new StubUpdatingIndex(StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE)); + FileUtil.copyDir(PathManager.getIndexRoot(), myStubIndicesRoot.toFile(), f -> f.getName().startsWith("rep.names")); } }