diff --git a/java/java-impl/src/com/intellij/internal/DumpIndexAction.java b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java index 68e4ef8fd9b3..ce6033066211 100644 --- a/java/java-impl/src/com/intellij/internal/DumpIndexAction.java +++ b/java/java-impl/src/com/intellij/internal/DumpIndexAction.java @@ -2,110 +2,292 @@ package com.intellij.internal; import com.intellij.codeInspection.bytecodeAnalysis.BytecodeAnalysisIndex; +import com.intellij.concurrency.JobLauncher; import com.intellij.find.ngrams.TrigramIndex; import com.intellij.openapi.actionSystem.AnAction; import com.intellij.openapi.actionSystem.AnActionEvent; +import com.intellij.openapi.application.ReadAction; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.fileChooser.FileChooser; import com.intellij.openapi.fileChooser.FileChooserDescriptor; import com.intellij.openapi.fileChooser.FileChooserDescriptorFactory; +import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleManager; +import com.intellij.openapi.progress.ProgressIndicator; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.progress.Task; import com.intellij.openapi.project.Project; import com.intellij.openapi.roots.*; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.openapi.vfs.VirtualFileUrlChangeAdapter; +import com.intellij.psi.impl.JavaSimplePropertyIndex; import com.intellij.psi.impl.cache.impl.id.IdIndex; +import com.intellij.psi.impl.cache.impl.todo.TodoIndex; +import com.intellij.psi.impl.java.JavaBinaryPlusExpressionIndex; +import com.intellij.psi.impl.java.JavaFunctionalExpressionIndex; import com.intellij.psi.impl.java.stubs.index.JavaAutoModuleNameIndex; -import com.intellij.psi.stubs.StubIndexKey; +import com.intellij.psi.impl.search.JavaNullMethodArgumentIndex; +import com.intellij.psi.stubs.StubUpdatingIndex; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.FileBasedIndexExtension; import com.intellij.util.indexing.ID; import com.intellij.util.indexing.IndexableSetContributor; import com.intellij.util.indexing.hash.HashBasedIndexGenerator; -import one.util.streamex.StreamEx; +import com.intellij.util.io.zip.JBZipEntry; +import com.intellij.util.io.zip.JBZipFile; +import gnu.trove.THashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.index.stubs.StubHashBasedIndexGenerator; import java.io.File; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.ZipEntry; public class DumpIndexAction extends AnAction { + private static final Logger LOG = Logger.getInstance(DumpIndexAction.class); @Override public void actionPerformed(@NotNull AnActionEvent event) { Project project = event.getProject(); if (project == null) return; - Stream libRoots = Arrays - .stream(ModuleManager.getInstance(project).getModules()) - .flatMap(m -> Arrays.stream(ModuleRootManager.getInstance(m).getOrderEntries())) - .filter(e -> e instanceof LibraryOrSdkOrderEntry) - .flatMap(e -> Stream.concat(Arrays.stream(e.getFiles(OrderRootType.CLASSES)), Arrays.stream(e.getFiles(OrderRootType.SOURCES)))); + Collection projectChunks = Arrays + .stream(ModuleManager.getInstance(project).getModules()) + .flatMap(m -> IndexChunk.generate(m)) + .collect(Collectors.toMap(ch -> ch.getName(), ch -> ch, IndexChunk::mergeUnsafe)) + .values(); - Stream additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(), - IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())); - Set roots = Stream.concat(libRoots, additionalRoots).collect(Collectors.toSet()); + Set + additionalRoots = IndexableSetContributor.EP_NAME.extensions().flatMap(contributor -> Stream.concat(IndexableSetContributor.getRootsToIndex(contributor).stream(), + IndexableSetContributor.getProjectRootsToIndex(contributor, project).stream())).collect( + Collectors.toSet()); + + Set synthRoots = new THashSet<>(); + for (AdditionalLibraryRootsProvider provider : AdditionalLibraryRootsProvider.EP_NAME.getExtensionList()) { + for (SyntheticLibrary library : provider.getAdditionalProjectLibraries(project)) { + for (VirtualFile root : library.getAllRoots()) { + // do not try to visit under-content-roots because the first task took care of that already + if (!ProjectFileIndex.getInstance(project).isInContent(root)) { + synthRoots.add(root); + } + } + } + } + + List chunks = Stream.concat(projectChunks.stream(), + Stream.of(new IndexChunk(additionalRoots, "ADDITIONAL"), + new IndexChunk(synthRoots, "SYNTH"))).collect(Collectors.toList()); + + //IndexChunk chunk = chunks.stream().reduce((c1, c2) -> IndexChunk.mergeUnsafe(c1, c2)).get(); FileChooserDescriptor descriptor = FileChooserDescriptorFactory.createSingleFolderDescriptor(); descriptor.withTitle("Select Index Dump Directory"); VirtualFile file = FileChooser.chooseFile(descriptor, project, null); - if (file == null) return; - File out = VfsUtilCore.virtualToIoFile(file); - FileUtil.delete(out); - exportIndices(roots, out); + if (file != null) { + ProgressManager.getInstance().run(new Task.Modal(project, "Exporting Indexes..." , true) { + @Override + public void run(@NotNull ProgressIndicator indicator) { + File out = VfsUtilCore.virtualToIoFile(file); + FileUtil.delete(out); + exportIndices(chunks, out, indicator, project); + } + }); + } } - public static void exportIndices(@NotNull Set roots, @NotNull File out) { - StubHashBasedIndexGenerator generator = new StubHashBasedIndexGenerator(out); - generator.generate(roots); + public static void exportIndices(@NotNull List chunks, + @NotNull File out, + @NotNull ProgressIndicator indicator, + @NotNull Project project) { + indicator.setIndeterminate(false); + AtomicInteger idx = new AtomicInteger(); + if (!JobLauncher.getInstance().invokeConcurrentlyUnderProgress(chunks, indicator, chunk -> { + indicator.setText("Indexing '" + chunk.getName() + "' chunk"); + File chunkOut = new File(out, chunk.getName()); + ReadAction.run(() -> { + Stream> fbIndexes = getExportableIndices(true).map(ex -> new HashBasedIndexGenerator(ex, chunkOut)); + Stream> stubIndex = Stream.of(new StubHashBasedIndexGenerator(chunkOut)); + List> indexes = Stream.concat(fbIndexes, stubIndex).collect(Collectors.toList()); + HashBasedIndexGenerator.generate(chunk.getRoots(),indexes, project, chunkOut); + }); + indicator.setFraction(((double) idx.incrementAndGet()) / chunks.size()); + return true; + })) { + throw new AssertionError(); + } - getExportableIndices().forEach(ex -> { - HashBasedIndexGenerator indexGenerator = new HashBasedIndexGenerator(ex, out); - indexGenerator.generate(roots); - }); + indicator.setIndeterminate(true); + indicator.setText("Zipping index pack"); + + File zipFile = new File(out.getAbsolutePath() + ".zip"); + FileUtil.delete(zipFile); + try (JBZipFile file = new JBZipFile(zipFile)) { + Path outPath = out.toPath(); + Files.walk(outPath).forEach(p -> { + if (Files.isDirectory(p)) return; + String relativePath = outPath.relativize(p).toString(); + try { + JBZipEntry entry = file.getOrCreateEntry(relativePath); + entry.setMethod(ZipEntry.STORED); + entry.setDataFromFile(p.toFile()); + } + catch (IOException e) { + LOG.error(e); + } + }); + } + catch (IOException e) { + LOG.error(e); + } } @NotNull - private static Stream getExportableIndices() { + private static Stream getExportableIndices(boolean all) { + if (all) { + return FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(ex -> ex.dependsOnFileContent()) + .filter(ex -> !(ex instanceof StubUpdatingIndex)); + } + //kt Stream ktIndices = - FileBasedIndexExtension - .EXTENSION_POINT_NAME - .extensions() - .filter(id -> id.getName().getName().contains("kotlin")); + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(id -> id.getName().getName().contains("kotlin")); Set> xmlIndexIds = ContainerUtil.set(ID.findByName("XmlTagNames"), - ID.findByName("XmlNamespaces"), - ID.findByName("SchemaTypeInheritance"), - ID.findByName("DomFileIndex"), - ID.findByName("xmlProperties")); + ID.findByName("XmlNamespaces"), + ID.findByName("SchemaTypeInheritance"), + ID.findByName("DomFileIndex"), + ID.findByName("xmlProperties")); //xml Stream xmlIndices = - FileBasedIndexExtension - .EXTENSION_POINT_NAME - .extensions() - .filter(id -> xmlIndexIds.contains(id.getName())); + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(id -> xmlIndexIds.contains(id.getName())); //base Stream coreIndices = - FileBasedIndexExtension - .EXTENSION_POINT_NAME - .extensions() - .filter(ex -> ex.getName().equals(TrigramIndex.INDEX_ID) || - ex.getName().equals(IdIndex.NAME)); + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(ex -> ex.getName().equals(TrigramIndex.INDEX_ID) || + ex.getName().equals(TodoIndex.NAME) || + ex.getName().equals(IdIndex.NAME) || + ex.getName().getName().equals("HashFragmentIndex")); //java Stream javaIndices = - FileBasedIndexExtension - .EXTENSION_POINT_NAME - .extensions() - .filter(ex -> ex instanceof BytecodeAnalysisIndex || ex instanceof JavaAutoModuleNameIndex); + FileBasedIndexExtension + .EXTENSION_POINT_NAME + .extensions() + .filter(ex -> ex instanceof BytecodeAnalysisIndex || + ex instanceof JavaAutoModuleNameIndex || + ex instanceof JavaFunctionalExpressionIndex || + ex instanceof JavaSimplePropertyIndex || + ex instanceof JavaNullMethodArgumentIndex || + ex instanceof JavaBinaryPlusExpressionIndex); return Stream.concat(Stream.concat(Stream.concat(coreIndices, javaIndices), xmlIndices), ktIndices); } + + private static final class IndexChunk { + private final Set myRoots; + private final String myName; + + IndexChunk(Set roots, String name) { + myRoots = roots; + myName = name; + } + + private String getName() { + return myName; + } + + private Set getRoots() { + return myRoots; + } + + static IndexChunk mergeUnsafe(IndexChunk ch1, IndexChunk ch2) { + ch1.getRoots().addAll(ch2.getRoots()); + return ch1; + } + + static Stream generate(Module module) { + Stream libChunks = Arrays.stream(ModuleRootManager.getInstance(module).getOrderEntries()) + .map(orderEntry -> { + if (orderEntry instanceof LibraryOrSdkOrderEntry) { + VirtualFile[] sources = orderEntry.getFiles(OrderRootType.SOURCES); + VirtualFile[] classes = orderEntry.getFiles(OrderRootType.CLASSES); + String name = null; + if (orderEntry instanceof JdkOrderEntry) { + name = ((JdkOrderEntry)orderEntry).getJdkName(); + } + else if (orderEntry instanceof LibraryOrderEntry) { + name = ((LibraryOrderEntry)orderEntry).getLibraryName(); + } + if (name == null) { + name = "unknown"; + } + return new IndexChunk(ContainerUtil.union(Arrays.asList(sources), Arrays.asList(classes)), reducePath(splitByDots(name))); + } + return null; + }) + .filter(Objects::nonNull); + + Set roots = + ContainerUtil.union(ContainerUtil.newTroveSet(ModuleRootManager.getInstance(module).getContentRoots()), + ContainerUtil.newTroveSet(ModuleRootManager.getInstance(module).getSourceRoots())); + Stream srcChunks = Stream.of(new IndexChunk(roots, getChunkName(module))); + + return Stream.concat(libChunks, srcChunks); + } + + private static String getChunkName(Module module) { + ModuleManager moduleManager = ModuleManager.getInstance(module.getProject()); + String[] path; + if (moduleManager.hasModuleGroups()) { + path = moduleManager.getModuleGroupPath(module); + assert path != null; + } else { + path = splitByDots(module.getName()); + } + return reducePath(path); + } + + @NotNull + private static String reducePath(String[] path) { + String[] reducedPath = Arrays.copyOfRange(path, 0, Math.min(1, path.length)); + return StringUtil.join(reducedPath, "."); + } + + private static String[] splitByDots(String name) { + return name.split("[-|:.]"); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IndexChunk chunk = (IndexChunk)o; + return Objects.equals(myRoots, chunk.myRoots) && + Objects.equals(myName, chunk.myName); + } + + @Override + public int hashCode() { + return Objects.hash(myRoots, myName); + } + } } diff --git a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java index b08bdc71728d..a41fc7d0ff02 100644 --- a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java +++ b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java @@ -45,11 +45,16 @@ public class SerializedStubTree { private Map> myIndexedStubs; private volatile SerializationManagerEx mySerializationManager; + private volatile StubForwardIndexExternalizer myStubIndexesExternalizer; - public void setSerializationManager(SerializationManagerEx serializationManager) { + public void setSerializationManager(@NotNull SerializationManagerEx serializationManager) { mySerializationManager = serializationManager; } + public void setStubIndexesExternalizer(@NotNull StubForwardIndexExternalizer stubIndexesExternalizer) { + myStubIndexesExternalizer = stubIndexesExternalizer; + } + public SerializedStubTree(@NotNull byte[] treeBytes, int treeByteLength, @Nullable Stub stubElement, @NotNull byte[] indexedStubBytes, int indexedStubByteLength, @Nullable Map> indexedStubs) { myTreeBytes = treeBytes; @@ -74,6 +79,7 @@ public class SerializedStubTree { forwardIndexExternalizer.save(new DataOutputStream(indexBytes), myIndexedStubs); myIndexedStubBytes = indexBytes.getInternalBuffer(); myIndexedStubByteLength = indexBytes.size(); + myStubIndexesExternalizer = forwardIndexExternalizer; } @NotNull @@ -94,7 +100,7 @@ public class SerializedStubTree { else { BufferExposingByteArrayOutputStream reSerializedStubIndices = new BufferExposingByteArrayOutputStream(); if (myIndexedStubs == null) { - restoreIndexedStubs(currentForwardIndexSerializer); + restoreIndexedStubs(); } assert myIndexedStubs != null; newForwardIndexSerializer.save(new DataOutputStream(reSerializedStubIndices), myIndexedStubs); @@ -102,18 +108,25 @@ public class SerializedStubTree { reSerializedIndexByteLength = reSerializedStubIndices.size(); } - return new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null, - reSerializedIndexBytes, reSerializedIndexByteLength, myIndexedStubs); + SerializedStubTree tree = new SerializedStubTree( + outStub.getInternalBuffer(), + outStub.size(), + null, + reSerializedIndexBytes, + reSerializedIndexByteLength, + myIndexedStubs); + tree.setStubIndexesExternalizer(myStubIndexesExternalizer); + return tree; } - void restoreIndexedStubs(@NotNull StubForwardIndexExternalizer dataExternalizer) throws IOException { + void restoreIndexedStubs() throws IOException { if (myIndexedStubs == null) { - myIndexedStubs = dataExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength))); + myIndexedStubs = myStubIndexesExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength))); } } - StubIdList restoreIndexedStubs(@NotNull StubForwardIndexExternalizer dataExternalizer, @NotNull StubIndexKey indexKey, @NotNull K key) throws IOException { - Map> incompleteMap = dataExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key); + StubIdList restoreIndexedStubs(@NotNull StubIndexKey indexKey, @NotNull K key) throws IOException { + Map> incompleteMap = myStubIndexesExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key); Map map = incompleteMap.get(indexKey); return map == null ? null : map.get(key); } @@ -125,7 +138,7 @@ public class SerializedStubTree { @TestOnly public Map> readStubIndicesValueMap() throws IOException { - restoreIndexedStubs(IDE_USED_EXTERNALIZER); + restoreIndexedStubs(); return myIndexedStubs; } diff --git a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java index 46d38fca1903..6fb71271d631 100644 --- a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java +++ b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java @@ -15,14 +15,16 @@ import java.io.IOException; public class SerializedStubTreeDataExternalizer implements DataExternalizer { private final boolean myIncludeInputs; private final SerializationManagerEx mySerializationManager; + private final StubForwardIndexExternalizer myStubIndexesExternalizer; public SerializedStubTreeDataExternalizer() { - this(true, null); + this(true, null, SerializedStubTree.IDE_USED_EXTERNALIZER); } - public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager) { + public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager, StubForwardIndexExternalizer externalizer) { myIncludeInputs = inputs; mySerializationManager = manager; + myStubIndexesExternalizer = externalizer; } @Override @@ -44,6 +46,7 @@ public class SerializedStubTreeDataExternalizer implements DataExternalizer ProvidedIndexExtension findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension) { - Path root = getPrebuiltIndexPath(); - if (root == null || !Files.exists(root.resolve( StringUtil.toLowerCase(originalExtension.getName().getName())))) return null; + private static UncompressedZipFileSystem ourFs; - return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID) - ? (ProvidedIndexExtension)new StubProvidedIndexExtension(root) - : new ProvidedIndexExtensionImpl<>(root, originalExtension); + @NotNull + @Override + public Stream> findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension) { + if (!originalExtension.dependsOnFileContent()) return Stream.empty(); + Path root = getPrebuiltIndexPath(); + if (root == null || !Files.exists(root)) return Stream.empty(); + + try { + // TODO properly close it + UncompressedZipFileSystem fs = getFs(root); + + Path fsRoot = fs.getPath("/").getRoot(); + return Files + .list(fsRoot) + .sorted(Comparator.comparing(p -> p.getFileName().toString())) + .map(p -> p.resolve(StringUtil.toLowerCase(originalExtension.getName().getName()))) + .map(p -> { + return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID) + ? (ProvidedIndexExtension)new StubProvidedIndexExtension(p) + : new ProvidedIndexExtensionImpl<>(p, originalExtension); + }); + } catch (IOException e) { + LOG.error(e); + return Stream.empty(); + } + } + + private synchronized static UncompressedZipFileSystem getFs(@NotNull Path root) throws IOException { + if (ourFs == null) { + ourFs = new UncompressedZipFileSystem(root, new UncompressedZipFileSystemProvider()); + ShutDownTracker.getInstance().registerShutdownTask(() -> { + try { + ourFs.close(); + } + catch (IOException e) { + LOG.error(e); + } + }); + } + return ourFs; } @Nullable private static Path getPrebuiltIndexPath() { - String path = System.getProperty(PREBUILT_INDEX_PATH_PROP); + String path = System.getProperty(PREBUILT_INDEX_ZIP_PROP); if (path == null) return null; - Path file; - try { - file = Paths.get(new URI(path)); - } - catch (URISyntaxException e) { - LOG.error(e); - return null; - } + Path file = Paths.get(path); return Files.exists(file) ? file : null; } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 22cbcbe9fab5..3cd61172d534 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -52,6 +52,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.function.IntPredicate; +import java.util.stream.Collectors; @State(name = "FileBasedIndex", storages = { @Storage(value = StoragePathMacros.CACHE_FILE), @@ -196,14 +197,13 @@ public final class StubIndexImpl extends StubIndexEx implements PersistentStateC UpdatableIndex index = new VfsAwareMapReduceIndex<>(wrappedExtension, memStorage, null, null, null, lock); if (stubUpdatingIndex instanceof MergedInvertedIndex) { - ProvidedIndexExtension ex = - ((MergedInvertedIndex)stubUpdatingIndex).getProvidedExtension(); - if (ex instanceof StubProvidedIndexExtension) { - ProvidedIndexExtension providedStubIndexExtension = - ((StubProvidedIndexExtension)ex).findProvidedStubIndex(extension); - if (providedStubIndexExtension != null) { - index = ProvidedIndexExtension.wrapWithProvidedIndex(providedStubIndexExtension, wrappedExtension, index); - } + List> providedIndexExtensions = ((MergedInvertedIndex)stubUpdatingIndex) + .getProvidedExtensions() + .filter(ex -> ex instanceof StubProvidedIndexExtension) + .map(ex -> ((StubProvidedIndexExtension)ex).findProvidedStubIndex(extension)) + .collect(Collectors.toList()); + if (!providedIndexExtensions.isEmpty()) { + index = ProvidedIndexExtension.wrapWithProvidedIndex(providedIndexExtensions, wrappedExtension, index, ((MergedInvertedIndex)stubUpdatingIndex).getHashIndex()); } } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java index 1393a40f8a17..9dbaa5d032e8 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java @@ -37,7 +37,7 @@ public final class StubProcessingHelper extends StubProcessingHelperBase { return null; } SerializedStubTree tree = data.values().iterator().next(); - StubIdList stubIdList = tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER, indexKey, key); + StubIdList stubIdList = tree.restoreIndexedStubs(indexKey, key); if (stubIdList == null) { LOG.error("Stub ids not found for key in index = " + indexKey.getName() + ", file type = " + file.getFileType()); onInternalError(file); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java index 4e71a819a7db..89abf9c4bc49 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java @@ -37,7 +37,7 @@ class StubUpdatingForwardIndexAccessor implements ForwardIndexAccessor data = dataRef.get(); SerializedStubTree tree = ContainerUtil.isEmpty(data) ? null : ContainerUtil.getFirstItem(data.values()); if (tree != null) { - tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER); + tree.restoreIndexedStubs(); } return new StubCumulativeInputDiffBuilder(inputId, tree); } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 79564493bbbf..74b374baeef9 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -47,6 +47,15 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension INDEX_ID = ID.create("Stubs"); private static final FileBasedIndex.InputFilter INPUT_FILTER = file -> canHaveStub(file); + private final StubForwardIndexExternalizer myStubIndexesExternalizer; + + public StubUpdatingIndex() { + myStubIndexesExternalizer = SerializedStubTree.IDE_USED_EXTERNALIZER; + } + + public StubUpdatingIndex(StubForwardIndexExternalizer stubIndexesExternalizer) { + myStubIndexesExternalizer = stubIndexesExternalizer; + } public static boolean canHaveStub(@NotNull VirtualFile file) { Project project = ProjectUtil.guessProjectForFile(file); @@ -238,7 +247,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension getValueExternalizer() { - return new SerializedStubTreeDataExternalizer(); + return new SerializedStubTreeDataExternalizer(true, null, myStubIndexesExternalizer); } @NotNull diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java b/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java index 5e5a410f1712..670b59885214 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java @@ -5,8 +5,6 @@ import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.stubs.*; -import com.intellij.util.indexing.FileBasedIndex; -import com.intellij.util.indexing.FileBasedIndexImpl; import com.intellij.util.indexing.ID; import com.intellij.util.indexing.provided.ProvidedIndexExtension; import com.intellij.util.io.DataExternalizer; @@ -16,7 +14,6 @@ import com.intellij.util.io.VoidDataExternalizer; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.File; import java.nio.file.Files; import java.nio.file.Path; @@ -52,7 +49,7 @@ public class StubProvidedIndexExtension implements ProvidedIndexExtension index = createIndex(extension, new MemoryIndexStorage<>(storage, name)); - ProvidedIndexExtension providedExtension = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension); - if (providedExtension != null) { - index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtension, extension, index); + if (!(extension instanceof FileContentHashIndexExtension)) { + List> providedExtensions = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension); + if (!providedExtensions.isEmpty()) { + Path[] paths = ContainerUtil.map2Array(providedExtensions, Path.class, ex -> ex.getIndexPath()); + FileContentHashIndex contentHashIndex = ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(paths, state); + + index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtensions, extension, index, contentHashIndex); + } } state.registerIndex(name, @@ -2677,19 +2683,20 @@ public final class FileBasedIndexImpl extends FileBasedIndex { } } - public synchronized FileContentHashIndex getFileContentHashIndex(@NotNull File enumeratorPath) { - UpdatableIndex index = getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); + public synchronized FileContentHashIndex getFileContentHashIndex(@Nullable Path[] enumeratorPaths, @NotNull IndexConfiguration state) { + UpdatableIndex index = state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); if (index == null) { + LOG.assertTrue(enumeratorPaths != null); IndicesRegistrationResult registrationResult = new IndicesRegistrationResult(); try { - registerIndexer(FileContentHashIndexExtension.create(enumeratorPath, ApplicationManager.getApplication()), myRegisteredIndexes.getState(), registrationResult); + registerIndexer(FileContentHashIndexExtension.create(enumeratorPaths, ApplicationManager.getApplication()), state, registrationResult); registrationResult.logChangedAndFullyBuiltIndices(LOG, "Version was changed for:", "Index is to be rebuilt:"); } catch (IOException e) { throw new RuntimeException(e); } } else return (FileContentHashIndex)index; - return (FileContentHashIndex)getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); + return (FileContentHashIndex)state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); } private static final boolean INDICES_ARE_PSI_DEPENDENT_BY_DEFAULT = SystemProperties.getBooleanProperty("idea.indices.psi.dependent.default", true); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java b/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java index 03878dc6ff3e..bbd6e1ecce55 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java @@ -12,14 +12,14 @@ import java.util.concurrent.atomic.AtomicReference; /** * @author peter */ -enum RebuildStatus { +public enum RebuildStatus { OK, REQUIRES_REBUILD, DOING_REBUILD; private static final Map, AtomicReference> ourRebuildStatus = new THashMap<>(); - static void registerIndex(ID indexId) { + public static void registerIndex(ID indexId) { ourRebuildStatus.put(indexId, new AtomicReference<>(OK)); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java index 336491ac0d64..7d6dc83a00f8 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java @@ -119,7 +119,7 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex mapInput(@Nullable Input content) { + protected InputData mapInput(int inputId, @Nullable Input content) { InputData data; boolean containsSnapshotData = true; if (mySnapshotInputMappings != null && content != null) { @@ -135,7 +135,7 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex)mySnapshotInputMappings).putData(content, data); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java index 1577253c850b..559af5628497 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java @@ -13,8 +13,8 @@ import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.Map; -public class FileContentHashIndex extends VfsAwareMapReduceIndex { - FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage storage) throws IOException { +public class FileContentHashIndex extends VfsAwareMapReduceIndex { + FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage storage) throws IOException { super(extension, storage, new PersistentMapBasedForwardIndex(IndexInfrastructure.getInputIndexStorageFile(extension.getName()).toPath(), false), @@ -23,22 +23,22 @@ public class FileContentHashIndex extends VfsAwareMapReduceIndex createIndexUpdateComputation(@NotNull AbstractUpdateData updateData) { + protected Computable createIndexUpdateComputation(@NotNull AbstractUpdateData updateData) { return new HashIndexUpdateComputable(super.createIndexUpdateComputation(updateData), updateData.newDataIsEmpty()); } - public int getHashId(int fileId) throws StorageException { - Map data = getIndexedFileData(fileId); - if (data.isEmpty()) return 0; + public Long getHashId(int fileId) throws StorageException { + Map data = getIndexedFileData(fileId); + if (data.isEmpty()) return FileContentHashIndexExtension.NULL_HASH_ID; return data.keySet().iterator().next(); } @NotNull - IntIntFunction toHashIdToFileIdFunction() { + IntIntFunction toHashIdToFileIdFunction(int indexId) { return hash -> { try { - ValueContainer data = getData(hash); - assert data.size() == 1; + ValueContainer data = getData(FileContentHashIndexExtension.getHashId(hash, indexId)); + if (data.size() == 0) return -1; return data.getValueIterator().getInputIdsIterator().next(); } catch (StorageException e) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java index 5505db0b08ea..0102b54f35a7 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java @@ -4,51 +4,68 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.Disposable; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Disposer; +import com.intellij.openapi.util.Key; import com.intellij.openapi.util.ShutDownTracker; import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; +import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.IndexStorage; -import com.intellij.util.io.*; +import com.intellij.util.indexing.snapshot.IndexedHashesSupport; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.DataInputOutputUtil; +import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.VoidDataExternalizer; import org.jetbrains.annotations.NotNull; import java.io.DataInput; import java.io.DataOutput; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; -public class FileContentHashIndexExtension extends FileBasedIndexExtension implements CustomImplementationFileBasedIndexExtension, CustomInputsIndexFileBasedIndexExtension, Disposable { +public class FileContentHashIndexExtension extends FileBasedIndexExtension implements CustomImplementationFileBasedIndexExtension, CustomInputsIndexFileBasedIndexExtension, Disposable { private static final Logger LOG = Logger.getInstance(FileContentHashIndexExtension.class); - public static final ID HASH_INDEX_ID = ID.create("file.content.hash.index"); + public static final ID HASH_INDEX_ID = ID.create("file.content.hash.index"); @NotNull - private final ContentHashesUtil.HashEnumerator myEnumerator; - private final int myDirHash; + private final ContentHashesUtil.HashEnumerator[] myEnumerators; @NotNull - public static FileContentHashIndexExtension create(@NotNull File enumeratorDir, @NotNull Disposable parent) throws IOException { - FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDir); + public static FileContentHashIndexExtension create(@NotNull Path[] enumeratorDirs, @NotNull Disposable parent) throws IOException { + FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDirs); + RebuildStatus.registerIndex(extension.getName()); Disposer.register(parent, extension); return extension; } - private FileContentHashIndexExtension(@NotNull File enumeratorDir) throws IOException { - myEnumerator = new ContentHashesUtil.HashEnumerator(enumeratorDir.toPath()); - myDirHash = enumeratorDir.getAbsolutePath().hashCode(); + private FileContentHashIndexExtension(@NotNull Path[] enumeratorDirs) throws IOException { + IOException[] exception = {null}; + myEnumerators = ContainerUtil.map2Array(enumeratorDirs, ContentHashesUtil.HashEnumerator.class, d -> { + try { + return new ContentHashesUtil.HashEnumerator(d.getParent().resolve("hashes")); + } + catch (IOException e) { + exception[0] = e; + return null; + } + }); + if (exception[0] != null) { + throw exception[0]; + } ShutDownTracker.getInstance().registerShutdownTask(() -> closeEnumerator()); } @NotNull @Override - public ID getName() { + public ID getName() { return HASH_INDEX_ID; } @NotNull @Override public FileBasedIndex.InputFilter getInputFilter() { - throw new UnsupportedOperationException(); + return file -> !file.isDirectory(); } @Override @@ -58,16 +75,15 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension getIndexer() { + public DataIndexer getIndexer() { return fc -> { - byte[] hash = ((FileContentImpl)fc).getHash(false); - LOG.assertTrue(hash != null); + long hashId = getHashId(fc); + if (hashId != NULL_HASH_ID) return Collections.singletonMap(hashId, null); + byte[] hash = IndexedHashesSupport.getOrInitIndexedHash((FileContentImpl) fc, false); try { - int id; - synchronized (myEnumerator) { - id = myEnumerator.tryEnumerate(hash); - } - return id == 0 ? Collections.emptyMap() : Collections.singletonMap(id, null); + hashId = tryEnumerate(hash); + setHashId(fc, hashId); + return hashId == NULL_HASH_ID ? Collections.emptyMap() : Collections.singletonMap(hashId, null); } catch (IOException e) { throw new RuntimeException(e); @@ -75,10 +91,44 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension getKeyDescriptor() { - return EnumeratorIntegerDescriptor.INSTANCE; + public KeyDescriptor getKeyDescriptor() { + return new KeyDescriptor() { + @Override + public int getHashCode(Long value) { + return value.hashCode(); + } + + @Override + public boolean isEqual(Long val1, Long val2) { + return val1.longValue() == val2.longValue(); + } + + @Override + public void save(@NotNull DataOutput out, Long value) throws IOException { + DataInputOutputUtil.writeLONG(out, value); + } + + @Override + public Long read(@NotNull DataInput in) throws IOException { + return DataInputOutputUtil.readLONG(in); + } + }; } @NotNull @@ -89,7 +139,7 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension> createExternalizer() { - return new DataExternalizer>() { + public DataExternalizer> createExternalizer() { + return new DataExternalizer>() { @Override - public void save(@NotNull DataOutput out, Collection value) throws IOException { + public void save(@NotNull DataOutput out, Collection value) throws IOException { assert value.isEmpty() || value.size() == 1; - DataInputOutputUtil.writeINT(out, value.isEmpty() ? 0 : value.iterator().next()); + DataInputOutputUtil.writeLONG(out, value.isEmpty() ? 0 : value.iterator().next()); } @Override - public Collection read(@NotNull DataInput in) throws IOException { - int id = DataInputOutputUtil.readINT(in); + public Collection read(@NotNull DataInput in) throws IOException { + long id = DataInputOutputUtil.readLONG(in); return id == 0 ? Collections.emptyList() : Collections.singleton(id); } }; } private void closeEnumerator() { - synchronized (myEnumerator) { - if (myEnumerator.isClosed()) return; - try { - myEnumerator.close(); - } - catch (IOException e) { - LOG.error(e); + for (ContentHashesUtil.HashEnumerator enumerator : myEnumerators) { + synchronized (enumerator) { + if (enumerator.isClosed()) return; + try { + enumerator.close(); + } + catch (IOException e) { + LOG.error(e); + } } } } @NotNull @Override - public UpdatableIndex createIndexImplementation(@NotNull FileBasedIndexExtension extension, - @NotNull IndexStorage storage) + public UpdatableIndex createIndexImplementation(@NotNull FileBasedIndexExtension extension, + @NotNull IndexStorage storage) throws IOException { return new FileContentHashIndex(((FileContentHashIndexExtension)extension), storage); } + + static int getIndexId(long hashId) { + return (int)(hashId >> 32); + } + + static int getInternalHashId(long hashId) { + return (int)hashId; + } + + static long getHashId(int internalHashId, int indexId) { + return (((long) indexId) << 32) | (internalHashId & 0xffffffffL); + } + + public static final long NULL_HASH_ID = getHashId(0, -1); + + public static final Key HASH_ID_KEY = Key.create("file.content.hash.id"); + + public static long getHashId(@NotNull FileContent content) { + Long value = HASH_ID_KEY.get(content); + return value == null ? NULL_HASH_ID : value; + } + + public static void setHashId(@NotNull FileContent content, long hashId) { + HASH_ID_KEY.set(content, hashId); + } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java index 3f217952c7cc..df95e9b7fd0a 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java @@ -2,11 +2,13 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.project.Project; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.VirtualFileVisitor; import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; +import com.intellij.psi.SingleRootFileViewProvider; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.InputData; import com.intellij.util.indexing.impl.MapIndexStorage; @@ -18,35 +20,33 @@ import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; +import java.util.*; +import java.util.concurrent.atomic.LongAdder; public class HashBasedIndexGenerator { - @NotNull private final File myOut; - @NotNull private final File myHashOut; - @NotNull private final FakeIndexExtension myExtension; - @NotNull private final FileBasedIndex.InputFilter myInputFilter; + @NotNull + private final File myOut; + @NotNull + private final FakeIndexExtension myExtension; + @NotNull + private final FileBasedIndex.InputFilter myInputFilter; - protected ContentHashesUtil.HashEnumerator myHashEnumerator; private InvertedIndex myIndex; public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension indexExtension, @NotNull File out) { this(indexExtension.getKeyDescriptor(), indexExtension.getValueExternalizer(), indexExtension, - out, - out); + out + ); } public HashBasedIndexGenerator(@NotNull KeyDescriptor keyDescriptor, @NotNull DataExternalizer valueExternalizer, @NotNull FileBasedIndexExtension originalExtension, - @NotNull File out, - @NotNull File hashOut) { + @NotNull File out) { myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension); myOut = out; - myHashOut = hashOut; FileBasedIndex.InputFilter filter = originalExtension.getInputFilter(); @@ -65,8 +65,8 @@ public class HashBasedIndexGenerator { } public void openIndex() throws IOException { - myHashEnumerator = getHashEnumerator(); String indexName = myExtension.getName().getName(); + boolean singleEntry = myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension; myIndex = new MapReduceIndex(myExtension, new MapIndexStorage(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName).toPath(), myExtension.getKeyDescriptor(), myExtension.getValueExternalizer(), @@ -77,6 +77,16 @@ public class HashBasedIndexGenerator { //ignore } }, null, null) { + @NotNull + @Override + protected Map mapByIndexer(int inputId, @NotNull FileContent content) { + Map data = super.mapByIndexer(inputId, content); + if (singleEntry && !data.isEmpty()) { + data = Collections.singletonMap((K)(Integer)inputId, data.values().iterator().next()); + } + return data; + } + @Override protected void updateForwardIndex(int inputId, @NotNull InputData data) throws IOException { super.updateForwardIndex(inputId, data); @@ -95,60 +105,83 @@ public class HashBasedIndexGenerator { @Override protected void requestRebuild(@NotNull Throwable e) { - throw new RuntimeException(e); + throw new RuntimeException("error while processing " + indexName, e); } }; } - @NotNull - protected ContentHashesUtil.HashEnumerator getHashEnumerator() throws IOException { - return new ContentHashesUtil.HashEnumerator(new File(myHashOut, "hashes").toPath()); - } - protected void visitInputData(int hashId, @NotNull InputData data) throws StorageException { } public void closeIndex() throws IOException { if (myIndex != null) myIndex.dispose(); - if (myHashEnumerator != null) myHashEnumerator.close(); } + public static void generate(@NotNull Collection roots, + @NotNull Collection> generators, + @NotNull Project project, + @NotNull File hashOut) { - public final void generate(@NotNull Collection roots) { + LongAdder l = new LongAdder(); try { - openIndex(); + ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(new File(hashOut, "hashes").toPath()); + + for (HashBasedIndexGenerator generator : generators) { + generator.openIndex(); + } for (VirtualFile root : roots) { VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor() { @Override public boolean visitFile(@NotNull VirtualFile file) { - if (!file.isDirectory() && myInputFilter.acceptInput(file)) { - indexFile(file); + + if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) { + for (HashBasedIndexGenerator generator : generators) { + if (generator.myInputFilter.acceptInput(file)) { + l.increment(); + generator.indexFile(file, project, hashEnumerator); + } + } } + return true; } }); } + + synchronized (hashEnumerator) { + hashEnumerator.close(); + } } catch (IOException e) { throw new RuntimeException(e); } finally { try { - closeIndex(); + for (HashBasedIndexGenerator generator : generators) { + generator.closeIndex(); + } } catch (IOException e) { throw new RuntimeException(e); } } + + System.out.println("Indexed " + l.sum() + " files to " + hashOut.getPath()); } - protected void indexFile(@NotNull VirtualFile f) { + protected void indexFile(@NotNull VirtualFile f, + @NotNull Project project, + @NotNull ContentHashesUtil.HashEnumerator hashEnumerator) { try { FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray()); - IndexedHashesSupport.initIndexedHash(fc); - int hashId = myHashEnumerator.enumerate(fc.getHash(false)); + byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(fc, false); + int hashId; + synchronized (hashEnumerator) { + hashId = Math.abs(hashEnumerator.enumerate(hash)); + } + fc.putUserData(IndexingDataKeys.PROJECT, project); if (!myIndex.update(hashId, fc).compute()) { throw new RuntimeException(); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java index fda43b0af67a..9cfd861edb42 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java @@ -2,14 +2,19 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.progress.ProgressManager; +import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.IntIntFunction; +import com.intellij.util.Processor; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.IndexStorage; import com.intellij.util.indexing.impl.MapIndexStorage; +import com.intellij.util.indexing.impl.UpdatableValueContainer; import com.intellij.util.indexing.provided.ProvidedIndexExtension; +import com.intellij.util.io.PersistentEnumeratorBase; +import com.intellij.util.io.PersistentHashMap; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; -import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -19,17 +24,20 @@ class HashBasedMapReduceIndex extends VfsAwareMapReduceIndex HashBasedMapReduceIndex create(@NotNull ProvidedIndexExtension providedExtension, - @NotNull FileBasedIndexExtension originalExtension) + @NotNull FileBasedIndexExtension originalExtension, + @NotNull FileContentHashIndex hashIndex, + int providedIndexId) throws IOException { Path file = providedExtension.getIndexPath(); - return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile())); + return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, hashIndex, providedIndexId); } private HashBasedMapReduceIndex(@NotNull Path baseFile, @NotNull FileBasedIndexExtension originalExtension, @NotNull ProvidedIndexExtension providedExtension, - @NotNull FileContentHashIndex hashIndex) throws IOException { - super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction()), null, null, null, null); + @NotNull FileContentHashIndex hashIndex, + int providedIndexId) throws IOException { + super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction(providedIndexId)), null, null, null, null); myProvidedExtension = providedExtension; } @@ -42,19 +50,51 @@ class HashBasedMapReduceIndex extends VfsAwareMapReduceIndex originalExtension, @NotNull ProvidedIndexExtension providedExtension, @NotNull IntIntFunction hashToFileId) throws IOException { - return new MapIndexStorage(baseFile.resolve(originalExtension.getName().getName()), - providedExtension.createKeyDescriptor(), - providedExtension.createValueExternalizer(), - originalExtension.getCacheSize(), - originalExtension.keyIsUniqueForIndexedFile(), - true, - true, - hashToFileId) { - @Override - protected void checkCanceled() { - ProgressManager.checkCanceled(); - } - }; + return new MyMapIndexStorage<>(baseFile, originalExtension, providedExtension, hashToFileId); } + private static class MyMapIndexStorage + extends MapIndexStorage + implements VfsAwareIndexStorage { + public MyMapIndexStorage(Path baseFile, + FileBasedIndexExtension originalExtension, + ProvidedIndexExtension providedExtension, + IntIntFunction hashToFileId) throws IOException { + super(baseFile.resolve(originalExtension.getName().getName()), providedExtension.createKeyDescriptor(), + providedExtension.createValueExternalizer(), originalExtension.getCacheSize(), originalExtension.keyIsUniqueForIndexedFile(), + true, true, hashToFileId); + } + + @Override + protected void checkCanceled() { + ProgressManager.checkCanceled(); + } + + @Override + public boolean processKeys(@NotNull Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) + throws StorageException { + l.lock(); + try { + myCache.clear(); // this will ensure that all new keys are made into the map + return doProcessKeys(processor); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + return unwrapCauseAndRethrow(e); + } + finally { + l.unlock(); + } + } + + private boolean doProcessKeys(@NotNull Processor processor) throws IOException { + return myMap instanceof PersistentHashMap && PersistentEnumeratorBase.inlineKeyStorage(myKeyDescriptor) + // process keys and check that they're already present in map because we don't have separated key storage we must check keys + ? ((PersistentHashMap>)myMap).processKeysWithExistingMapping(processor) + // optimization: process all keys, some of them might be already deleted but we don't care. We just read key storage file here + : myMap.processKeys(processor); + } + } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java index 0c60ea617422..66136caed8e5 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java @@ -4,57 +4,74 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.util.Computable; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; +import com.intellij.util.SmartList; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.AbstractUpdateData; +import com.intellij.util.indexing.impl.MergedValueContainer; import com.intellij.util.indexing.provided.ProvidedIndexExtension; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; +import java.nio.file.Files; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; +import java.util.stream.Stream; public class MergedInvertedIndex implements UpdatableIndex { @NotNull - private final HashBasedMapReduceIndex myProvidedIndex; + private final HashBasedMapReduceIndex[] myProvidedIndexes; @NotNull private final FileContentHashIndex myHashIndex; @NotNull - private final UpdatableIndex myBaseIndex; + public final UpdatableIndex myBaseIndex; @NotNull - public static MergedInvertedIndex create(@NotNull ProvidedIndexExtension providedExtension, + public static MergedInvertedIndex create(@NotNull List> providedExtensions, @NotNull FileBasedIndexExtension originalExtension, - @NotNull UpdatableIndex baseIndex) - throws IOException { - Path file = providedExtension.getIndexPath(); - HashBasedMapReduceIndex index = HashBasedMapReduceIndex.create(providedExtension, originalExtension); - return new MergedInvertedIndex<>(index, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile()), baseIndex); + @NotNull UpdatableIndex baseIndex, + @NotNull FileContentHashIndex contentHashIndex) throws IOException { + HashBasedMapReduceIndex[] providedIndexes = new HashBasedMapReduceIndex[providedExtensions.size()]; + for (int i = 0; i < providedExtensions.size(); i++) { + ProvidedIndexExtension extension = providedExtensions.get(i); + providedIndexes[i] = extension != null && Files.exists(extension.getIndexPath()) ? HashBasedMapReduceIndex.create(extension, originalExtension, contentHashIndex, i) : null; + } + + return new MergedInvertedIndex<>(providedIndexes, contentHashIndex, baseIndex); } - public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex index, + public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex[] indexes, @NotNull FileContentHashIndex hashIndex, @NotNull UpdatableIndex baseIndex) { - myProvidedIndex = index; + myProvidedIndexes = indexes; myHashIndex = hashIndex; myBaseIndex = baseIndex; } + @NotNull - public ProvidedIndexExtension getProvidedExtension() { - return myProvidedIndex.getProvidedExtension(); + public FileContentHashIndex getHashIndex() { + return myHashIndex; + } + + @NotNull + public Stream> getProvidedExtensions() { + return Stream.of(myProvidedIndexes).map(index -> index.getProvidedExtension()); } @NotNull @Override public Computable update(int inputId, @Nullable FileContent content) { if (content != null) { + long hashId = FileContentHashIndexExtension.getHashId(content); + if (hashId != FileContentHashIndexExtension.NULL_HASH_ID) { + return () -> Boolean.TRUE; + } //TODO if content == null Computable update = myHashIndex.update(inputId, content); if (!((FileContentHashIndex.HashIndexUpdateComputable)update).isEmptyInput()) return update; @@ -91,13 +108,28 @@ public class MergedInvertedIndex implements UpdatableIndex getData(@NotNull Key key) throws StorageException { - return MergedValueContainer.merge(myBaseIndex.getData(key), myProvidedIndex.getData(key)); + List> data = new SmartList<>(); + data.add(myBaseIndex.getData(key)); + for (HashBasedMapReduceIndex index : myProvidedIndexes) { + if (index == null) continue; + data.add(index.getData(key)); + } + return new MergedValueContainer<>(data); } @Override public boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException { - return myBaseIndex.processAllKeys(processor, scope, idFilter) && myProvidedIndex.processAllKeys(processor, scope, idFilter); + if (!myBaseIndex.processAllKeys(processor, scope, idFilter)) { + return false; + } + for (HashBasedMapReduceIndex index : myProvidedIndexes) { + if (index == null) continue; + if (!index.processAllKeys(processor, scope, idFilter)) { + return false; + } + } + return true; } @NotNull @@ -123,9 +155,9 @@ public class MergedInvertedIndex implements UpdatableIndex getIndexedFileData(int fileId) throws StorageException { Map data = myBaseIndex.getIndexedFileData(fileId); if (!data.isEmpty()) return data; - int hashId = myHashIndex.getHashId(fileId); - if (hashId == 0) return Collections.emptyMap(); - return myProvidedIndex.getIndexedFileData(hashId); + Long hashId = myHashIndex.getHashId(fileId); + if (hashId == null || hashId == FileContentHashIndexExtension.NULL_HASH_ID) return Collections.emptyMap(); + return myProvidedIndexes[FileContentHashIndexExtension.getIndexId(hashId)].getIndexedFileData(FileContentHashIndexExtension.getInternalHashId(hashId)); } @Override diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java deleted file mode 100644 index 42b2f3638b5e..000000000000 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -package com.intellij.util.indexing.hash; - -import com.intellij.util.indexing.ValueContainer; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -public class MergedValueContainer extends ValueContainer { - private final ValueContainer myContainer1; - private final ValueContainer myContainer2; - - @NotNull - public static ValueContainer merge(@NotNull ValueContainer container1, @NotNull ValueContainer container2) { - if (container1.size() == 0) return container2; - if (container2.size() == 0) return container1; - return new MergedValueContainer<>(container1, container2); - } - - private MergedValueContainer(@NotNull ValueContainer container1, @NotNull ValueContainer container2) { - myContainer1 = container1; - myContainer2 = container2; - } - - @NotNull - @Override - public ValueIterator getValueIterator() { - return new ValueIterator() { - boolean mySecondIsUsed; - ValueIterator myCurrent = myContainer1.getValueIterator(); - - @NotNull - @Override - public IntIterator getInputIdsIterator() { - return myCurrent.getInputIdsIterator(); - } - - @Nullable - @Override - public IntPredicate getValueAssociationPredicate() { - return myCurrent.getValueAssociationPredicate(); - } - - @Override - public boolean hasNext() { - if (myCurrent.hasNext()) return true; - if (!mySecondIsUsed) { - myCurrent = myContainer2.getValueIterator(); - mySecondIsUsed = true; - return hasNext(); - } - return false; - } - - @Override - public Value next() { - return myCurrent.next(); - } - }; - } - - @Override - public int size() { - return myContainer1.size() + myContainer2.size(); - } -} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java index fc604a9a2ab0..41645f077e81 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java @@ -2,15 +2,19 @@ package com.intellij.util.indexing.provided; import com.intellij.openapi.diagnostic.Logger; -import com.intellij.util.indexing.*; +import com.intellij.util.indexing.FileBasedIndexExtension; +import com.intellij.util.indexing.FileContent; +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.UpdatableIndex; +import com.intellij.util.indexing.hash.FileContentHashIndex; import com.intellij.util.indexing.hash.MergedInvertedIndex; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.KeyDescriptor; import org.jetbrains.annotations.NotNull; -import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.List; public interface ProvidedIndexExtension { Logger LOG = Logger.getInstance(ProvidedIndexExtension.class); @@ -28,15 +32,16 @@ public interface ProvidedIndexExtension { DataExternalizer createValueExternalizer(); @NotNull - static UpdatableIndex wrapWithProvidedIndex(@NotNull ProvidedIndexExtension providedIndexExtension, + static UpdatableIndex wrapWithProvidedIndex(@NotNull List> providedIndexExtensions, @NotNull FileBasedIndexExtension originalExtension, - @NotNull UpdatableIndex index) { + @NotNull UpdatableIndex index, + @NotNull FileContentHashIndex contentHashIndex) { try { - return MergedInvertedIndex.create(providedIndexExtension, originalExtension, index); + return MergedInvertedIndex.create(providedIndexExtensions, originalExtension, index, contentHashIndex); } catch (IOException e) { LOG.error(e); return index; } } -} +} \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java index 816a7d96f491..0a1911a9f6a9 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java @@ -4,18 +4,20 @@ package com.intellij.util.indexing.provided; import com.intellij.openapi.extensions.ExtensionPointName; import com.intellij.util.indexing.FileBasedIndexExtension; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; +import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; public interface ProvidedIndexExtensionLocator { ExtensionPointName EP_NAME = ExtensionPointName.create("com.intellij.fileBasedIndex.providedLocator"); - @Nullable - ProvidedIndexExtension findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension); + @NotNull + Stream> findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension); - @Nullable - static ProvidedIndexExtension findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension originalExtension) { - return EP_NAME.extensions().map(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).findFirst().orElse(null); + @NotNull + static List> findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension originalExtension) { + return EP_NAME.extensions().flatMap(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).collect(Collectors.toList()); } -} +} \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java index 0303e54a36c2..51318f3e0874 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java @@ -64,6 +64,17 @@ public class IndexedHashesSupport { content.setHashes(fileContentHash, documentHash != null ? documentHash : fileContentHash); } + @NotNull + public static byte[] getOrInitIndexedHash(@NotNull FileContentImpl content, boolean fromDocument) { + byte[] hash = content.getHash(fromDocument); + if (hash == null) { + initIndexedHash(content); + hash = content.getHash(fromDocument); + LOG.assertTrue(hash != null); + } + return hash; + } + @NotNull private static byte[] calculateIndexedHashForFileContent(@NotNull FileContentImpl content, boolean binary) { byte[] contentHash = null; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java index babdf7b29479..459f42008bc7 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java @@ -280,13 +280,7 @@ public class SnapshotInputMappings implements UpdatableSnapsh Integer previouslyCalculatedContentHashId = content.getUserData(key); if (previouslyCalculatedContentHashId == null) { - byte[] hash = content.getHash(fromDocument); - if (hash == null) { - IndexedHashesSupport.initIndexedHash(content); - hash = content.getHash(fromDocument); - LOG.assertTrue(hash != null); - } - + byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(content, fromDocument); previouslyCalculatedContentHashId = IndexedHashesSupport.enumerateHash(hash); content.putUserData(key, previouslyCalculatedContentHashId); } diff --git a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java index 74ceb672c0cb..ce242349b4c0 100644 --- a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java +++ b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java @@ -1,10 +1,10 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package org.jetbrains.index.stubs; +import com.google.common.collect.Maps; import com.intellij.openapi.application.PathManager; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; import com.intellij.psi.stubs.*; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.FileBasedIndexExtension; @@ -26,19 +26,19 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator myUsedKeys = new HashSet<>(); public StubHashBasedIndexGenerator(@NotNull File out) { - super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(false, null), getExtension(), out, out); + super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer( + true, + null, + StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), getExtension(), out); for (StubIndexExtension stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) { - myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(stubIndexExtension.getKeyDescriptor(), - StubIndexImpl.StubIdExternalizer.INSTANCE, - StubIndexImpl - .wrapStubIndexExtension(stubIndexExtension), - new File(out, getStubsDir()), - out) { - @NotNull - @Override - protected ContentHashesUtil.HashEnumerator getHashEnumerator() { - return StubHashBasedIndexGenerator.this.myHashEnumerator; - } + FileBasedIndexExtension ex = StubIndexImpl + .wrapStubIndexExtension(stubIndexExtension); + myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(ex.getKeyDescriptor(), + ex.getValueExternalizer(), + ex, + new File(out, getStubsDir()) + ) { + }); } } @@ -59,7 +59,8 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator value = entry.getValue(); myUsedKeys.add(key); MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex(); - index.updateWithMap(new UpdateData(hashId, value, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null)); + Map reducedValue = Maps.asMap(value.keySet(), k -> null); + index.updateWithMap(new UpdateData(hashId, reducedValue, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null)); } } @@ -83,6 +84,6 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator ex instanceof StubUpdatingIndex).findAny().get(); + return (new StubUpdatingIndex(StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE)); } } diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java index c492f166b961..b71bdeb7e66b 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java +++ b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java @@ -264,7 +264,7 @@ public abstract class MapReduceIndex implements InvertedIndex< @NotNull protected UpdateData calculateUpdateData(final int inputId, @Nullable Input content) { - final InputData data = mapInput(content); + final InputData data = mapInput(inputId, content); return createUpdateData(inputId, data.getKeyValues(), () -> getKeysDiffBuilder(inputId), @@ -302,16 +302,21 @@ public abstract class MapReduceIndex implements InvertedIndex< } @NotNull - protected InputData mapInput(@Nullable Input content) { + protected InputData mapInput(int inputId, @Nullable Input content) { if (content == null) { return InputData.empty(); } - Map data = myIndexer.map(content); + Map data = mapByIndexer(inputId, content); checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer); checkCanceled(); return new InputData<>(data); } + @NotNull + protected Map mapByIndexer(int inputId, @NotNull Input content) { + return myIndexer.map(content); + } + public abstract void checkCanceled(); protected abstract void requestRebuild(@NotNull Throwable e); diff --git a/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java b/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java index b238a1577670..4bbdc30277a5 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java +++ b/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java @@ -12,13 +12,6 @@ public class MergedValueContainer extends ValueContainer { private final List> myContainers; private int mySize; - @NotNull - public static ValueContainer merge(@NotNull ValueContainer container1, @NotNull ValueContainer container2) { - if (container1.size() == 0) return container2; - if (container2.size() == 0) return container1; - return new MergedValueContainer<>(Arrays.asList(container1, container2)); - } - public MergedValueContainer(@NotNull List> containers) { if (containers.isEmpty()) { throw new IllegalArgumentException();