From dc12dbeda1e86205fa342c3e9412484d4d4c2133 Mon Sep 17 00:00:00 2001 From: Dmitry Batkovich Date: Mon, 23 Dec 2019 17:12:43 +0300 Subject: [PATCH] Revert "allow to load multiple shared indexes" This reverts commit 9b108cbe GitOrigin-RevId: 2c1c84c91929936bb4c1f49d4ab833edeabfc552 --- .../psi/stubs/SerializedStubTree.java | 31 +--- .../SerializedStubTreeDataExternalizer.java | 16 +- .../index/BasicProvidedExtensionLocator.java | 66 ++------ .../psi/stubs/StubProcessingHelper.java | 2 +- .../StubUpdatingForwardIndexAccessor.java | 2 +- .../intellij/psi/stubs/StubUpdatingIndex.java | 11 +- .../provided/StubProvidedIndexExtension.java | 5 +- .../util/indexing/FileBasedIndexImpl.java | 21 +-- .../intellij/util/indexing/RebuildStatus.java | 4 +- .../util/indexing/VfsAwareMapReduceIndex.java | 4 +- .../indexing/hash/FileContentHashIndex.java | 18 +- .../hash/FileContentHashIndexExtension.java | 155 +++++------------- .../hash/HashBasedIndexGenerator.java | 91 ++++------ .../hash/HashBasedMapReduceIndex.java | 76 ++------- .../indexing/hash/MergedInvertedIndex.java | 70 +++----- .../indexing/hash/MergedValueContainer.java | 65 ++++++++ .../provided/ProvidedIndexExtension.java | 17 +- .../ProvidedIndexExtensionLocator.java | 16 +- .../snapshot/IndexedHashesSupport.java | 11 -- .../snapshot/SnapshotInputMappings.java | 8 +- .../stubs/StubHashBasedIndexGenerator.java | 31 ++-- .../util/indexing/impl/MapReduceIndex.java | 11 +- .../indexing/impl/MergedValueContainer.java | 7 + 23 files changed, 275 insertions(+), 463 deletions(-) create mode 100644 platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java diff --git a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java index a41fc7d0ff02..b08bdc71728d 100644 --- a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java +++ b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java @@ -45,16 +45,11 @@ public class SerializedStubTree { private Map> myIndexedStubs; private volatile SerializationManagerEx mySerializationManager; - private volatile StubForwardIndexExternalizer myStubIndexesExternalizer; - public void setSerializationManager(@NotNull SerializationManagerEx serializationManager) { + public void setSerializationManager(SerializationManagerEx serializationManager) { mySerializationManager = serializationManager; } - public void setStubIndexesExternalizer(@NotNull StubForwardIndexExternalizer stubIndexesExternalizer) { - myStubIndexesExternalizer = stubIndexesExternalizer; - } - public SerializedStubTree(@NotNull byte[] treeBytes, int treeByteLength, @Nullable Stub stubElement, @NotNull byte[] indexedStubBytes, int indexedStubByteLength, @Nullable Map> indexedStubs) { myTreeBytes = treeBytes; @@ -79,7 +74,6 @@ public class SerializedStubTree { forwardIndexExternalizer.save(new DataOutputStream(indexBytes), myIndexedStubs); myIndexedStubBytes = indexBytes.getInternalBuffer(); myIndexedStubByteLength = indexBytes.size(); - myStubIndexesExternalizer = forwardIndexExternalizer; } @NotNull @@ -100,7 +94,7 @@ public class SerializedStubTree { else { BufferExposingByteArrayOutputStream reSerializedStubIndices = new BufferExposingByteArrayOutputStream(); if (myIndexedStubs == null) { - restoreIndexedStubs(); + restoreIndexedStubs(currentForwardIndexSerializer); } assert myIndexedStubs != null; newForwardIndexSerializer.save(new DataOutputStream(reSerializedStubIndices), myIndexedStubs); @@ -108,25 +102,18 @@ public class SerializedStubTree { reSerializedIndexByteLength = reSerializedStubIndices.size(); } - SerializedStubTree tree = new SerializedStubTree( - outStub.getInternalBuffer(), - outStub.size(), - null, - reSerializedIndexBytes, - reSerializedIndexByteLength, - myIndexedStubs); - tree.setStubIndexesExternalizer(myStubIndexesExternalizer); - return tree; + return new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null, + reSerializedIndexBytes, reSerializedIndexByteLength, myIndexedStubs); } - void restoreIndexedStubs() throws IOException { + void restoreIndexedStubs(@NotNull StubForwardIndexExternalizer dataExternalizer) throws IOException { if (myIndexedStubs == null) { - myIndexedStubs = myStubIndexesExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength))); + myIndexedStubs = dataExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength))); } } - StubIdList restoreIndexedStubs(@NotNull StubIndexKey indexKey, @NotNull K key) throws IOException { - Map> incompleteMap = myStubIndexesExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key); + StubIdList restoreIndexedStubs(@NotNull StubForwardIndexExternalizer dataExternalizer, @NotNull StubIndexKey indexKey, @NotNull K key) throws IOException { + Map> incompleteMap = dataExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key); Map map = incompleteMap.get(indexKey); return map == null ? null : map.get(key); } @@ -138,7 +125,7 @@ public class SerializedStubTree { @TestOnly public Map> readStubIndicesValueMap() throws IOException { - restoreIndexedStubs(); + restoreIndexedStubs(IDE_USED_EXTERNALIZER); return myIndexedStubs; } diff --git a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java index 6fb71271d631..46d38fca1903 100644 --- a/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java +++ b/platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java @@ -15,16 +15,14 @@ import java.io.IOException; public class SerializedStubTreeDataExternalizer implements DataExternalizer { private final boolean myIncludeInputs; private final SerializationManagerEx mySerializationManager; - private final StubForwardIndexExternalizer myStubIndexesExternalizer; public SerializedStubTreeDataExternalizer() { - this(true, null, SerializedStubTree.IDE_USED_EXTERNALIZER); + this(true, null); } - public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager, StubForwardIndexExternalizer externalizer) { + public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager) { myIncludeInputs = inputs; mySerializationManager = manager; - myStubIndexesExternalizer = externalizer; } @Override @@ -46,7 +44,6 @@ public class SerializedStubTreeDataExternalizer implements DataExternalizer Stream> findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension) { - if (!originalExtension.dependsOnFileContent()) return Stream.empty(); + public ProvidedIndexExtension findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension) { Path root = getPrebuiltIndexPath(); - if (root == null || !Files.exists(root)) return Stream.empty(); + if (root == null || !Files.exists(root.resolve( StringUtil.toLowerCase(originalExtension.getName().getName())))) return null; - try { - // TODO properly close it - UncompressedZipFileSystem fs = getFs(root); - - Path fsRoot = fs.getPath("/").getRoot(); - return Files - .list(fsRoot) - .sorted(Comparator.comparing(p -> p.getFileName().toString())) - .map(p -> p.resolve(StringUtil.toLowerCase(originalExtension.getName().getName()))) - .map(p -> { - return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID) - ? (ProvidedIndexExtension)new StubProvidedIndexExtension(p) - : new ProvidedIndexExtensionImpl<>(p, originalExtension); - }); - } catch (IOException e) { - LOG.error(e); - return Stream.empty(); - } - } - - private synchronized static UncompressedZipFileSystem getFs(@NotNull Path root) throws IOException { - if (ourFs == null) { - ourFs = new UncompressedZipFileSystem(root, new UncompressedZipFileSystemProvider()); - ShutDownTracker.getInstance().registerShutdownTask(() -> { - try { - ourFs.close(); - } - catch (IOException e) { - LOG.error(e); - } - }); - } - return ourFs; + return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID) + ? (ProvidedIndexExtension)new StubProvidedIndexExtension(root) + : new ProvidedIndexExtensionImpl<>(root, originalExtension); } @Nullable private static Path getPrebuiltIndexPath() { - String path = System.getProperty(PREBUILT_INDEX_ZIP_PROP); + String path = System.getProperty(PREBUILT_INDEX_PATH_PROP); if (path == null) return null; - Path file = Paths.get(path); + Path file; + try { + file = Paths.get(new URI(path)); + } + catch (URISyntaxException e) { + LOG.error(e); + return null; + } return Files.exists(file) ? file : null; } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java index 9dbaa5d032e8..1393a40f8a17 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java @@ -37,7 +37,7 @@ public final class StubProcessingHelper extends StubProcessingHelperBase { return null; } SerializedStubTree tree = data.values().iterator().next(); - StubIdList stubIdList = tree.restoreIndexedStubs(indexKey, key); + StubIdList stubIdList = tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER, indexKey, key); if (stubIdList == null) { LOG.error("Stub ids not found for key in index = " + indexKey.getName() + ", file type = " + file.getFileType()); onInternalError(file); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java index 89abf9c4bc49..4e71a819a7db 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java @@ -37,7 +37,7 @@ class StubUpdatingForwardIndexAccessor implements ForwardIndexAccessor data = dataRef.get(); SerializedStubTree tree = ContainerUtil.isEmpty(data) ? null : ContainerUtil.getFirstItem(data.values()); if (tree != null) { - tree.restoreIndexedStubs(); + tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER); } return new StubCumulativeInputDiffBuilder(inputId, tree); } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 74b374baeef9..79564493bbbf 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -47,15 +47,6 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension INDEX_ID = ID.create("Stubs"); private static final FileBasedIndex.InputFilter INPUT_FILTER = file -> canHaveStub(file); - private final StubForwardIndexExternalizer myStubIndexesExternalizer; - - public StubUpdatingIndex() { - myStubIndexesExternalizer = SerializedStubTree.IDE_USED_EXTERNALIZER; - } - - public StubUpdatingIndex(StubForwardIndexExternalizer stubIndexesExternalizer) { - myStubIndexesExternalizer = stubIndexesExternalizer; - } public static boolean canHaveStub(@NotNull VirtualFile file) { Project project = ProjectUtil.guessProjectForFile(file); @@ -247,7 +238,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension getValueExternalizer() { - return new SerializedStubTreeDataExternalizer(true, null, myStubIndexesExternalizer); + return new SerializedStubTreeDataExternalizer(); } @NotNull diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java b/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java index 670b59885214..5e5a410f1712 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java @@ -5,6 +5,8 @@ import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.text.StringUtil; import com.intellij.psi.stubs.*; +import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.FileBasedIndexImpl; import com.intellij.util.indexing.ID; import com.intellij.util.indexing.provided.ProvidedIndexExtension; import com.intellij.util.io.DataExternalizer; @@ -14,6 +16,7 @@ import com.intellij.util.io.VoidDataExternalizer; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; @@ -49,7 +52,7 @@ public class StubProvidedIndexExtension implements ProvidedIndexExtension index = createIndex(extension, new MemoryIndexStorage<>(storage, name)); - if (!(extension instanceof FileContentHashIndexExtension)) { - List> providedExtensions = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension); - if (!providedExtensions.isEmpty()) { - Path[] paths = ContainerUtil.map2Array(providedExtensions, Path.class, ex -> ex.getIndexPath()); - FileContentHashIndex contentHashIndex = ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(paths, state); - - index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtensions, extension, index, contentHashIndex); - } + ProvidedIndexExtension providedExtension = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension); + if (providedExtension != null) { + index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtension, extension, index); } state.registerIndex(name, @@ -2683,20 +2677,19 @@ public final class FileBasedIndexImpl extends FileBasedIndex { } } - public synchronized FileContentHashIndex getFileContentHashIndex(@Nullable Path[] enumeratorPaths, @NotNull IndexConfiguration state) { - UpdatableIndex index = state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); + public synchronized FileContentHashIndex getFileContentHashIndex(@NotNull File enumeratorPath) { + UpdatableIndex index = getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); if (index == null) { - LOG.assertTrue(enumeratorPaths != null); IndicesRegistrationResult registrationResult = new IndicesRegistrationResult(); try { - registerIndexer(FileContentHashIndexExtension.create(enumeratorPaths, ApplicationManager.getApplication()), state, registrationResult); + registerIndexer(FileContentHashIndexExtension.create(enumeratorPath, ApplicationManager.getApplication()), myRegisteredIndexes.getState(), registrationResult); registrationResult.logChangedAndFullyBuiltIndices(LOG, "Version was changed for:", "Index is to be rebuilt:"); } catch (IOException e) { throw new RuntimeException(e); } } else return (FileContentHashIndex)index; - return (FileContentHashIndex)state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); + return (FileContentHashIndex)getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID); } private static final boolean INDICES_ARE_PSI_DEPENDENT_BY_DEFAULT = SystemProperties.getBooleanProperty("idea.indices.psi.dependent.default", true); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java b/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java index bbd6e1ecce55..03878dc6ff3e 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java @@ -12,14 +12,14 @@ import java.util.concurrent.atomic.AtomicReference; /** * @author peter */ -public enum RebuildStatus { +enum RebuildStatus { OK, REQUIRES_REBUILD, DOING_REBUILD; private static final Map, AtomicReference> ourRebuildStatus = new THashMap<>(); - public static void registerIndex(ID indexId) { + static void registerIndex(ID indexId) { ourRebuildStatus.put(indexId, new AtomicReference<>(OK)); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java index 7d6dc83a00f8..336491ac0d64 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java @@ -119,7 +119,7 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex mapInput(int inputId, @Nullable Input content) { + protected InputData mapInput(@Nullable Input content) { InputData data; boolean containsSnapshotData = true; if (mySnapshotInputMappings != null && content != null) { @@ -135,7 +135,7 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex)mySnapshotInputMappings).putData(content, data); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java index 559af5628497..1577253c850b 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java @@ -13,8 +13,8 @@ import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.Map; -public class FileContentHashIndex extends VfsAwareMapReduceIndex { - FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage storage) throws IOException { +public class FileContentHashIndex extends VfsAwareMapReduceIndex { + FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage storage) throws IOException { super(extension, storage, new PersistentMapBasedForwardIndex(IndexInfrastructure.getInputIndexStorageFile(extension.getName()).toPath(), false), @@ -23,22 +23,22 @@ public class FileContentHashIndex extends VfsAwareMapReduceIndex createIndexUpdateComputation(@NotNull AbstractUpdateData updateData) { + protected Computable createIndexUpdateComputation(@NotNull AbstractUpdateData updateData) { return new HashIndexUpdateComputable(super.createIndexUpdateComputation(updateData), updateData.newDataIsEmpty()); } - public Long getHashId(int fileId) throws StorageException { - Map data = getIndexedFileData(fileId); - if (data.isEmpty()) return FileContentHashIndexExtension.NULL_HASH_ID; + public int getHashId(int fileId) throws StorageException { + Map data = getIndexedFileData(fileId); + if (data.isEmpty()) return 0; return data.keySet().iterator().next(); } @NotNull - IntIntFunction toHashIdToFileIdFunction(int indexId) { + IntIntFunction toHashIdToFileIdFunction() { return hash -> { try { - ValueContainer data = getData(FileContentHashIndexExtension.getHashId(hash, indexId)); - if (data.size() == 0) return -1; + ValueContainer data = getData(hash); + assert data.size() == 1; return data.getValueIterator().getInputIdsIterator().next(); } catch (StorageException e) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java index 0102b54f35a7..5505db0b08ea 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java @@ -4,68 +4,51 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.Disposable; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Disposer; -import com.intellij.openapi.util.Key; import com.intellij.openapi.util.ShutDownTracker; import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; -import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.IndexStorage; -import com.intellij.util.indexing.snapshot.IndexedHashesSupport; -import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.DataInputOutputUtil; -import com.intellij.util.io.KeyDescriptor; -import com.intellij.util.io.VoidDataExternalizer; +import com.intellij.util.io.*; import org.jetbrains.annotations.NotNull; import java.io.DataInput; import java.io.DataOutput; +import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.Collection; import java.util.Collections; -public class FileContentHashIndexExtension extends FileBasedIndexExtension implements CustomImplementationFileBasedIndexExtension, CustomInputsIndexFileBasedIndexExtension, Disposable { +public class FileContentHashIndexExtension extends FileBasedIndexExtension implements CustomImplementationFileBasedIndexExtension, CustomInputsIndexFileBasedIndexExtension, Disposable { private static final Logger LOG = Logger.getInstance(FileContentHashIndexExtension.class); - public static final ID HASH_INDEX_ID = ID.create("file.content.hash.index"); + public static final ID HASH_INDEX_ID = ID.create("file.content.hash.index"); @NotNull - private final ContentHashesUtil.HashEnumerator[] myEnumerators; + private final ContentHashesUtil.HashEnumerator myEnumerator; + private final int myDirHash; @NotNull - public static FileContentHashIndexExtension create(@NotNull Path[] enumeratorDirs, @NotNull Disposable parent) throws IOException { - FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDirs); - RebuildStatus.registerIndex(extension.getName()); + public static FileContentHashIndexExtension create(@NotNull File enumeratorDir, @NotNull Disposable parent) throws IOException { + FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDir); Disposer.register(parent, extension); return extension; } - private FileContentHashIndexExtension(@NotNull Path[] enumeratorDirs) throws IOException { - IOException[] exception = {null}; - myEnumerators = ContainerUtil.map2Array(enumeratorDirs, ContentHashesUtil.HashEnumerator.class, d -> { - try { - return new ContentHashesUtil.HashEnumerator(d.getParent().resolve("hashes")); - } - catch (IOException e) { - exception[0] = e; - return null; - } - }); - if (exception[0] != null) { - throw exception[0]; - } + private FileContentHashIndexExtension(@NotNull File enumeratorDir) throws IOException { + myEnumerator = new ContentHashesUtil.HashEnumerator(enumeratorDir.toPath()); + myDirHash = enumeratorDir.getAbsolutePath().hashCode(); ShutDownTracker.getInstance().registerShutdownTask(() -> closeEnumerator()); } @NotNull @Override - public ID getName() { + public ID getName() { return HASH_INDEX_ID; } @NotNull @Override public FileBasedIndex.InputFilter getInputFilter() { - return file -> !file.isDirectory(); + throw new UnsupportedOperationException(); } @Override @@ -75,15 +58,16 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension getIndexer() { + public DataIndexer getIndexer() { return fc -> { - long hashId = getHashId(fc); - if (hashId != NULL_HASH_ID) return Collections.singletonMap(hashId, null); - byte[] hash = IndexedHashesSupport.getOrInitIndexedHash((FileContentImpl) fc, false); + byte[] hash = ((FileContentImpl)fc).getHash(false); + LOG.assertTrue(hash != null); try { - hashId = tryEnumerate(hash); - setHashId(fc, hashId); - return hashId == NULL_HASH_ID ? Collections.emptyMap() : Collections.singletonMap(hashId, null); + int id; + synchronized (myEnumerator) { + id = myEnumerator.tryEnumerate(hash); + } + return id == 0 ? Collections.emptyMap() : Collections.singletonMap(id, null); } catch (IOException e) { throw new RuntimeException(e); @@ -91,44 +75,10 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension getKeyDescriptor() { - return new KeyDescriptor() { - @Override - public int getHashCode(Long value) { - return value.hashCode(); - } - - @Override - public boolean isEqual(Long val1, Long val2) { - return val1.longValue() == val2.longValue(); - } - - @Override - public void save(@NotNull DataOutput out, Long value) throws IOException { - DataInputOutputUtil.writeLONG(out, value); - } - - @Override - public Long read(@NotNull DataInput in) throws IOException { - return DataInputOutputUtil.readLONG(in); - } - }; + public KeyDescriptor getKeyDescriptor() { + return EnumeratorIntegerDescriptor.INSTANCE; } @NotNull @@ -139,7 +89,7 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension> createExternalizer() { - return new DataExternalizer>() { + public DataExternalizer> createExternalizer() { + return new DataExternalizer>() { @Override - public void save(@NotNull DataOutput out, Collection value) throws IOException { + public void save(@NotNull DataOutput out, Collection value) throws IOException { assert value.isEmpty() || value.size() == 1; - DataInputOutputUtil.writeLONG(out, value.isEmpty() ? 0 : value.iterator().next()); + DataInputOutputUtil.writeINT(out, value.isEmpty() ? 0 : value.iterator().next()); } @Override - public Collection read(@NotNull DataInput in) throws IOException { - long id = DataInputOutputUtil.readLONG(in); + public Collection read(@NotNull DataInput in) throws IOException { + int id = DataInputOutputUtil.readINT(in); return id == 0 ? Collections.emptyList() : Collections.singleton(id); } }; } private void closeEnumerator() { - for (ContentHashesUtil.HashEnumerator enumerator : myEnumerators) { - synchronized (enumerator) { - if (enumerator.isClosed()) return; - try { - enumerator.close(); - } - catch (IOException e) { - LOG.error(e); - } + synchronized (myEnumerator) { + if (myEnumerator.isClosed()) return; + try { + myEnumerator.close(); + } + catch (IOException e) { + LOG.error(e); } } } @NotNull @Override - public UpdatableIndex createIndexImplementation(@NotNull FileBasedIndexExtension extension, - @NotNull IndexStorage storage) + public UpdatableIndex createIndexImplementation(@NotNull FileBasedIndexExtension extension, + @NotNull IndexStorage storage) throws IOException { return new FileContentHashIndex(((FileContentHashIndexExtension)extension), storage); } - - static int getIndexId(long hashId) { - return (int)(hashId >> 32); - } - - static int getInternalHashId(long hashId) { - return (int)hashId; - } - - static long getHashId(int internalHashId, int indexId) { - return (((long) indexId) << 32) | (internalHashId & 0xffffffffL); - } - - public static final long NULL_HASH_ID = getHashId(0, -1); - - public static final Key HASH_ID_KEY = Key.create("file.content.hash.id"); - - public static long getHashId(@NotNull FileContent content) { - Long value = HASH_ID_KEY.get(content); - return value == null ? NULL_HASH_ID : value; - } - - public static void setHashId(@NotNull FileContent content, long hashId) { - HASH_ID_KEY.set(content, hashId); - } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java index df95e9b7fd0a..3f217952c7cc 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java @@ -2,13 +2,11 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.fileTypes.FileType; -import com.intellij.openapi.project.Project; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.VirtualFileVisitor; import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; -import com.intellij.psi.SingleRootFileViewProvider; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.InputData; import com.intellij.util.indexing.impl.MapIndexStorage; @@ -20,33 +18,35 @@ import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; -import java.util.*; -import java.util.concurrent.atomic.LongAdder; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; public class HashBasedIndexGenerator { - @NotNull - private final File myOut; - @NotNull - private final FakeIndexExtension myExtension; - @NotNull - private final FileBasedIndex.InputFilter myInputFilter; + @NotNull private final File myOut; + @NotNull private final File myHashOut; + @NotNull private final FakeIndexExtension myExtension; + @NotNull private final FileBasedIndex.InputFilter myInputFilter; + protected ContentHashesUtil.HashEnumerator myHashEnumerator; private InvertedIndex myIndex; public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension indexExtension, @NotNull File out) { this(indexExtension.getKeyDescriptor(), indexExtension.getValueExternalizer(), indexExtension, - out - ); + out, + out); } public HashBasedIndexGenerator(@NotNull KeyDescriptor keyDescriptor, @NotNull DataExternalizer valueExternalizer, @NotNull FileBasedIndexExtension originalExtension, - @NotNull File out) { + @NotNull File out, + @NotNull File hashOut) { myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension); myOut = out; + myHashOut = hashOut; FileBasedIndex.InputFilter filter = originalExtension.getInputFilter(); @@ -65,8 +65,8 @@ public class HashBasedIndexGenerator { } public void openIndex() throws IOException { + myHashEnumerator = getHashEnumerator(); String indexName = myExtension.getName().getName(); - boolean singleEntry = myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension; myIndex = new MapReduceIndex(myExtension, new MapIndexStorage(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName).toPath(), myExtension.getKeyDescriptor(), myExtension.getValueExternalizer(), @@ -77,16 +77,6 @@ public class HashBasedIndexGenerator { //ignore } }, null, null) { - @NotNull - @Override - protected Map mapByIndexer(int inputId, @NotNull FileContent content) { - Map data = super.mapByIndexer(inputId, content); - if (singleEntry && !data.isEmpty()) { - data = Collections.singletonMap((K)(Integer)inputId, data.values().iterator().next()); - } - return data; - } - @Override protected void updateForwardIndex(int inputId, @NotNull InputData data) throws IOException { super.updateForwardIndex(inputId, data); @@ -105,83 +95,60 @@ public class HashBasedIndexGenerator { @Override protected void requestRebuild(@NotNull Throwable e) { - throw new RuntimeException("error while processing " + indexName, e); + throw new RuntimeException(e); } }; } + @NotNull + protected ContentHashesUtil.HashEnumerator getHashEnumerator() throws IOException { + return new ContentHashesUtil.HashEnumerator(new File(myHashOut, "hashes").toPath()); + } + protected void visitInputData(int hashId, @NotNull InputData data) throws StorageException { } public void closeIndex() throws IOException { if (myIndex != null) myIndex.dispose(); + if (myHashEnumerator != null) myHashEnumerator.close(); } - public static void generate(@NotNull Collection roots, - @NotNull Collection> generators, - @NotNull Project project, - @NotNull File hashOut) { - LongAdder l = new LongAdder(); + public final void generate(@NotNull Collection roots) { try { - ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(new File(hashOut, "hashes").toPath()); - - for (HashBasedIndexGenerator generator : generators) { - generator.openIndex(); - } + openIndex(); for (VirtualFile root : roots) { VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor() { @Override public boolean visitFile(@NotNull VirtualFile file) { - - if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) { - for (HashBasedIndexGenerator generator : generators) { - if (generator.myInputFilter.acceptInput(file)) { - l.increment(); - generator.indexFile(file, project, hashEnumerator); - } - } + if (!file.isDirectory() && myInputFilter.acceptInput(file)) { + indexFile(file); } - return true; } }); } - - synchronized (hashEnumerator) { - hashEnumerator.close(); - } } catch (IOException e) { throw new RuntimeException(e); } finally { try { - for (HashBasedIndexGenerator generator : generators) { - generator.closeIndex(); - } + closeIndex(); } catch (IOException e) { throw new RuntimeException(e); } } - - System.out.println("Indexed " + l.sum() + " files to " + hashOut.getPath()); } - protected void indexFile(@NotNull VirtualFile f, - @NotNull Project project, - @NotNull ContentHashesUtil.HashEnumerator hashEnumerator) { + protected void indexFile(@NotNull VirtualFile f) { try { FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray()); - byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(fc, false); - int hashId; - synchronized (hashEnumerator) { - hashId = Math.abs(hashEnumerator.enumerate(hash)); - } - fc.putUserData(IndexingDataKeys.PROJECT, project); + IndexedHashesSupport.initIndexedHash(fc); + int hashId = myHashEnumerator.enumerate(fc.getHash(false)); if (!myIndex.update(hashId, fc).compute()) { throw new RuntimeException(); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java index 9cfd861edb42..fda43b0af67a 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java @@ -2,19 +2,14 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.progress.ProgressManager; -import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.IntIntFunction; -import com.intellij.util.Processor; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.IndexStorage; import com.intellij.util.indexing.impl.MapIndexStorage; -import com.intellij.util.indexing.impl.UpdatableValueContainer; import com.intellij.util.indexing.provided.ProvidedIndexExtension; -import com.intellij.util.io.PersistentEnumeratorBase; -import com.intellij.util.io.PersistentHashMap; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; +import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -24,20 +19,17 @@ class HashBasedMapReduceIndex extends VfsAwareMapReduceIndex HashBasedMapReduceIndex create(@NotNull ProvidedIndexExtension providedExtension, - @NotNull FileBasedIndexExtension originalExtension, - @NotNull FileContentHashIndex hashIndex, - int providedIndexId) + @NotNull FileBasedIndexExtension originalExtension) throws IOException { Path file = providedExtension.getIndexPath(); - return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, hashIndex, providedIndexId); + return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile())); } private HashBasedMapReduceIndex(@NotNull Path baseFile, @NotNull FileBasedIndexExtension originalExtension, @NotNull ProvidedIndexExtension providedExtension, - @NotNull FileContentHashIndex hashIndex, - int providedIndexId) throws IOException { - super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction(providedIndexId)), null, null, null, null); + @NotNull FileContentHashIndex hashIndex) throws IOException { + super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction()), null, null, null, null); myProvidedExtension = providedExtension; } @@ -50,51 +42,19 @@ class HashBasedMapReduceIndex extends VfsAwareMapReduceIndex originalExtension, @NotNull ProvidedIndexExtension providedExtension, @NotNull IntIntFunction hashToFileId) throws IOException { - return new MyMapIndexStorage<>(baseFile, originalExtension, providedExtension, hashToFileId); + return new MapIndexStorage(baseFile.resolve(originalExtension.getName().getName()), + providedExtension.createKeyDescriptor(), + providedExtension.createValueExternalizer(), + originalExtension.getCacheSize(), + originalExtension.keyIsUniqueForIndexedFile(), + true, + true, + hashToFileId) { + @Override + protected void checkCanceled() { + ProgressManager.checkCanceled(); + } + }; } - private static class MyMapIndexStorage - extends MapIndexStorage - implements VfsAwareIndexStorage { - public MyMapIndexStorage(Path baseFile, - FileBasedIndexExtension originalExtension, - ProvidedIndexExtension providedExtension, - IntIntFunction hashToFileId) throws IOException { - super(baseFile.resolve(originalExtension.getName().getName()), providedExtension.createKeyDescriptor(), - providedExtension.createValueExternalizer(), originalExtension.getCacheSize(), originalExtension.keyIsUniqueForIndexedFile(), - true, true, hashToFileId); - } - - @Override - protected void checkCanceled() { - ProgressManager.checkCanceled(); - } - - @Override - public boolean processKeys(@NotNull Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) - throws StorageException { - l.lock(); - try { - myCache.clear(); // this will ensure that all new keys are made into the map - return doProcessKeys(processor); - } - catch (IOException e) { - throw new StorageException(e); - } - catch (RuntimeException e) { - return unwrapCauseAndRethrow(e); - } - finally { - l.unlock(); - } - } - - private boolean doProcessKeys(@NotNull Processor processor) throws IOException { - return myMap instanceof PersistentHashMap && PersistentEnumeratorBase.inlineKeyStorage(myKeyDescriptor) - // process keys and check that they're already present in map because we don't have separated key storage we must check keys - ? ((PersistentHashMap>)myMap).processKeysWithExistingMapping(processor) - // optimization: process all keys, some of them might be already deleted but we don't care. We just read key storage file here - : myMap.processKeys(processor); - } - } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java index 66136caed8e5..0c60ea617422 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java @@ -4,74 +4,57 @@ package com.intellij.util.indexing.hash; import com.intellij.openapi.util.Computable; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; -import com.intellij.util.SmartList; import com.intellij.util.indexing.*; import com.intellij.util.indexing.impl.AbstractUpdateData; -import com.intellij.util.indexing.impl.MergedValueContainer; import com.intellij.util.indexing.provided.ProvidedIndexExtension; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; +import java.io.File; import java.io.IOException; -import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; -import java.util.stream.Stream; public class MergedInvertedIndex implements UpdatableIndex { @NotNull - private final HashBasedMapReduceIndex[] myProvidedIndexes; + private final HashBasedMapReduceIndex myProvidedIndex; @NotNull private final FileContentHashIndex myHashIndex; @NotNull - public final UpdatableIndex myBaseIndex; + private final UpdatableIndex myBaseIndex; @NotNull - public static MergedInvertedIndex create(@NotNull List> providedExtensions, + public static MergedInvertedIndex create(@NotNull ProvidedIndexExtension providedExtension, @NotNull FileBasedIndexExtension originalExtension, - @NotNull UpdatableIndex baseIndex, - @NotNull FileContentHashIndex contentHashIndex) throws IOException { - HashBasedMapReduceIndex[] providedIndexes = new HashBasedMapReduceIndex[providedExtensions.size()]; - for (int i = 0; i < providedExtensions.size(); i++) { - ProvidedIndexExtension extension = providedExtensions.get(i); - providedIndexes[i] = extension != null && Files.exists(extension.getIndexPath()) ? HashBasedMapReduceIndex.create(extension, originalExtension, contentHashIndex, i) : null; - } - - return new MergedInvertedIndex<>(providedIndexes, contentHashIndex, baseIndex); + @NotNull UpdatableIndex baseIndex) + throws IOException { + Path file = providedExtension.getIndexPath(); + HashBasedMapReduceIndex index = HashBasedMapReduceIndex.create(providedExtension, originalExtension); + return new MergedInvertedIndex<>(index, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile()), baseIndex); } - public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex[] indexes, + public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex index, @NotNull FileContentHashIndex hashIndex, @NotNull UpdatableIndex baseIndex) { - myProvidedIndexes = indexes; + myProvidedIndex = index; myHashIndex = hashIndex; myBaseIndex = baseIndex; } - @NotNull - public FileContentHashIndex getHashIndex() { - return myHashIndex; - } - - @NotNull - public Stream> getProvidedExtensions() { - return Stream.of(myProvidedIndexes).map(index -> index.getProvidedExtension()); + public ProvidedIndexExtension getProvidedExtension() { + return myProvidedIndex.getProvidedExtension(); } @NotNull @Override public Computable update(int inputId, @Nullable FileContent content) { if (content != null) { - long hashId = FileContentHashIndexExtension.getHashId(content); - if (hashId != FileContentHashIndexExtension.NULL_HASH_ID) { - return () -> Boolean.TRUE; - } //TODO if content == null Computable update = myHashIndex.update(inputId, content); if (!((FileContentHashIndex.HashIndexUpdateComputable)update).isEmptyInput()) return update; @@ -108,28 +91,13 @@ public class MergedInvertedIndex implements UpdatableIndex getData(@NotNull Key key) throws StorageException { - List> data = new SmartList<>(); - data.add(myBaseIndex.getData(key)); - for (HashBasedMapReduceIndex index : myProvidedIndexes) { - if (index == null) continue; - data.add(index.getData(key)); - } - return new MergedValueContainer<>(data); + return MergedValueContainer.merge(myBaseIndex.getData(key), myProvidedIndex.getData(key)); } @Override public boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException { - if (!myBaseIndex.processAllKeys(processor, scope, idFilter)) { - return false; - } - for (HashBasedMapReduceIndex index : myProvidedIndexes) { - if (index == null) continue; - if (!index.processAllKeys(processor, scope, idFilter)) { - return false; - } - } - return true; + return myBaseIndex.processAllKeys(processor, scope, idFilter) && myProvidedIndex.processAllKeys(processor, scope, idFilter); } @NotNull @@ -155,9 +123,9 @@ public class MergedInvertedIndex implements UpdatableIndex getIndexedFileData(int fileId) throws StorageException { Map data = myBaseIndex.getIndexedFileData(fileId); if (!data.isEmpty()) return data; - Long hashId = myHashIndex.getHashId(fileId); - if (hashId == null || hashId == FileContentHashIndexExtension.NULL_HASH_ID) return Collections.emptyMap(); - return myProvidedIndexes[FileContentHashIndexExtension.getIndexId(hashId)].getIndexedFileData(FileContentHashIndexExtension.getInternalHashId(hashId)); + int hashId = myHashIndex.getHashId(fileId); + if (hashId == 0) return Collections.emptyMap(); + return myProvidedIndex.getIndexedFileData(hashId); } @Override diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java new file mode 100644 index 000000000000..42b2f3638b5e --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java @@ -0,0 +1,65 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.hash; + +import com.intellij.util.indexing.ValueContainer; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class MergedValueContainer extends ValueContainer { + private final ValueContainer myContainer1; + private final ValueContainer myContainer2; + + @NotNull + public static ValueContainer merge(@NotNull ValueContainer container1, @NotNull ValueContainer container2) { + if (container1.size() == 0) return container2; + if (container2.size() == 0) return container1; + return new MergedValueContainer<>(container1, container2); + } + + private MergedValueContainer(@NotNull ValueContainer container1, @NotNull ValueContainer container2) { + myContainer1 = container1; + myContainer2 = container2; + } + + @NotNull + @Override + public ValueIterator getValueIterator() { + return new ValueIterator() { + boolean mySecondIsUsed; + ValueIterator myCurrent = myContainer1.getValueIterator(); + + @NotNull + @Override + public IntIterator getInputIdsIterator() { + return myCurrent.getInputIdsIterator(); + } + + @Nullable + @Override + public IntPredicate getValueAssociationPredicate() { + return myCurrent.getValueAssociationPredicate(); + } + + @Override + public boolean hasNext() { + if (myCurrent.hasNext()) return true; + if (!mySecondIsUsed) { + myCurrent = myContainer2.getValueIterator(); + mySecondIsUsed = true; + return hasNext(); + } + return false; + } + + @Override + public Value next() { + return myCurrent.next(); + } + }; + } + + @Override + public int size() { + return myContainer1.size() + myContainer2.size(); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java index 41645f077e81..fc604a9a2ab0 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java @@ -2,19 +2,15 @@ package com.intellij.util.indexing.provided; import com.intellij.openapi.diagnostic.Logger; -import com.intellij.util.indexing.FileBasedIndexExtension; -import com.intellij.util.indexing.FileContent; -import com.intellij.util.indexing.ID; -import com.intellij.util.indexing.UpdatableIndex; -import com.intellij.util.indexing.hash.FileContentHashIndex; +import com.intellij.util.indexing.*; import com.intellij.util.indexing.hash.MergedInvertedIndex; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.KeyDescriptor; import org.jetbrains.annotations.NotNull; +import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.util.List; public interface ProvidedIndexExtension { Logger LOG = Logger.getInstance(ProvidedIndexExtension.class); @@ -32,16 +28,15 @@ public interface ProvidedIndexExtension { DataExternalizer createValueExternalizer(); @NotNull - static UpdatableIndex wrapWithProvidedIndex(@NotNull List> providedIndexExtensions, + static UpdatableIndex wrapWithProvidedIndex(@NotNull ProvidedIndexExtension providedIndexExtension, @NotNull FileBasedIndexExtension originalExtension, - @NotNull UpdatableIndex index, - @NotNull FileContentHashIndex contentHashIndex) { + @NotNull UpdatableIndex index) { try { - return MergedInvertedIndex.create(providedIndexExtensions, originalExtension, index, contentHashIndex); + return MergedInvertedIndex.create(providedIndexExtension, originalExtension, index); } catch (IOException e) { LOG.error(e); return index; } } -} \ No newline at end of file +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java index 0a1911a9f6a9..816a7d96f491 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java @@ -4,20 +4,18 @@ package com.intellij.util.indexing.provided; import com.intellij.openapi.extensions.ExtensionPointName; import com.intellij.util.indexing.FileBasedIndexExtension; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; -import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; public interface ProvidedIndexExtensionLocator { ExtensionPointName EP_NAME = ExtensionPointName.create("com.intellij.fileBasedIndex.providedLocator"); - @NotNull - Stream> findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension); + @Nullable + ProvidedIndexExtension findProvidedIndexExtension(@NotNull FileBasedIndexExtension originalExtension); - @NotNull - static List> findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension originalExtension) { - return EP_NAME.extensions().flatMap(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).collect(Collectors.toList()); + @Nullable + static ProvidedIndexExtension findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension originalExtension) { + return EP_NAME.extensions().map(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).findFirst().orElse(null); } -} \ No newline at end of file +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java index 51318f3e0874..0303e54a36c2 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java @@ -64,17 +64,6 @@ public class IndexedHashesSupport { content.setHashes(fileContentHash, documentHash != null ? documentHash : fileContentHash); } - @NotNull - public static byte[] getOrInitIndexedHash(@NotNull FileContentImpl content, boolean fromDocument) { - byte[] hash = content.getHash(fromDocument); - if (hash == null) { - initIndexedHash(content); - hash = content.getHash(fromDocument); - LOG.assertTrue(hash != null); - } - return hash; - } - @NotNull private static byte[] calculateIndexedHashForFileContent(@NotNull FileContentImpl content, boolean binary) { byte[] contentHash = null; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java index 459f42008bc7..babdf7b29479 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java @@ -280,7 +280,13 @@ public class SnapshotInputMappings implements UpdatableSnapsh Integer previouslyCalculatedContentHashId = content.getUserData(key); if (previouslyCalculatedContentHashId == null) { - byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(content, fromDocument); + byte[] hash = content.getHash(fromDocument); + if (hash == null) { + IndexedHashesSupport.initIndexedHash(content); + hash = content.getHash(fromDocument); + LOG.assertTrue(hash != null); + } + previouslyCalculatedContentHashId = IndexedHashesSupport.enumerateHash(hash); content.putUserData(key, previouslyCalculatedContentHashId); } diff --git a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java index ce242349b4c0..74ceb672c0cb 100644 --- a/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java +++ b/platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java @@ -1,10 +1,10 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package org.jetbrains.index.stubs; -import com.google.common.collect.Maps; import com.intellij.openapi.application.PathManager; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil; import com.intellij.psi.stubs.*; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.FileBasedIndexExtension; @@ -26,19 +26,19 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator myUsedKeys = new HashSet<>(); public StubHashBasedIndexGenerator(@NotNull File out) { - super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer( - true, - null, - StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), getExtension(), out); + super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(false, null), getExtension(), out, out); for (StubIndexExtension stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) { - FileBasedIndexExtension ex = StubIndexImpl - .wrapStubIndexExtension(stubIndexExtension); - myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(ex.getKeyDescriptor(), - ex.getValueExternalizer(), - ex, - new File(out, getStubsDir()) - ) { - + myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(stubIndexExtension.getKeyDescriptor(), + StubIndexImpl.StubIdExternalizer.INSTANCE, + StubIndexImpl + .wrapStubIndexExtension(stubIndexExtension), + new File(out, getStubsDir()), + out) { + @NotNull + @Override + protected ContentHashesUtil.HashEnumerator getHashEnumerator() { + return StubHashBasedIndexGenerator.this.myHashEnumerator; + } }); } } @@ -59,8 +59,7 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator value = entry.getValue(); myUsedKeys.add(key); MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex(); - Map reducedValue = Maps.asMap(value.keySet(), k -> null); - index.updateWithMap(new UpdateData(hashId, reducedValue, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null)); + index.updateWithMap(new UpdateData(hashId, value, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null)); } } @@ -84,6 +83,6 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator ex instanceof StubUpdatingIndex).findAny().get(); } } diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java index b71bdeb7e66b..c492f166b961 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java +++ b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java @@ -264,7 +264,7 @@ public abstract class MapReduceIndex implements InvertedIndex< @NotNull protected UpdateData calculateUpdateData(final int inputId, @Nullable Input content) { - final InputData data = mapInput(inputId, content); + final InputData data = mapInput(content); return createUpdateData(inputId, data.getKeyValues(), () -> getKeysDiffBuilder(inputId), @@ -302,21 +302,16 @@ public abstract class MapReduceIndex implements InvertedIndex< } @NotNull - protected InputData mapInput(int inputId, @Nullable Input content) { + protected InputData mapInput(@Nullable Input content) { if (content == null) { return InputData.empty(); } - Map data = mapByIndexer(inputId, content); + Map data = myIndexer.map(content); checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer); checkCanceled(); return new InputData<>(data); } - @NotNull - protected Map mapByIndexer(int inputId, @NotNull Input content) { - return myIndexer.map(content); - } - public abstract void checkCanceled(); protected abstract void requestRebuild(@NotNull Throwable e); diff --git a/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java b/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java index 4bbdc30277a5..b238a1577670 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java +++ b/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java @@ -12,6 +12,13 @@ public class MergedValueContainer extends ValueContainer { private final List> myContainers; private int mySize; + @NotNull + public static ValueContainer merge(@NotNull ValueContainer container1, @NotNull ValueContainer container2) { + if (container1.size() == 0) return container2; + if (container2.size() == 0) return container1; + return new MergedValueContainer<>(Arrays.asList(container1, container2)); + } + public MergedValueContainer(@NotNull List> containers) { if (containers.isEmpty()) { throw new IllegalArgumentException();