MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Revert "allow to load multiple shared indexes"

Browse Source 
This reverts commit 9b108cbe

GitOrigin-RevId: 2c1c84c91929936bb4c1f49d4ab833edeabfc552
This commit is contained in:
Dmitry Batkovich
2019-12-23 15:07:19 +00:00
committed by intellij-monorepo-bot
parent 0b7732a047
commit dc12dbeda1
23 changed files with 275 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTree.java
+9 -22
View File
@@ -45,16 +45,11 @@ public class SerializedStubTree {
private Map<StubIndexKey, Map<Object, StubIdList>> myIndexedStubs;
private volatile SerializationManagerEx mySerializationManager;
private volatile StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
public void setSerializationManager(@NotNull SerializationManagerEx serializationManager) {
public void setSerializationManager(SerializationManagerEx serializationManager) {
mySerializationManager = serializationManager;
}
public void setStubIndexesExternalizer(@NotNull StubForwardIndexExternalizer<?> stubIndexesExternalizer) {
myStubIndexesExternalizer = stubIndexesExternalizer;
}
public SerializedStubTree(@NotNull byte[] treeBytes, int treeByteLength, @Nullable Stub stubElement,
@NotNull byte[] indexedStubBytes, int indexedStubByteLength, @Nullable Map<StubIndexKey, Map<Object, StubIdList>> indexedStubs) {
myTreeBytes = treeBytes;
@@ -79,7 +74,6 @@ public class SerializedStubTree {
forwardIndexExternalizer.save(new DataOutputStream(indexBytes), myIndexedStubs);
myIndexedStubBytes = indexBytes.getInternalBuffer();
myIndexedStubByteLength = indexBytes.size();
myStubIndexesExternalizer = forwardIndexExternalizer;
}
@NotNull
@@ -100,7 +94,7 @@ public class SerializedStubTree {
else {
BufferExposingByteArrayOutputStream reSerializedStubIndices = new BufferExposingByteArrayOutputStream();
if (myIndexedStubs == null) {
restoreIndexedStubs();
restoreIndexedStubs(currentForwardIndexSerializer);
}
assert myIndexedStubs != null;
newForwardIndexSerializer.save(new DataOutputStream(reSerializedStubIndices), myIndexedStubs);
@@ -108,25 +102,18 @@ public class SerializedStubTree {
reSerializedIndexByteLength = reSerializedStubIndices.size();
}
SerializedStubTree tree = new SerializedStubTree(
outStub.getInternalBuffer(),
outStub.size(),
null,
reSerializedIndexBytes,
reSerializedIndexByteLength,
myIndexedStubs);
tree.setStubIndexesExternalizer(myStubIndexesExternalizer);
return tree;
return new SerializedStubTree(outStub.getInternalBuffer(), outStub.size(), null,
reSerializedIndexBytes, reSerializedIndexByteLength, myIndexedStubs);
}
void restoreIndexedStubs() throws IOException {
void restoreIndexedStubs(@NotNull StubForwardIndexExternalizer<?> dataExternalizer) throws IOException {
if (myIndexedStubs == null) {
myIndexedStubs = myStubIndexesExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)));
myIndexedStubs = dataExternalizer.read(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)));
}
}
<K> StubIdList restoreIndexedStubs(@NotNull StubIndexKey<K, ?> indexKey, @NotNull K key) throws IOException {
Map<StubIndexKey, Map<Object, StubIdList>> incompleteMap = myStubIndexesExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key);
<K> StubIdList restoreIndexedStubs(@NotNull StubForwardIndexExternalizer<?> dataExternalizer, @NotNull StubIndexKey<K, ?> indexKey, @NotNull K key) throws IOException {
Map<StubIndexKey, Map<Object, StubIdList>> incompleteMap = dataExternalizer.doRead(new DataInputStream(new ByteArrayInputStream(myIndexedStubBytes, 0, myIndexedStubByteLength)), indexKey, key);
Map<Object, StubIdList> map = incompleteMap.get(indexKey);
return map == null ? null : map.get(key);
}
@@ -138,7 +125,7 @@ public class SerializedStubTree {
@TestOnly
public Map<StubIndexKey, Map<Object, StubIdList>> readStubIndicesValueMap() throws IOException {
restoreIndexedStubs();
restoreIndexedStubs(IDE_USED_EXTERNALIZER);
return myIndexedStubs;
}
platform/analysis-impl/src/com/intellij/psi/stubs/SerializedStubTreeDataExternalizer.java
+6 -10
View File
@@ -15,16 +15,14 @@ import java.io.IOException;
public class SerializedStubTreeDataExternalizer implements DataExternalizer<SerializedStubTree> {
private final boolean myIncludeInputs;
private final SerializationManagerEx mySerializationManager;
private final StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
public SerializedStubTreeDataExternalizer() {
this(true, null, SerializedStubTree.IDE_USED_EXTERNALIZER);
this(true, null);
}
public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager, StubForwardIndexExternalizer<?> externalizer) {
public SerializedStubTreeDataExternalizer(boolean inputs, SerializationManagerEx manager) {
myIncludeInputs = inputs;
mySerializationManager = manager;
myStubIndexesExternalizer = externalizer;
}
@Override
@@ -46,7 +44,6 @@ public class SerializedStubTreeDataExternalizer implements DataExternalizer<Seri
@NotNull
@Override
public final SerializedStubTree read(@NotNull final DataInput in) throws IOException {
SerializedStubTree tree;
if (PersistentHashMapValueStorage.COMPRESSION_ENABLED) {
int serializedStubsLength = DataInputOutputUtil.readINT(in);
byte[] bytes = new byte[serializedStubsLength];
@@ -61,15 +58,14 @@ public class SerializedStubTreeDataExternalizer implements DataExternalizer<Seri
indexedStubByteLength = 0;
indexedStubBytes = ArrayUtil.EMPTY_BYTE_ARRAY;
}
tree = new SerializedStubTree(bytes, bytes.length, null, indexedStubBytes, indexedStubByteLength, null);
SerializedStubTree tree = new SerializedStubTree(bytes, bytes.length, null, indexedStubBytes, indexedStubByteLength, null);
if (mySerializationManager != null) tree.setSerializationManager(mySerializationManager);
return tree;
}
else {
byte[] treeBytes = CompressionUtil.readCompressed(in);
byte[] indexedStubBytes = myIncludeInputs ? CompressionUtil.readCompressed(in) : ArrayUtil.EMPTY_BYTE_ARRAY;
tree = new SerializedStubTree(treeBytes, treeBytes.length, null, indexedStubBytes, indexedStubBytes.length, null);
return new SerializedStubTree(treeBytes, treeBytes.length, null, indexedStubBytes, indexedStubBytes.length, null);
}
if (mySerializationManager != null) tree.setSerializationManager(mySerializationManager);
tree.setStubIndexesExternalizer(myStubIndexesExternalizer);
return tree;
}
}
platform/lang-impl/src/com/intellij/index/BasicProvidedExtensionLocator.java
+18 -48
View File
@@ -2,7 +2,6 @@
package com.intellij.index;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.ShutDownTracker;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.stubs.StubUpdatingIndex;
import com.intellij.psi.stubs.provided.StubProvidedIndexExtension;
@@ -10,73 +9,44 @@ import com.intellij.util.indexing.FileBasedIndexExtension;
import com.intellij.util.indexing.ID;
import com.intellij.util.indexing.provided.ProvidedIndexExtension;
import com.intellij.util.indexing.provided.ProvidedIndexExtensionLocator;
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem;
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.KeyDescriptor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.stream.Stream;
public class BasicProvidedExtensionLocator implements ProvidedIndexExtensionLocator {
private static final String PREBUILT_INDEX_ZIP_PROP = "prebuilt.hash.index.zip";
private static final String PREBUILT_INDEX_PATH_PROP = "prebuilt.hash.index.dir";
private static final Logger LOG = Logger.getInstance(BasicProvidedExtensionLocator.class);
private static UncompressedZipFileSystem ourFs;
@NotNull
@Nullable
@Override
public <K, V> Stream<ProvidedIndexExtension<K, V>> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
if (!originalExtension.dependsOnFileContent()) return Stream.empty();
public <K, V> ProvidedIndexExtension<K, V> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
Path root = getPrebuiltIndexPath();
if (root == null || !Files.exists(root)) return Stream.empty();
if (root == null || !Files.exists(root.resolve( StringUtil.toLowerCase(originalExtension.getName().getName())))) return null;
try {
// TODO properly close it
UncompressedZipFileSystem fs = getFs(root);
Path fsRoot = fs.getPath("/").getRoot();
return Files
.list(fsRoot)
.sorted(Comparator.comparing(p -> p.getFileName().toString()))
.map(p -> p.resolve(StringUtil.toLowerCase(originalExtension.getName().getName())))
.map(p -> {
return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID)
? (ProvidedIndexExtension<K, V>)new StubProvidedIndexExtension(p)
: new ProvidedIndexExtensionImpl<>(p, originalExtension);
});
} catch (IOException e) {
LOG.error(e);
return Stream.empty();
}
}
private synchronized static UncompressedZipFileSystem getFs(@NotNull Path root) throws IOException {
if (ourFs == null) {
ourFs = new UncompressedZipFileSystem(root, new UncompressedZipFileSystemProvider());
ShutDownTracker.getInstance().registerShutdownTask(() -> {
try {
ourFs.close();
}
catch (IOException e) {
LOG.error(e);
}
});
}
return ourFs;
return originalExtension.getName().equals(StubUpdatingIndex.INDEX_ID)
? (ProvidedIndexExtension<K, V>)new StubProvidedIndexExtension(root)
: new ProvidedIndexExtensionImpl<>(root, originalExtension);
}
@Nullable
private static Path getPrebuiltIndexPath() {
String path = System.getProperty(PREBUILT_INDEX_ZIP_PROP);
String path = System.getProperty(PREBUILT_INDEX_PATH_PROP);
if (path == null) return null;
Path file = Paths.get(path);
Path file;
try {
file = Paths.get(new URI(path));
}
catch (URISyntaxException e) {
LOG.error(e);
return null;
}
return Files.exists(file) ? file : null;
}
platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java
+1 -1
View File
@@ -37,7 +37,7 @@ public final class StubProcessingHelper extends StubProcessingHelperBase {
return null;
}
SerializedStubTree tree = data.values().iterator().next();
StubIdList stubIdList = tree.restoreIndexedStubs(indexKey, key);
StubIdList stubIdList = tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER, indexKey, key);
if (stubIdList == null) {
LOG.error("Stub ids not found for key in index = " + indexKey.getName() + ", file type = " + file.getFileType());
onInternalError(file);
platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingForwardIndexAccessor.java
+1 -1
View File
@@ -37,7 +37,7 @@ class StubUpdatingForwardIndexAccessor implements ForwardIndexAccessor<Integer,
Map<Integer, SerializedStubTree> data = dataRef.get();
SerializedStubTree tree = ContainerUtil.isEmpty(data) ? null : ContainerUtil.getFirstItem(data.values());
if (tree != null) {
tree.restoreIndexedStubs();
tree.restoreIndexedStubs(SerializedStubTree.IDE_USED_EXTERNALIZER);
}
return new StubCumulativeInputDiffBuilder(inputId, tree);
}
platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java
+1 -10
View File
@@ -47,15 +47,6 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
public static final ID<Integer, SerializedStubTree> INDEX_ID = ID.create("Stubs");
private static final FileBasedIndex.InputFilter INPUT_FILTER = file -> canHaveStub(file);
private final StubForwardIndexExternalizer<?> myStubIndexesExternalizer;
public StubUpdatingIndex() {
myStubIndexesExternalizer = SerializedStubTree.IDE_USED_EXTERNALIZER;
}
public StubUpdatingIndex(StubForwardIndexExternalizer<?> stubIndexesExternalizer) {
myStubIndexesExternalizer = stubIndexesExternalizer;
}
public static boolean canHaveStub(@NotNull VirtualFile file) {
Project project = ProjectUtil.guessProjectForFile(file);
@@ -247,7 +238,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
@NotNull
@Override
public DataExternalizer<SerializedStubTree> getValueExternalizer() {
return new SerializedStubTreeDataExternalizer(true, null, myStubIndexesExternalizer);
return new SerializedStubTreeDataExternalizer();
}
@NotNull
platform/lang-impl/src/com/intellij/psi/stubs/provided/StubProvidedIndexExtension.java
+4 -1
View File
@@ -5,6 +5,8 @@ import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.util.Disposer;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.stubs.*;
import com.intellij.util.indexing.FileBasedIndex;
import com.intellij.util.indexing.FileBasedIndexImpl;
import com.intellij.util.indexing.ID;
import com.intellij.util.indexing.provided.ProvidedIndexExtension;
import com.intellij.util.io.DataExternalizer;
@@ -14,6 +16,7 @@ import com.intellij.util.io.VoidDataExternalizer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -49,7 +52,7 @@ public class StubProvidedIndexExtension implements ProvidedIndexExtension<Intege
new SerializationManagerImpl(path.resolve(StringUtil.toLowerCase(StubUpdatingIndex.INDEX_ID.getName())).resolve("rep.names"),
true);
Disposer.register(ApplicationManager.getApplication(), manager);
return new SerializedStubTreeDataExternalizer(false, manager, StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE);
return new SerializedStubTreeDataExternalizer(false, manager);
}
@Nullable
platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
+7 -14
View File
@@ -93,7 +93,6 @@ import org.jetbrains.annotations.TestOnly;
import java.io.*;
import java.lang.ref.SoftReference;
import java.lang.ref.WeakReference;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -450,14 +449,9 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
UpdatableIndex<K, V, FileContent> index = createIndex(extension, new MemoryIndexStorage<>(storage, name));
if (!(extension instanceof FileContentHashIndexExtension)) {
List<ProvidedIndexExtension<K, V>> providedExtensions = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension);
if (!providedExtensions.isEmpty()) {
Path[] paths = ContainerUtil.map2Array(providedExtensions, Path.class, ex -> ex.getIndexPath());
FileContentHashIndex contentHashIndex = ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(paths, state);
index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtensions, extension, index, contentHashIndex);
}
ProvidedIndexExtension<K, V> providedExtension = ProvidedIndexExtensionLocator.findProvidedIndexExtensionFor(extension);
if (providedExtension != null) {
index = ProvidedIndexExtension.wrapWithProvidedIndex(providedExtension, extension, index);
}
state.registerIndex(name,
@@ -2683,20 +2677,19 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
}
}
public synchronized FileContentHashIndex getFileContentHashIndex(@Nullable Path[] enumeratorPaths, @NotNull IndexConfiguration state) {
UpdatableIndex<Long, Void, FileContent> index = state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
public synchronized FileContentHashIndex getFileContentHashIndex(@NotNull File enumeratorPath) {
UpdatableIndex<Integer, Void, FileContent> index = getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
if (index == null) {
LOG.assertTrue(enumeratorPaths != null);
IndicesRegistrationResult registrationResult = new IndicesRegistrationResult();
try {
registerIndexer(FileContentHashIndexExtension.create(enumeratorPaths, ApplicationManager.getApplication()), state, registrationResult);
registerIndexer(FileContentHashIndexExtension.create(enumeratorPath, ApplicationManager.getApplication()), myRegisteredIndexes.getState(), registrationResult);
registrationResult.logChangedAndFullyBuiltIndices(LOG, "Version was changed for:", "Index is to be rebuilt:");
}
catch (IOException e) {
throw new RuntimeException(e);
}
} else return (FileContentHashIndex)index;
return (FileContentHashIndex)state.getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
return (FileContentHashIndex)getState().getIndex(FileContentHashIndexExtension.HASH_INDEX_ID);
}
private static final boolean INDICES_ARE_PSI_DEPENDENT_BY_DEFAULT = SystemProperties.getBooleanProperty("idea.indices.psi.dependent.default", true);
platform/lang-impl/src/com/intellij/util/indexing/RebuildStatus.java
+2 -2
View File
@@ -12,14 +12,14 @@ import java.util.concurrent.atomic.AtomicReference;
/**
* @author peter
*/
public enum RebuildStatus {
enum RebuildStatus {
OK,
REQUIRES_REBUILD,
DOING_REBUILD;
private static final Map<ID<?, ?>, AtomicReference<RebuildStatus>> ourRebuildStatus = new THashMap<>();
public static void registerIndex(ID<?, ?> indexId) {
static void registerIndex(ID<?, ?> indexId) {
ourRebuildStatus.put(indexId, new AtomicReference<>(OK));
}
platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java
+2 -2
View File
@@ -119,7 +119,7 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
@NotNull
@Override
protected InputData<Key, Value> mapInput(int inputId, @Nullable Input content) {
protected InputData<Key, Value> mapInput(@Nullable Input content) {
InputData<Key, Value> data;
boolean containsSnapshotData = true;
if (mySnapshotInputMappings != null && content != null) {
@@ -135,7 +135,7 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
throw new RuntimeException(e);
}
}
data = super.mapInput(inputId, content);
data = super.mapInput(content);
if (!containsSnapshotData && !UpdatableSnapshotInputMappingIndex.ignoreMappingIndexUpdate(content)) {
try {
return ((UpdatableSnapshotInputMappingIndex<Key, Value, Input>)mySnapshotInputMappings).putData(content, data);
platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndex.java
+9 -9
View File
@@ -13,8 +13,8 @@ import org.jetbrains.annotations.NotNull;
import java.io.IOException;
import java.util.Map;
public class FileContentHashIndex extends VfsAwareMapReduceIndex<Long, Void, FileContent> {
FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage<Long, Void> storage) throws IOException {
public class FileContentHashIndex extends VfsAwareMapReduceIndex<Integer, Void, FileContent> {
FileContentHashIndex(@NotNull FileContentHashIndexExtension extension, IndexStorage<Integer, Void> storage) throws IOException {
super(extension,
storage,
new PersistentMapBasedForwardIndex(IndexInfrastructure.getInputIndexStorageFile(extension.getName()).toPath(), false),
@@ -23,22 +23,22 @@ public class FileContentHashIndex extends VfsAwareMapReduceIndex<Long, Void, Fil
@NotNull
@Override
protected Computable<Boolean> createIndexUpdateComputation(@NotNull AbstractUpdateData<Long, Void> updateData) {
protected Computable<Boolean> createIndexUpdateComputation(@NotNull AbstractUpdateData<Integer, Void> updateData) {
return new HashIndexUpdateComputable(super.createIndexUpdateComputation(updateData), updateData.newDataIsEmpty());
}
public Long getHashId(int fileId) throws StorageException {
Map<Long, Void> data = getIndexedFileData(fileId);
if (data.isEmpty()) return FileContentHashIndexExtension.NULL_HASH_ID;
public int getHashId(int fileId) throws StorageException {
Map<Integer, Void> data = getIndexedFileData(fileId);
if (data.isEmpty()) return 0;
return data.keySet().iterator().next();
}
@NotNull
IntIntFunction toHashIdToFileIdFunction(int indexId) {
IntIntFunction toHashIdToFileIdFunction() {
return hash -> {
try {
ValueContainer<Void> data = getData(FileContentHashIndexExtension.getHashId(hash, indexId));
if (data.size() == 0) return -1;
ValueContainer<Void> data = getData(hash);
assert data.size() == 1;
return data.getValueIterator().getInputIdsIterator().next();
}
catch (StorageException e) {
platform/lang-impl/src/com/intellij/util/indexing/hash/FileContentHashIndexExtension.java
+39 -116
View File
@@ -4,68 +4,51 @@ package com.intellij.util.indexing.hash;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Disposer;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.ShutDownTracker;
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.*;
import com.intellij.util.indexing.impl.IndexStorage;
import com.intellij.util.indexing.snapshot.IndexedHashesSupport;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.DataInputOutputUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.VoidDataExternalizer;
import com.intellij.util.io.*;
import org.jetbrains.annotations.NotNull;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
public class FileContentHashIndexExtension extends FileBasedIndexExtension<Long, Void> implements CustomImplementationFileBasedIndexExtension<Long, Void>, CustomInputsIndexFileBasedIndexExtension<Long>, Disposable {
public class FileContentHashIndexExtension extends FileBasedIndexExtension<Integer, Void> implements CustomImplementationFileBasedIndexExtension<Integer, Void>, CustomInputsIndexFileBasedIndexExtension<Integer>, Disposable {
private static final Logger LOG = Logger.getInstance(FileContentHashIndexExtension.class);
public static final ID<Long, Void> HASH_INDEX_ID = ID.create("file.content.hash.index");
public static final ID<Integer, Void> HASH_INDEX_ID = ID.create("file.content.hash.index");
@NotNull
private final ContentHashesUtil.HashEnumerator[] myEnumerators;
private final ContentHashesUtil.HashEnumerator myEnumerator;
private final int myDirHash;
@NotNull
public static FileContentHashIndexExtension create(@NotNull Path[] enumeratorDirs, @NotNull Disposable parent) throws IOException {
FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDirs);
RebuildStatus.registerIndex(extension.getName());
public static FileContentHashIndexExtension create(@NotNull File enumeratorDir, @NotNull Disposable parent) throws IOException {
FileContentHashIndexExtension extension = new FileContentHashIndexExtension(enumeratorDir);
Disposer.register(parent, extension);
return extension;
}
private FileContentHashIndexExtension(@NotNull Path[] enumeratorDirs) throws IOException {
IOException[] exception = {null};
myEnumerators = ContainerUtil.map2Array(enumeratorDirs, ContentHashesUtil.HashEnumerator.class, d -> {
try {
return new ContentHashesUtil.HashEnumerator(d.getParent().resolve("hashes"));
}
catch (IOException e) {
exception[0] = e;
return null;
}
});
if (exception[0] != null) {
throw exception[0];
}
private FileContentHashIndexExtension(@NotNull File enumeratorDir) throws IOException {
myEnumerator = new ContentHashesUtil.HashEnumerator(enumeratorDir.toPath());
myDirHash = enumeratorDir.getAbsolutePath().hashCode();
ShutDownTracker.getInstance().registerShutdownTask(() -> closeEnumerator());
}
@NotNull
@Override
public ID<Long, Void> getName() {
public ID<Integer, Void> getName() {
return HASH_INDEX_ID;
}
@NotNull
@Override
public FileBasedIndex.InputFilter getInputFilter() {
return file -> !file.isDirectory();
throw new UnsupportedOperationException();
}
@Override
@@ -75,15 +58,16 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Long,
@NotNull
@Override
public DataIndexer<Long, Void, FileContent> getIndexer() {
public DataIndexer<Integer, Void, FileContent> getIndexer() {
return fc -> {
long hashId = getHashId(fc);
if (hashId != NULL_HASH_ID) return Collections.singletonMap(hashId, null);
byte[] hash = IndexedHashesSupport.getOrInitIndexedHash((FileContentImpl) fc, false);
byte[] hash = ((FileContentImpl)fc).getHash(false);
LOG.assertTrue(hash != null);
try {
hashId = tryEnumerate(hash);
setHashId(fc, hashId);
return hashId == NULL_HASH_ID ? Collections.emptyMap() : Collections.singletonMap(hashId, null);
int id;
synchronized (myEnumerator) {
id = myEnumerator.tryEnumerate(hash);
}
return id == 0 ? Collections.emptyMap() : Collections.singletonMap(id, null);
}
catch (IOException e) {
throw new RuntimeException(e);
@@ -91,44 +75,10 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Long,
};
}
private Long tryEnumerate(byte[] hash) throws IOException {
for (int i = 0; i < myEnumerators.length; i++) {
ContentHashesUtil.HashEnumerator enumerator = myEnumerators[i];
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (enumerator) {
int id = Math.abs(enumerator.tryEnumerate(hash));
if (id != 0) {
return getHashId(id, i);
}
}
}
return NULL_HASH_ID;
}
@NotNull
@Override
public KeyDescriptor<Long> getKeyDescriptor() {
return new KeyDescriptor<Long>() {
@Override
public int getHashCode(Long value) {
return value.hashCode();
}
@Override
public boolean isEqual(Long val1, Long val2) {
return val1.longValue() == val2.longValue();
}
@Override
public void save(@NotNull DataOutput out, Long value) throws IOException {
DataInputOutputUtil.writeLONG(out, value);
}
@Override
public Long read(@NotNull DataInput in) throws IOException {
return DataInputOutputUtil.readLONG(in);
}
};
public KeyDescriptor<Integer> getKeyDescriptor() {
return EnumeratorIntegerDescriptor.INSTANCE;
}
@NotNull
@@ -139,7 +89,7 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Long,
@Override
public int getVersion() {
return 0;
return myDirHash;
}
@Override
@@ -149,66 +99,39 @@ public class FileContentHashIndexExtension extends FileBasedIndexExtension<Long,
@NotNull
@Override
public DataExternalizer<Collection<Long>> createExternalizer() {
return new DataExternalizer<Collection<Long>>() {
public DataExternalizer<Collection<Integer>> createExternalizer() {
return new DataExternalizer<Collection<Integer>>() {
@Override
public void save(@NotNull DataOutput out, Collection<Long> value) throws IOException {
public void save(@NotNull DataOutput out, Collection<Integer> value) throws IOException {
assert value.isEmpty() || value.size() == 1;
DataInputOutputUtil.writeLONG(out, value.isEmpty() ? 0 : value.iterator().next());
DataInputOutputUtil.writeINT(out, value.isEmpty() ? 0 : value.iterator().next());
}
@Override
public Collection<Long> read(@NotNull DataInput in) throws IOException {
long id = DataInputOutputUtil.readLONG(in);
public Collection<Integer> read(@NotNull DataInput in) throws IOException {
int id = DataInputOutputUtil.readINT(in);
return id == 0 ? Collections.emptyList() : Collections.singleton(id);
}
};
}
private void closeEnumerator() {
for (ContentHashesUtil.HashEnumerator enumerator : myEnumerators) {
synchronized (enumerator) {
if (enumerator.isClosed()) return;
try {
enumerator.close();
}
catch (IOException e) {
LOG.error(e);
}
synchronized (myEnumerator) {
if (myEnumerator.isClosed()) return;
try {
myEnumerator.close();
}
catch (IOException e) {
LOG.error(e);
}
}
}
@NotNull
@Override
public UpdatableIndex<Long, Void, FileContent> createIndexImplementation(@NotNull FileBasedIndexExtension<Long, Void> extension,
@NotNull IndexStorage<Long, Void> storage)
public UpdatableIndex<Integer, Void, FileContent> createIndexImplementation(@NotNull FileBasedIndexExtension<Integer, Void> extension,
@NotNull IndexStorage<Integer, Void> storage)
throws IOException {
return new FileContentHashIndex(((FileContentHashIndexExtension)extension), storage);
}
static int getIndexId(long hashId) {
return (int)(hashId >> 32);
}
static int getInternalHashId(long hashId) {
return (int)hashId;
}
static long getHashId(int internalHashId, int indexId) {
return (((long) indexId) << 32) | (internalHashId & 0xffffffffL);
}
public static final long NULL_HASH_ID = getHashId(0, -1);
public static final Key<Long> HASH_ID_KEY = Key.create("file.content.hash.id");
public static long getHashId(@NotNull FileContent content) {
Long value = HASH_ID_KEY.get(content);
return value == null ? NULL_HASH_ID : value;
}
public static void setHashId(@NotNull FileContent content, long hashId) {
HASH_ID_KEY.set(content, hashId);
}
}
platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedIndexGenerator.java
+29 -62
View File
@@ -2,13 +2,11 @@
package com.intellij.util.indexing.hash;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.VfsUtilCore;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileVisitor;
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
import com.intellij.psi.SingleRootFileViewProvider;
import com.intellij.util.indexing.*;
import com.intellij.util.indexing.impl.InputData;
import com.intellij.util.indexing.impl.MapIndexStorage;
@@ -20,33 +18,35 @@ import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.LongAdder;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
public class HashBasedIndexGenerator<K, V> {
@NotNull
private final File myOut;
@NotNull
private final FakeIndexExtension<K, V> myExtension;
@NotNull
private final FileBasedIndex.InputFilter myInputFilter;
@NotNull private final File myOut;
@NotNull private final File myHashOut;
@NotNull private final FakeIndexExtension<K, V> myExtension;
@NotNull private final FileBasedIndex.InputFilter myInputFilter;
protected ContentHashesUtil.HashEnumerator myHashEnumerator;
private InvertedIndex<K, V, FileContent> myIndex;
public HashBasedIndexGenerator(@NotNull FileBasedIndexExtension<K, V> indexExtension, @NotNull File out) {
this(indexExtension.getKeyDescriptor(),
indexExtension.getValueExternalizer(),
indexExtension,
out
);
out,
out);
}
public HashBasedIndexGenerator(@NotNull KeyDescriptor<K> keyDescriptor,
@NotNull DataExternalizer<V> valueExternalizer,
@NotNull FileBasedIndexExtension<K, V> originalExtension,
@NotNull File out) {
@NotNull File out,
@NotNull File hashOut) {
myExtension = new FakeIndexExtension<>(keyDescriptor, valueExternalizer, originalExtension);
myOut = out;
myHashOut = hashOut;
FileBasedIndex.InputFilter filter = originalExtension.getInputFilter();
@@ -65,8 +65,8 @@ public class HashBasedIndexGenerator<K, V> {
}
public void openIndex() throws IOException {
myHashEnumerator = getHashEnumerator();
String indexName = myExtension.getName().getName();
boolean singleEntry = myExtension.myOriginalExtension instanceof SingleEntryFileBasedIndexExtension;
myIndex = new MapReduceIndex<K, V, FileContent>(myExtension, new MapIndexStorage<K, V>(new File(new File(myOut, StringUtil.toLowerCase(indexName)), indexName).toPath(),
myExtension.getKeyDescriptor(),
myExtension.getValueExternalizer(),
@@ -77,16 +77,6 @@ public class HashBasedIndexGenerator<K, V> {
//ignore
}
}, null, null) {
@NotNull
@Override
protected Map<K, V> mapByIndexer(int inputId, @NotNull FileContent content) {
Map<K, V> data = super.mapByIndexer(inputId, content);
if (singleEntry && !data.isEmpty()) {
data = Collections.singletonMap((K)(Integer)inputId, data.values().iterator().next());
}
return data;
}
@Override
protected void updateForwardIndex(int inputId, @NotNull InputData<K, V> data) throws IOException {
super.updateForwardIndex(inputId, data);
@@ -105,83 +95,60 @@ public class HashBasedIndexGenerator<K, V> {
@Override
protected void requestRebuild(@NotNull Throwable e) {
throw new RuntimeException("error while processing " + indexName, e);
throw new RuntimeException(e);
}
};
}
@NotNull
protected ContentHashesUtil.HashEnumerator getHashEnumerator() throws IOException {
return new ContentHashesUtil.HashEnumerator(new File(myHashOut, "hashes").toPath());
}
protected void visitInputData(int hashId, @NotNull InputData<K, V> data) throws StorageException {
}
public void closeIndex() throws IOException {
if (myIndex != null) myIndex.dispose();
if (myHashEnumerator != null) myHashEnumerator.close();
}
public static void generate(@NotNull Collection<VirtualFile> roots,
@NotNull Collection<HashBasedIndexGenerator<?, ?>> generators,
@NotNull Project project,
@NotNull File hashOut) {
LongAdder l = new LongAdder();
public final void generate(@NotNull Collection<VirtualFile> roots) {
try {
ContentHashesUtil.HashEnumerator hashEnumerator = new ContentHashesUtil.HashEnumerator(new File(hashOut, "hashes").toPath());
for (HashBasedIndexGenerator<?, ?> generator : generators) {
generator.openIndex();
}
openIndex();
for (VirtualFile root : roots) {
VfsUtilCore.visitChildrenRecursively(root, new VirtualFileVisitor<Boolean>() {
@Override
public boolean visitFile(@NotNull VirtualFile file) {
if (!file.isDirectory() && !SingleRootFileViewProvider.isTooLargeForIntelligence(file)) {
for (HashBasedIndexGenerator<?, ?> generator : generators) {
if (generator.myInputFilter.acceptInput(file)) {
l.increment();
generator.indexFile(file, project, hashEnumerator);
}
}
if (!file.isDirectory() && myInputFilter.acceptInput(file)) {
indexFile(file);
}
return true;
}
});
}
synchronized (hashEnumerator) {
hashEnumerator.close();
}
}
catch (IOException e) {
throw new RuntimeException(e);
}
finally {
try {
for (HashBasedIndexGenerator<?, ?> generator : generators) {
generator.closeIndex();
}
closeIndex();
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
System.out.println("Indexed " + l.sum() + " files to " + hashOut.getPath());
}
protected void indexFile(@NotNull VirtualFile f,
@NotNull Project project,
@NotNull ContentHashesUtil.HashEnumerator hashEnumerator) {
protected void indexFile(@NotNull VirtualFile f) {
try {
FileContentImpl fc = new FileContentImpl(f, f.contentsToByteArray());
byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(fc, false);
int hashId;
synchronized (hashEnumerator) {
hashId = Math.abs(hashEnumerator.enumerate(hash));
}
fc.putUserData(IndexingDataKeys.PROJECT, project);
IndexedHashesSupport.initIndexedHash(fc);
int hashId = myHashEnumerator.enumerate(fc.getHash(false));
if (!myIndex.update(hashId, fc).compute()) {
throw new RuntimeException();
}
platform/lang-impl/src/com/intellij/util/indexing/hash/HashBasedMapReduceIndex.java
+18 -58
View File
@@ -2,19 +2,14 @@
package com.intellij.util.indexing.hash;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.IntIntFunction;
import com.intellij.util.Processor;
import com.intellij.util.indexing.*;
import com.intellij.util.indexing.impl.IndexStorage;
import com.intellij.util.indexing.impl.MapIndexStorage;
import com.intellij.util.indexing.impl.UpdatableValueContainer;
import com.intellij.util.indexing.provided.ProvidedIndexExtension;
import com.intellij.util.io.PersistentEnumeratorBase;
import com.intellij.util.io.PersistentHashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
@@ -24,20 +19,17 @@ class HashBasedMapReduceIndex<Key, Value> extends VfsAwareMapReduceIndex<Key, Va
@NotNull
static <Key, Value> HashBasedMapReduceIndex<Key, Value> create(@NotNull ProvidedIndexExtension<Key, Value> providedExtension,
@NotNull FileBasedIndexExtension<Key, Value> originalExtension,
@NotNull FileContentHashIndex hashIndex,
int providedIndexId)
@NotNull FileBasedIndexExtension<Key, Value> originalExtension)
throws IOException {
Path file = providedExtension.getIndexPath();
return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, hashIndex, providedIndexId);
return new HashBasedMapReduceIndex<>(file, originalExtension, providedExtension, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile()));
}
private HashBasedMapReduceIndex(@NotNull Path baseFile,
@NotNull FileBasedIndexExtension<Key, Value> originalExtension,
@NotNull ProvidedIndexExtension<Key, Value> providedExtension,
@NotNull FileContentHashIndex hashIndex,
int providedIndexId) throws IOException {
super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction(providedIndexId)), null, null, null, null);
@NotNull FileContentHashIndex hashIndex) throws IOException {
super(originalExtension, createStorage(baseFile, originalExtension, providedExtension, hashIndex.toHashIdToFileIdFunction()), null, null, null, null);
myProvidedExtension = providedExtension;
}
@@ -50,51 +42,19 @@ class HashBasedMapReduceIndex<Key, Value> extends VfsAwareMapReduceIndex<Key, Va
@NotNull FileBasedIndexExtension<Key, Value> originalExtension,
@NotNull ProvidedIndexExtension<Key, Value> providedExtension,
@NotNull IntIntFunction hashToFileId) throws IOException {
return new MyMapIndexStorage<>(baseFile, originalExtension, providedExtension, hashToFileId);
return new MapIndexStorage<Key, Value>(baseFile.resolve(originalExtension.getName().getName()),
providedExtension.createKeyDescriptor(),
providedExtension.createValueExternalizer(),
originalExtension.getCacheSize(),
originalExtension.keyIsUniqueForIndexedFile(),
true,
true,
hashToFileId) {
@Override
protected void checkCanceled() {
ProgressManager.checkCanceled();
}
};
}
private static class MyMapIndexStorage<Key, Value>
extends MapIndexStorage<Key, Value>
implements VfsAwareIndexStorage<Key, Value> {
public MyMapIndexStorage(Path baseFile,
FileBasedIndexExtension<Key, Value> originalExtension,
ProvidedIndexExtension<Key, Value> providedExtension,
IntIntFunction hashToFileId) throws IOException {
super(baseFile.resolve(originalExtension.getName().getName()), providedExtension.createKeyDescriptor(),
providedExtension.createValueExternalizer(), originalExtension.getCacheSize(), originalExtension.keyIsUniqueForIndexedFile(),
true, true, hashToFileId);
}
@Override
protected void checkCanceled() {
ProgressManager.checkCanceled();
}
@Override
public boolean processKeys(@NotNull Processor<? super Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter)
throws StorageException {
l.lock();
try {
myCache.clear(); // this will ensure that all new keys are made into the map
return doProcessKeys(processor);
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
return unwrapCauseAndRethrow(e);
}
finally {
l.unlock();
}
}
private boolean doProcessKeys(@NotNull Processor<? super Key> processor) throws IOException {
return myMap instanceof PersistentHashMap && PersistentEnumeratorBase.inlineKeyStorage(myKeyDescriptor)
// process keys and check that they're already present in map because we don't have separated key storage we must check keys
? ((PersistentHashMap<Key, UpdatableValueContainer<Value>>)myMap).processKeysWithExistingMapping(processor)
// optimization: process all keys, some of them might be already deleted but we don't care. We just read key storage file here
: myMap.processKeys(processor);
}
}
}
platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java
+19 -51
View File
@@ -4,74 +4,57 @@ package com.intellij.util.indexing.hash;
import com.intellij.openapi.util.Computable;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import com.intellij.util.SmartList;
import com.intellij.util.indexing.*;
import com.intellij.util.indexing.impl.AbstractUpdateData;
import com.intellij.util.indexing.impl.MergedValueContainer;
import com.intellij.util.indexing.provided.ProvidedIndexExtension;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.stream.Stream;
public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Value, FileContent> {
@NotNull
private final HashBasedMapReduceIndex<Key, Value>[] myProvidedIndexes;
private final HashBasedMapReduceIndex<Key, Value> myProvidedIndex;
@NotNull
private final FileContentHashIndex myHashIndex;
@NotNull
public final UpdatableIndex<Key, Value, FileContent> myBaseIndex;
private final UpdatableIndex<Key, Value, FileContent> myBaseIndex;
@NotNull
public static <Key, Value> MergedInvertedIndex<Key, Value> create(@NotNull List<ProvidedIndexExtension<Key, Value>> providedExtensions,
public static <Key, Value> MergedInvertedIndex<Key, Value> create(@NotNull ProvidedIndexExtension<Key, Value> providedExtension,
@NotNull FileBasedIndexExtension<Key, Value> originalExtension,
@NotNull UpdatableIndex<Key, Value, FileContent> baseIndex,
@NotNull FileContentHashIndex contentHashIndex) throws IOException {
HashBasedMapReduceIndex<Key, Value>[] providedIndexes = new HashBasedMapReduceIndex[providedExtensions.size()];
for (int i = 0; i < providedExtensions.size(); i++) {
ProvidedIndexExtension<Key, Value> extension = providedExtensions.get(i);
providedIndexes[i] = extension != null && Files.exists(extension.getIndexPath()) ? HashBasedMapReduceIndex.create(extension, originalExtension, contentHashIndex, i) : null;
}
return new MergedInvertedIndex<>(providedIndexes, contentHashIndex, baseIndex);
@NotNull UpdatableIndex<Key, Value, FileContent> baseIndex)
throws IOException {
Path file = providedExtension.getIndexPath();
HashBasedMapReduceIndex<Key, Value> index = HashBasedMapReduceIndex.create(providedExtension, originalExtension);
return new MergedInvertedIndex<>(index, ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getFileContentHashIndex(file.toFile()), baseIndex);
}
public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex<Key, Value>[] indexes,
public MergedInvertedIndex(@NotNull HashBasedMapReduceIndex<Key, Value> index,
@NotNull FileContentHashIndex hashIndex,
@NotNull UpdatableIndex<Key, Value, FileContent> baseIndex) {
myProvidedIndexes = indexes;
myProvidedIndex = index;
myHashIndex = hashIndex;
myBaseIndex = baseIndex;
}
@NotNull
public FileContentHashIndex getHashIndex() {
return myHashIndex;
}
@NotNull
public Stream<ProvidedIndexExtension<Key, Value>> getProvidedExtensions() {
return Stream.of(myProvidedIndexes).map(index -> index.getProvidedExtension());
public ProvidedIndexExtension<Key, Value> getProvidedExtension() {
return myProvidedIndex.getProvidedExtension();
}
@NotNull
@Override
public Computable<Boolean> update(int inputId, @Nullable FileContent content) {
if (content != null) {
long hashId = FileContentHashIndexExtension.getHashId(content);
if (hashId != FileContentHashIndexExtension.NULL_HASH_ID) {
return () -> Boolean.TRUE;
}
//TODO if content == null
Computable<Boolean> update = myHashIndex.update(inputId, content);
if (!((FileContentHashIndex.HashIndexUpdateComputable)update).isEmptyInput()) return update;
@@ -108,28 +91,13 @@ public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Valu
@NotNull
@Override
public ValueContainer<Value> getData(@NotNull Key key) throws StorageException {
List<ValueContainer<Value>> data = new SmartList<>();
data.add(myBaseIndex.getData(key));
for (HashBasedMapReduceIndex<Key, Value> index : myProvidedIndexes) {
if (index == null) continue;
data.add(index.getData(key));
}
return new MergedValueContainer<>(data);
return MergedValueContainer.merge(myBaseIndex.getData(key), myProvidedIndex.getData(key));
}
@Override
public boolean processAllKeys(@NotNull Processor<? super Key> processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter)
throws StorageException {
if (!myBaseIndex.processAllKeys(processor, scope, idFilter)) {
return false;
}
for (HashBasedMapReduceIndex<Key, Value> index : myProvidedIndexes) {
if (index == null) continue;
if (!index.processAllKeys(processor, scope, idFilter)) {
return false;
}
}
return true;
return myBaseIndex.processAllKeys(processor, scope, idFilter) && myProvidedIndex.processAllKeys(processor, scope, idFilter);
}
@NotNull
@@ -155,9 +123,9 @@ public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Valu
public Map<Key, Value> getIndexedFileData(int fileId) throws StorageException {
Map<Key, Value> data = myBaseIndex.getIndexedFileData(fileId);
if (!data.isEmpty()) return data;
Long hashId = myHashIndex.getHashId(fileId);
if (hashId == null || hashId == FileContentHashIndexExtension.NULL_HASH_ID) return Collections.emptyMap();
return myProvidedIndexes[FileContentHashIndexExtension.getIndexId(hashId)].getIndexedFileData(FileContentHashIndexExtension.getInternalHashId(hashId));
int hashId = myHashIndex.getHashId(fileId);
if (hashId == 0) return Collections.emptyMap();
return myProvidedIndex.getIndexedFileData(hashId);
}
@Override
platform/lang-impl/src/com/intellij/util/indexing/hash/MergedValueContainer.java
+65
View File
@@ -0,0 +1,65 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util.indexing.hash;
import com.intellij.util.indexing.ValueContainer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public class MergedValueContainer<Value> extends ValueContainer<Value> {
private final ValueContainer<Value> myContainer1;
private final ValueContainer<Value> myContainer2;
@NotNull
public static <Value> ValueContainer<Value> merge(@NotNull ValueContainer<Value> container1, @NotNull ValueContainer<Value> container2) {
if (container1.size() == 0) return container2;
if (container2.size() == 0) return container1;
return new MergedValueContainer<>(container1, container2);
}
private MergedValueContainer(@NotNull ValueContainer<Value> container1, @NotNull ValueContainer<Value> container2) {
myContainer1 = container1;
myContainer2 = container2;
}
@NotNull
@Override
public ValueIterator<Value> getValueIterator() {
return new ValueIterator<Value>() {
boolean mySecondIsUsed;
ValueIterator<Value> myCurrent = myContainer1.getValueIterator();
@NotNull
@Override
public IntIterator getInputIdsIterator() {
return myCurrent.getInputIdsIterator();
}
@Nullable
@Override
public IntPredicate getValueAssociationPredicate() {
return myCurrent.getValueAssociationPredicate();
}
@Override
public boolean hasNext() {
if (myCurrent.hasNext()) return true;
if (!mySecondIsUsed) {
myCurrent = myContainer2.getValueIterator();
mySecondIsUsed = true;
return hasNext();
}
return false;
}
@Override
public Value next() {
return myCurrent.next();
}
};
}
@Override
public int size() {
return myContainer1.size() + myContainer2.size();
}
}
platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtension.java
+6 -11
View File
@@ -2,19 +2,15 @@
package com.intellij.util.indexing.provided;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.indexing.FileBasedIndexExtension;
import com.intellij.util.indexing.FileContent;
import com.intellij.util.indexing.ID;
import com.intellij.util.indexing.UpdatableIndex;
import com.intellij.util.indexing.hash.FileContentHashIndex;
import com.intellij.util.indexing.*;
import com.intellij.util.indexing.hash.MergedInvertedIndex;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.KeyDescriptor;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
public interface ProvidedIndexExtension<K, V> {
Logger LOG = Logger.getInstance(ProvidedIndexExtension.class);
@@ -32,16 +28,15 @@ public interface ProvidedIndexExtension<K, V> {
DataExternalizer<V> createValueExternalizer();
@NotNull
static <K, V> UpdatableIndex<K, V, FileContent> wrapWithProvidedIndex(@NotNull List<ProvidedIndexExtension<K, V>> providedIndexExtensions,
static <K, V> UpdatableIndex<K, V, FileContent> wrapWithProvidedIndex(@NotNull ProvidedIndexExtension<K, V> providedIndexExtension,
@NotNull FileBasedIndexExtension<K, V> originalExtension,
@NotNull UpdatableIndex<K, V, FileContent> index,
@NotNull FileContentHashIndex contentHashIndex) {
@NotNull UpdatableIndex<K, V, FileContent> index) {
try {
return MergedInvertedIndex.create(providedIndexExtensions, originalExtension, index, contentHashIndex);
return MergedInvertedIndex.create(providedIndexExtension, originalExtension, index);
}
catch (IOException e) {
LOG.error(e);
return index;
}
}
}
}
platform/lang-impl/src/com/intellij/util/indexing/provided/ProvidedIndexExtensionLocator.java
+7 -9
View File
@@ -4,20 +4,18 @@ package com.intellij.util.indexing.provided;
import com.intellij.openapi.extensions.ExtensionPointName;
import com.intellij.util.indexing.FileBasedIndexExtension;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public interface ProvidedIndexExtensionLocator {
ExtensionPointName<ProvidedIndexExtensionLocator> EP_NAME = ExtensionPointName.create("com.intellij.fileBasedIndex.providedLocator");
@NotNull
<K, V> Stream<ProvidedIndexExtension<K, V>> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension);
@Nullable
<K, V> ProvidedIndexExtension<K, V> findProvidedIndexExtension(@NotNull FileBasedIndexExtension<K, V> originalExtension);
@NotNull
static <K, V> List<ProvidedIndexExtension<K, V>> findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
return EP_NAME.extensions().flatMap(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).collect(Collectors.toList());
@Nullable
static <K, V> ProvidedIndexExtension<K, V> findProvidedIndexExtensionFor(@NotNull FileBasedIndexExtension<K, V> originalExtension) {
return EP_NAME.extensions().map(ex -> ex.findProvidedIndexExtension(originalExtension)).filter(Objects::nonNull).findFirst().orElse(null);
}
}
}
platform/lang-impl/src/com/intellij/util/indexing/snapshot/IndexedHashesSupport.java
-11
View File
@@ -64,17 +64,6 @@ public class IndexedHashesSupport {
content.setHashes(fileContentHash, documentHash != null ? documentHash : fileContentHash);
}
@NotNull
public static byte[] getOrInitIndexedHash(@NotNull FileContentImpl content, boolean fromDocument) {
byte[] hash = content.getHash(fromDocument);
if (hash == null) {
initIndexedHash(content);
hash = content.getHash(fromDocument);
LOG.assertTrue(hash != null);
}
return hash;
}
@NotNull
private static byte[] calculateIndexedHashForFileContent(@NotNull FileContentImpl content, boolean binary) {
byte[] contentHash = null;
platform/lang-impl/src/com/intellij/util/indexing/snapshot/SnapshotInputMappings.java
+7 -1
View File
@@ -280,7 +280,13 @@ public class SnapshotInputMappings<Key, Value, Input> implements UpdatableSnapsh
Integer previouslyCalculatedContentHashId = content.getUserData(key);
if (previouslyCalculatedContentHashId == null) {
byte[] hash = IndexedHashesSupport.getOrInitIndexedHash(content, fromDocument);
byte[] hash = content.getHash(fromDocument);
if (hash == null) {
IndexedHashesSupport.initIndexedHash(content);
hash = content.getHash(fromDocument);
LOG.assertTrue(hash != null);
}
previouslyCalculatedContentHashId = IndexedHashesSupport.enumerateHash(hash);
content.putUserData(key, previouslyCalculatedContentHashId);
}
platform/lang-impl/src/org/jetbrains/index/stubs/StubHashBasedIndexGenerator.java
+15 -16
View File
@@ -1,10 +1,10 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.index.stubs;
import com.google.common.collect.Maps;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.newvfs.persistent.ContentHashesUtil;
import com.intellij.psi.stubs.*;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.FileBasedIndexExtension;
@@ -26,19 +26,19 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
private final Set<StubIndexKey> myUsedKeys = new HashSet<>();
public StubHashBasedIndexGenerator(@NotNull File out) {
super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(
true,
null,
StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE), getExtension(), out);
super(EnumeratorIntegerDescriptor.INSTANCE, new SerializedStubTreeDataExternalizer(false, null), getExtension(), out, out);
for (StubIndexExtension<?, ?> stubIndexExtension : StubIndexExtension.EP_NAME.getExtensionList()) {
FileBasedIndexExtension<?, Void> ex = StubIndexImpl
.wrapStubIndexExtension(stubIndexExtension);
myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(ex.getKeyDescriptor(),
ex.getValueExternalizer(),
ex,
new File(out, getStubsDir())
) {
myStubIndexesGeneratorMap.put(stubIndexExtension.getKey(), new HashBasedIndexGenerator(stubIndexExtension.getKeyDescriptor(),
StubIndexImpl.StubIdExternalizer.INSTANCE,
StubIndexImpl
.wrapStubIndexExtension(stubIndexExtension),
new File(out, getStubsDir()),
out) {
@NotNull
@Override
protected ContentHashesUtil.HashEnumerator getHashEnumerator() {
return StubHashBasedIndexGenerator.this.myHashEnumerator;
}
});
}
}
@@ -59,8 +59,7 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
Map<Object, StubIdList> value = entry.getValue();
myUsedKeys.add(key);
MapReduceIndex index = (MapReduceIndex)myStubIndexesGeneratorMap.get(key).getIndex();
Map<Object, Object> reducedValue = Maps.asMap(value.keySet(), k -> null);
index.updateWithMap(new UpdateData(hashId, reducedValue, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null));
index.updateWithMap(new UpdateData(hashId, value, () -> new EmptyInputDataDiffBuilder(hashId), index.getExtension().getName(), null));
}
}
@@ -84,6 +83,6 @@ public class StubHashBasedIndexGenerator extends HashBasedIndexGenerator<Integer
}
private static StubUpdatingIndex getExtension() {
return (new StubUpdatingIndex(StubForwardIndexExternalizer.FileLocalStubForwardIndexExternalizer.INSTANCE));
return (StubUpdatingIndex)FileBasedIndexExtension.EXTENSION_POINT_NAME.extensions().filter(ex -> ex instanceof StubUpdatingIndex).findAny().get();
}
}
platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java
+3 -8
View File
@@ -264,7 +264,7 @@ public abstract class MapReduceIndex<Key,Value, Input> implements InvertedIndex<
@NotNull
protected UpdateData<Key, Value> calculateUpdateData(final int inputId, @Nullable Input content) {
final InputData<Key, Value> data = mapInput(inputId, content);
final InputData<Key, Value> data = mapInput(content);
return createUpdateData(inputId,
data.getKeyValues(),
() -> getKeysDiffBuilder(inputId),
@@ -302,21 +302,16 @@ public abstract class MapReduceIndex<Key,Value, Input> implements InvertedIndex<
}
@NotNull
protected InputData<Key, Value> mapInput(int inputId, @Nullable Input content) {
protected InputData<Key, Value> mapInput(@Nullable Input content) {
if (content == null) {
return InputData.empty();
}
Map<Key, Value> data = mapByIndexer(inputId, content);
Map<Key, Value> data = myIndexer.map(content);
checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer);
checkCanceled();
return new InputData<>(data);
}
@NotNull
protected Map<Key, Value> mapByIndexer(int inputId, @NotNull Input content) {
return myIndexer.map(content);
}
public abstract void checkCanceled();
protected abstract void requestRebuild(@NotNull Throwable e);
platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java
+7
View File
@@ -12,6 +12,13 @@ public class MergedValueContainer<Value> extends ValueContainer<Value> {
private final List<ValueContainer<Value>> myContainers;
private int mySize;
@NotNull
public static <Value> ValueContainer<Value> merge(@NotNull ValueContainer<Value> container1, @NotNull ValueContainer<Value> container2) {
if (container1.size() == 0) return container2;
if (container2.size() == 0) return container1;
return new MergedValueContainer<>(Arrays.asList(container1, container2));
}
public MergedValueContainer(@NotNull List<ValueContainer<Value>> containers) {
if (containers.isEmpty()) {
throw new IllegalArgumentException();