MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

extract MapReduceIndexBase (doesn't depend on VFS & IDE's components) from MapReduceIndex

Browse Source
This commit is contained in:
Dmitry Batkovich
2016-11-16 13:04:17 +03:00
parent a97039fadb
commit d931df063c
33 changed files with 1180 additions and 782 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java
+15 -193
View File
@@ -25,12 +25,10 @@ import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.psi.search.ProjectAndLibrariesScope;
import com.intellij.psi.search.ProjectScopeImpl;
import com.intellij.util.Processor;
import com.intellij.util.Processors;
import com.intellij.util.SystemProperties;
import com.intellij.util.ThrowableRunnable;
import com.intellij.util.containers.ConcurrentIntObjectMap;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.SLRUCache;
import com.intellij.util.io.*;
import com.intellij.util.io.DataOutputStream;
import gnu.trove.TIntHashSet;
@@ -39,31 +37,18 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* @author Eugene Zhuravlev
* Date: Dec 20, 2007
*/
public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
public final class MapIndexStorage<Key, Value> extends MapIndexStorageBase<Key, Value> implements VfsAwareIndexStorage<Key, Value> {
private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true);
private final boolean myBuildKeyHashToVirtualFileMapping;
private PersistentMap<Key, ValueContainer<Value>> myMap;
private AppendableStorageBackedByResizableMappedFile myKeyHashToVirtualFileMapping;
private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
private volatile int myLastScannedId;
private final File myBaseStorageFile;
private final KeyDescriptor<Key> myKeyDescriptor;
private final int myCacheSize;
private final Lock l = new ReentrantLock();
private final DataExternalizer<Value> myDataExternalizer;
private final boolean myKeyIsUniqueForIndexedFile;
private static final ConcurrentIntObjectMap<Boolean> ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap();
public MapIndexStorage(@NotNull File storageFile,
@@ -80,85 +65,18 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
final int cacheSize,
boolean keyIsUniqueForIndexedFile,
boolean buildKeyHashToVirtualFileMapping) throws IOException {
myBaseStorageFile = storageFile;
myKeyDescriptor = keyDescriptor;
myCacheSize = cacheSize;
myDataExternalizer = valueExternalizer;
myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
super(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile);
myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
initMapAndCache();
}
private static final PersistentHashMapValueStorage.ExceptionalIOCancellationCallback ourProgressManagerCheckCancelledIOCanceller =
new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() {
@Override
public void checkCancellation() {
ProgressManager.checkCanceled();
}
};
private void initMapAndCache() throws IOException {
final ValueContainerMap<Key, Value> map;
PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(ourProgressManagerCheckCancelledIOCanceller);
PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE);
try {
map = new ValueContainerMap<>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile);
} finally {
PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null);
PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null);
}
myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
@Override
@NotNull
public ChangeTrackingValueContainer<Value> createValue(final Key key) {
return new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer<Value>() {
@NotNull
@Override
public Object getLock() {
return map.getDataAccessLock();
}
@Nullable
@Override
public ValueContainer<Value> compute() {
ValueContainer<Value> value;
try {
value = map.get(key);
if (value == null) {
value = new ValueContainerImpl<>();
}
}
catch (IOException e) {
throw new RuntimeException(e);
}
return value;
}
});
}
@Override
protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
if (valueContainer.isDirty()) {
try {
map.put(key, valueContainer);
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
myMap = map;
@Override
protected void initMapAndCache() throws IOException {
super.initMapAndCache();
myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ?
new AppendableStorageBackedByResizableMappedFile(getProjectFile(), 4096, null, PagedFileStorage.MB, true) : null;
}
@NotNull
private File getStorageFile() {
return new File(myBaseStorageFile.getPath() + ".storage");
}
@NotNull
private File getProjectFile() {
return new File(myBaseStorageFile.getPath() + ".project");
@@ -172,14 +90,12 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
myKeyHashToVirtualFileMapping.getPagedFileStorage().unlock();
}
}
@Override
public void flush() {
l.lock();
try {
if (!myMap.isClosed()) {
myCache.clear();
if (myMap.isDirty()) myMap.force();
}
super.flush();
if (myKeyHashToVirtualFileMapping != null && myKeyHashToVirtualFileMapping.isDirty()) {
withLock(() -> myKeyHashToVirtualFileMapping.force());
}
@@ -191,57 +107,34 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
@Override
public void close() throws StorageException {
super.close();
try {
flush();
if (myKeyHashToVirtualFileMapping != null) {
withLock(() -> myKeyHashToVirtualFileMapping.close());
}
myMap.close();
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
final Throwable cause = e.getCause();
if (cause instanceof IOException) {
throw new StorageException(cause);
}
if (cause instanceof StorageException) {
throw (StorageException)cause;
}
throw e;
unwrapCauseAndRethrow(e);
}
}
@Override
public void clear() throws StorageException{
try {
myMap.close();
if (myKeyHashToVirtualFileMapping != null) {
withLock(() -> myKeyHashToVirtualFileMapping.close());
}
}
catch (IOException|RuntimeException e) {
catch (RuntimeException e) {
LOG.error(e);
}
try {
IOUtil.deleteAllFilesStartingWith(getStorageFile());
if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile());
initMapAndCache();
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
final Throwable cause = e.getCause();
if (cause instanceof IOException) {
throw new StorageException(cause);
}
if (cause instanceof StorageException) {
throw (StorageException)cause;
}
throw e;
unwrapCauseAndRethrow(e);
}
super.clear();
}
@Override
@@ -302,20 +195,13 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
return processor.process(key);
});
}
return myMap.processKeys(processor);
return processKeys(processor);
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
final Throwable cause = e.getCause();
if (cause instanceof IOException) {
throw new StorageException(cause);
}
if (cause instanceof StorageException) {
throw (StorageException)cause;
}
throw e;
return unwrapCauseAndRethrow(e);
}
finally {
l.unlock();
@@ -406,36 +292,6 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
return new File(getSessionDir(), getProjectFile().getName() + "." + project.hashCode() + "." + id + "." + scope.isSearchInLibraries());
}
@NotNull
@Override
public Collection<Key> getKeys() throws StorageException {
List<Key> keys = new ArrayList<>();
processKeys(Processors.cancelableCollectProcessor(keys), null, null);
return keys;
}
@Override
@NotNull
public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
l.lock();
try {
return myCache.get(key);
}
catch (RuntimeException e) {
final Throwable cause = e.getCause();
if (cause instanceof IOException) {
throw new StorageException(cause);
}
if (cause instanceof StorageException) {
throw (StorageException)cause;
}
throw e;
}
finally {
l.unlock();
}
}
@Override
public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
try {
@@ -447,41 +303,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
myLastScannedId = 0;
}
}
myMap.markDirty();
if (!myKeyIsUniqueForIndexedFile) {
read(key).addValue(inputId, value);
return;
}
ChangeTrackingValueContainer<Value> cached;
try {
l.lock();
cached = myCache.getIfCached(key);
} finally {
l.unlock();
}
if (cached != null) {
cached.addValue(inputId, value);
return;
}
// do not pollute the cache with keys unique to indexed file
ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<>(null);
valueContainer.addValue(inputId, value);
myMap.put(key, valueContainer);
}
catch (IOException e) {
throw new StorageException(e);
}
}
@Override
public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
try {
myMap.markDirty();
// important: assuming the key exists in the index
read(key).removeAssociatedValue(inputId);
super.addValue(key, inputId, value);
}
catch (IOException e) {
throw new StorageException(e);
platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java
+67 -329
View File
@@ -24,7 +24,6 @@ import com.intellij.openapi.editor.Document;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.util.*;
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.openapi.util.io.ByteSequence;
@@ -46,72 +45,43 @@ import java.io.*;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* @author Eugene Zhuravlev
* Date: Dec 10, 2007
*/
public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Value, Input> {
public class MapReduceIndex<Key, Value, Input> extends MapReduceIndexBase<Key,Value, Input> implements UpdatableIndex<Key,Value, Input>{
private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
private static final int NULL_MAPPING = 0;
@NotNull private final ID<Key, Value> myIndexId;
private final DataIndexer<Key, Value, Input> myIndexer;
@NotNull protected final IndexStorage<Key, Value> myStorage;
private final boolean myHasSnapshotMapping;
private final DataExternalizer<Value> myValueExternalizer;
private final DataExternalizer<Collection<Key>> mySnapshotIndexExternalizer;
private final boolean myIsPsiBackedIndex;
private final IndexExtension<Key, Value, Input> myExtension;
private final AtomicBoolean myInMemoryMode = new AtomicBoolean();
private final AtomicLong myModificationStamp = new AtomicLong();
private final TIntObjectHashMap<Collection<Key>> myInMemoryKeys = new TIntObjectHashMap<>();
private PersistentHashMap<Integer, ByteSequence> myContents;
private PersistentHashMap<Integer, Integer> myInputsSnapshotMapping;
@Nullable protected PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
private PersistentHashMap<Integer, String> myIndexingTrace;
private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();
private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() {
@Override
public void run() {
try {
if (myStorage instanceof MemoryIndexStorage) {
Lock writeLock = getWriteLock();
if (writeLock.tryLock()) {
try {
((MemoryIndexStorage<Key, Value>)myStorage).clearCaches();
} finally {
writeLock.unlock();
}
}
}
flush();
} catch (StorageException e) {
LOG.info(e);
requestRebuild(null);
}
static {
if (!DebugAssertions.DEBUG) {
final Application app = ApplicationManager.getApplication();
DebugAssertions.DEBUG = app.isEAP() || app.isInternal();
}
});
}
public MapReduceIndex(IndexExtension<Key, Value, Input> extension,
@NotNull IndexStorage<Key, Value> storage) throws IOException {
myIndexId = extension.getName();
myExtension = extension;
super(extension, storage, false);
SharedIndicesData.registerIndex(myIndexId, extension);
myIndexer = extension.getIndexer();
myStorage = storage;
myHasSnapshotMapping = extension instanceof FileBasedIndexExtension &&
((FileBasedIndexExtension<Key, Value>)extension).hasSnapshotMapping() &&
IdIndex.ourSnapshotMappingsEnabled;
mySnapshotIndexExternalizer = createInputsIndexExternalizer(extension, myIndexId, extension.getKeyDescriptor());
myValueExternalizer = extension.getValueExternalizer();
myIsPsiBackedIndex = extension instanceof PsiDependentIndex;
myContents = createContentsIndex(); // todo
@@ -146,21 +116,9 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
private static <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension,
ID<K, ?> indexId,
KeyDescriptor<K> keyDescriptor) {
DataExternalizer<Collection<K>> externalizer;
if (extension instanceof CustomInputsIndexFileBasedIndexExtension) {
externalizer = ((CustomInputsIndexFileBasedIndexExtension<K>)extension).createExternalizer();
} else {
externalizer = new InputIndexDataExternalizer<>(keyDescriptor, indexId);
}
return externalizer;
}
@NotNull
private static <K> PersistentHashMap<Integer, Collection<K>> createIdToDataKeysIndex(@NotNull IndexExtension <K, ?, ?> extension,
@NotNull MemoryIndexStorage<K, ?> storage)
private <K> PersistentHashMap<Integer, Collection<K>> createIdToDataKeysIndex(@NotNull IndexExtension <K, ?, ?> extension,
@NotNull MemoryIndexStorageBase<K, ?, ?> storage)
throws IOException {
ID<K, ?> indexId = extension.getName();
KeyDescriptor<K> keyDescriptor = extension.getKeyDescriptor();
@@ -186,41 +144,19 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
@NotNull
public IndexStorage<Key, Value> getStorage() {
return myStorage;
}
@Override
public void clear() throws StorageException {
try {
getWriteLock().lock();
myStorage.clear();
if (myInputsIndex != null) {
cleanMapping(myInputsIndex);
myInputsIndex = createInputsIndex();
}
if (myInputsSnapshotMapping != null) {
cleanMapping(myInputsSnapshotMapping);
myInputsSnapshotMapping = createInputSnapshotMapping();
}
if (myIndexingTrace != null) {
cleanMapping(myIndexingTrace);
myIndexingTrace = createIndexingTrace();
}
if (myContents != null) {
cleanMapping(myContents);
myContents = createContentsIndex();
}
protected void cleanMappings() throws IOException {
if (myInputsSnapshotMapping != null) {
cleanMapping(myInputsSnapshotMapping);
myInputsSnapshotMapping = createInputSnapshotMapping();
}
catch (StorageException e) {
LOG.error(e);
if (myIndexingTrace != null) {
cleanMapping(myIndexingTrace);
myIndexingTrace = createIndexingTrace();
}
catch (IOException e) {
LOG.error(e);
}
finally {
getWriteLock().unlock();
if (myContents != null) {
cleanMapping(myContents);
myContents = createContentsIndex();
}
}
@@ -265,72 +201,18 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
private static void cleanMapping(@NotNull PersistentHashMap<?, ?> index) {
final File baseFile = index.getBaseFile();
try {
index.close();
}
catch (Throwable ignored) {
}
IOUtil.deleteAllFilesStartingWith(baseFile);
@Override
protected void doForce() {
doForce(myInputsSnapshotMapping);
doForce(myIndexingTrace);
doForce(myContents);
}
@Override
public void flush() throws StorageException{
try {
getReadLock().lock();
doForce(myInputsIndex);
doForce(myInputsSnapshotMapping);
doForce(myIndexingTrace);
doForce(myContents);
myStorage.flush();
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
final Throwable cause = e.getCause();
if (cause instanceof StorageException || cause instanceof IOException) {
throw new StorageException(cause);
}
else {
throw e;
}
}
finally {
getReadLock().unlock();
}
}
private static void doForce(@Nullable PersistentHashMap<?, ?> inputsIndex) {
if (inputsIndex != null && inputsIndex.isDirty()) {
inputsIndex.force();
}
}
@Override
public void dispose() {
myLowMemoryFlusher.stop();
final Lock lock = getWriteLock();
try {
lock.lock();
try {
myStorage.close();
}
finally {
doClose(myInputsIndex);
doClose(myInputsSnapshotMapping);
doClose(myIndexingTrace);
doClose(myContents);
}
}
catch (StorageException e) {
LOG.error(e);
}
finally {
lock.unlock();
}
protected void doClose() {
doClose(myInputsSnapshotMapping);
doClose(myIndexingTrace);
doClose(myContents);
}
@Override
@@ -348,35 +230,12 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
return IndexingStamp.isFileIndexedStateCurrent(fileId, myIndexId);
}
private static void doClose(@Nullable PersistentHashMap<?, ?> index) {
if (index != null) {
try {
index.close();
}
catch (IOException e) {
LOG.error(e);
}
}
}
@NotNull
@Override
public final Lock getReadLock() {
return myLock.readLock();
}
@NotNull
@Override
public final Lock getWriteLock() {
return myLock.writeLock();
}
@Override
public boolean processAllKeys(@NotNull Processor<Key> processor, @NotNull GlobalSearchScope scope, IdFilter idFilter) throws StorageException {
final Lock lock = getReadLock();
try {
lock.lock();
return myStorage.processKeys(processor, scope, idFilter);
return ((VfsAwareIndexStorage<Key, Value>)myStorage).processKeys(processor, scope, idFilter);
}
finally {
lock.unlock();
@@ -384,22 +243,8 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
@Override
@NotNull
public ValueContainer<Value> getData(@NotNull final Key key) throws StorageException {
final Lock lock = getReadLock();
try {
lock.lock();
ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
return myStorage.read(key);
}
finally {
ValueContainerImpl.ourDebugIndexInfo.set(null);
lock.unlock();
}
}
protected PersistentHashMap<Integer, Collection<Key>> createInputsIndex() throws IOException {
return createIdToDataKeysIndex(myExtension, (MemoryIndexStorage<Key, ?>)myStorage);
return createIdToDataKeysIndex(myExtension, (MemoryIndexStorageBase<Key, ?, ?>)myStorage);
}
private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true);
@@ -457,9 +302,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
if (data == null) {
data = content != null ? myIndexer.map(content) : Collections.emptyMap();
if (DebugAssertions.DEBUG) {
checkValuesHaveProperEqualsAndHashCode(data);
}
checkValuesHaveProperEqualsAndHashCode(data);
}
if (hashId != null && !havePersistentData) {
@@ -477,7 +320,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
}
ProgressManager.checkCanceled();
myStorage.checkCanceled();
UpdateData<Key, Value> optimizedUpdateData = null;
final NotNullComputable<Collection<Key>> oldKeysGetter;
@@ -559,15 +402,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
throw new RuntimeException(ex);
}
} else {
oldKeysGetter = () -> {
try {
Collection<Key> oldKeys = readInputKeys(inputId);
return oldKeys == null? Collections.<Key>emptyList() : oldKeys;
}
catch (IOException e) {
throw new RuntimeException(e);
}
};
oldKeysGetter = createOldKeysGetterByInputIndex(inputId);
savedInputId = inputId;
}
@@ -580,13 +415,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
catch (StorageException|ProcessCanceledException ex) {
LOG.info("Exception during updateWithMap:" + ex);
Application application = ApplicationManager.getApplication();
if (application.isUnitTestMode() || application.isHeadlessEnvironment()) {
// avoid deadlock due to synchronous update in DumbServiceImpl#queueTask
application.invokeLater(() -> requestRebuild(ex), ModalityState.any());
} else {
requestRebuild(ex);
}
requestRebuild(ex);
return Boolean.FALSE;
}
@@ -594,15 +423,35 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
};
}
protected void requestRebuild(@Nullable Exception ex) {
if (ex == null) {
FileBasedIndex.getInstance().requestRebuild(myIndexId);
}
else {
FileBasedIndex.getInstance().requestRebuild(myIndexId, ex);
@Override
public void requestRebuild(@Nullable Exception ex) {
Runnable action = () -> {
if (ex == null) {
FileBasedIndex.getInstance().requestRebuild(myIndexId);
}
else {
FileBasedIndex.getInstance().requestRebuild(myIndexId, ex);
}
};
Application application = ApplicationManager.getApplication();
if (application.isUnitTestMode() || application.isHeadlessEnvironment()) {
// avoid deadlock due to synchronous update in DumbServiceImpl#queueTask
application.invokeLater(action, ModalityState.any());
} else {
action.run();
}
}
@Override
protected <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension,
ID<K, ?> indexId,
KeyDescriptor<K> keyDescriptor) {
return extension instanceof CustomInputsIndexFileBasedIndexExtension ?
((CustomInputsIndexFileBasedIndexExtension<K>)extension).createExternalizer() :
super.createInputsIndexExternalizer(extension, indexId, keyDescriptor);
}
@Override
protected UpdateData<Key, Value> buildUpdateData(Map<Key, Value> data, NotNullComputable<Collection<Key>> oldKeysGetter, int savedInputId) {
return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter);
}
@@ -678,7 +527,8 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.put(inputId, savedInputId);
}
private Collection<Key> readInputKeys(int inputId) throws IOException {
@Override
protected Collection<Key> readInputKeys(int inputId) throws IOException {
if (myInMemoryMode.get()) {
synchronized (myInMemoryKeys) {
Collection<Key> keys = myInMemoryKeys.get(inputId);
@@ -710,10 +560,11 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
return keys;
}
return myInputsIndex != null ? myInputsIndex.get(inputId) : null;
return super.readInputKeys(inputId);
}
private void saveInputKeys(int inputId, int savedInputId, Map<Key, Value> newData) throws IOException {
@Override
protected void saveInputKeys(int inputId, int savedInputId, Map<Key, Value> newData) throws IOException {
if (myInMemoryMode.get()) {
synchronized (myInMemoryKeys) {
myInMemoryKeys.put(inputId, newData.keySet());
@@ -722,14 +573,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
if (myHasSnapshotMapping) {
saveInputHashId(inputId, savedInputId);
} else {
if (myInputsIndex != null) {
if (newData.size() > 0) {
myInputsIndex.put(inputId, newData.keySet());
}
else {
myInputsIndex.remove(inputId);
}
}
super.saveInputKeys(inputId, savedInputId, newData);
if (SharedIndicesData.ourFileSharedIndicesEnabled) {
Set<Key> newKeys = newData.keySet();
@@ -740,32 +584,6 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
}
}
private void checkValuesHaveProperEqualsAndHashCode(Map<Key, Value> data) {
for(Map.Entry<Key, Value> e: data.entrySet()) {
final Value value = e.getValue();
if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) {
LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter");
}
if (myValueExternalizer != null) {
try {
final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream();
DataOutputStream outputStream = new DataOutputStream(out);
myValueExternalizer.save(outputStream, value);
outputStream.close();
final Value deserializedValue =
myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size())));
if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) {
LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter");
}
} catch (IOException ex) {
LOG.error(ex);
}
}
}
}
private StringBuilder buildDiff(Map<Key, Value> data, Map<Key, Value> contentData) {
StringBuilder moreInfo = new StringBuilder();
if (contentData.size() != data.size()) {
@@ -906,84 +724,4 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val
private static final com.intellij.openapi.util.Key<Integer> ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id");
private static final com.intellij.openapi.util.Key<Integer> ourSavedUncommittedHashIdKey = com.intellij.openapi.util.Key.create("saved.uncommitted.hash.id");
public IndexExtension<Key, Value, Input> getExtension() {
return myExtension;
}
public long getModificationStamp() {
return myModificationStamp.get();
}
public class SimpleUpdateData extends UpdateData<Key, Value> {
private final int savedInputId;
private final @NotNull Map<Key, Value> newData;
protected final @NotNull NotNullComputable<Collection<Key>> oldKeysGetter;
public SimpleUpdateData(ID<Key,Value> indexId, int id, @NotNull Map<Key, Value> data, @NotNull NotNullComputable<Collection<Key>> getter) {
super(indexId);
savedInputId = id;
newData = data;
oldKeysGetter = getter;
}
public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Key> consumer) throws StorageException {
MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer);
}
public void iterateAddedKeys(final int inputId, final AddedKeyProcessor<Key, Value> consumer) throws StorageException {
MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData);
}
@Override
public void save(int inputId) throws IOException {
saveInputKeys(inputId, savedInputId, newData);
}
public @NotNull Map<Key, Value> getNewData() {
return newData;
}
}
private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>
myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>() {
@Override
public void process(Key key, int inputId) throws StorageException {
myModificationStamp.incrementAndGet();
myStorage.removeAllValues(key, inputId);
}
};
private final MapDiffUpdateData.AddedKeyProcessor<Key, Value> myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor<Key, Value>() {
@Override
public void process(Key key, Value value, int inputId) throws StorageException {
myModificationStamp.incrementAndGet();
myStorage.addValue(key, inputId, value);
}
};
protected void updateWithMap(final int inputId,
@NotNull UpdateData<Key, Value> updateData) throws StorageException {
getWriteLock().lock();
try {
try {
ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation);
updateData.iterateAddedKeys(inputId, myAddedKeyProcessor);
updateData.save(inputId);
}
catch (ProcessCanceledException pce) {
throw pce; // extra care
}
catch (Throwable e) { // e.g. IOException, AssertionError
throw new StorageException(e);
}
finally {
ValueContainerImpl.ourDebugIndexInfo.set(null);
}
}
finally {
getWriteLock().unlock();
}
}
}
platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java
+5 -182
View File
@@ -18,199 +18,22 @@ package com.intellij.util.indexing;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import com.intellij.util.Processors;
import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
import java.io.IOException;
import java.util.*;
/**
* This storage is needed for indexing yet unsaved data without saving those changes to 'main' backend storage
*
* @author Eugene Zhuravlev
* Date: Dec 10, 2007
*/
public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value> {
private final Map<Key, ChangeTrackingValueContainer<Value>> myMap = new HashMap<>();
@NotNull
private final IndexStorage<Key, Value> myBackendStorage;
private final List<BufferingStateListener> myListeners = ContainerUtil.createLockFreeCopyOnWriteList();
private final ID<?, ?> myIndexId;
private boolean myBufferingEnabled;
public interface BufferingStateListener {
void bufferingStateChanged(boolean newState);
void memoryStorageCleared();
}
public MemoryIndexStorage(@NotNull IndexStorage<Key, Value> backend) {
this(backend, null);
}
public MemoryIndexStorage(@NotNull IndexStorage<Key, Value> backend, ID<?, ?> indexId) {
myBackendStorage = backend;
myIndexId = indexId;
}
@NotNull
public IndexStorage<Key, Value> getBackendStorage() {
return myBackendStorage;
}
public void addBufferingStateListener(@NotNull BufferingStateListener listener) {
myListeners.add(listener);
}
public void removeBufferingStateListener(@NotNull BufferingStateListener listener) {
myListeners.remove(listener);
}
public void setBufferingEnabled(boolean enabled) {
final boolean wasEnabled = myBufferingEnabled;
assert wasEnabled != enabled;
myBufferingEnabled = enabled;
for (BufferingStateListener listener : myListeners) {
listener.bufferingStateChanged(enabled);
}
}
public boolean isBufferingEnabled() {
return myBufferingEnabled;
}
public void clearMemoryMap() {
myMap.clear();
}
public void fireMemoryStorageCleared() {
for (BufferingStateListener listener : myListeners) {
listener.memoryStorageCleared();
}
}
void clearCaches() {
if (myMap.size() == 0) return;
if (DebugAssertions.DEBUG) {
String message = "Dropping caches for " + (myIndexId != null ? myIndexId:this) + ", number of items:" + myMap.size();
FileBasedIndexImpl.LOG.info(message);
}
for(ChangeTrackingValueContainer<Value> v:myMap.values()) {
v.dropMergedData();
}
}
@Override
public void close() throws StorageException {
myBackendStorage.close();
}
@Override
public void clear() throws StorageException {
clearMemoryMap();
myBackendStorage.clear();
}
@Override
public void flush() throws IOException {
myBackendStorage.flush();
}
@NotNull
@Override
public Collection<Key> getKeys() throws StorageException {
final Set<Key> keys = new HashSet<>();
processKeys(Processors.cancelableCollectProcessor(keys), null, null);
return keys;
public class MemoryIndexStorage<Key, Value> extends MemoryIndexStorageBase<Key, Value, VfsAwareIndexStorage<Key, Value>> implements VfsAwareIndexStorage<Key, Value> {
public MemoryIndexStorage(@NotNull VfsAwareIndexStorage<Key, Value> backend, ID<?, ?> indexId) {
super(backend, indexId);
}
@Override
public boolean processKeys(@NotNull final Processor<Key> processor, GlobalSearchScope scope, IdFilter idFilter) throws StorageException {
final Set<Key> stopList = new HashSet<>();
Processor<Key> decoratingProcessor = key -> {
if (stopList.contains(key)) return true;
final UpdatableValueContainer<Value> container = myMap.get(key);
if (container != null && container.size() == 0) {
return true;
}
return processor.process(key);
};
for (Key key : myMap.keySet()) {
if (!decoratingProcessor.process(key)) {
return false;
}
stopList.add(key);
}
return myBackendStorage.processKeys(stopList.isEmpty() && myMap.isEmpty() ? processor : decoratingProcessor, scope, idFilter);
}
@Override
public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
if (myBufferingEnabled) {
getMemValueContainer(key).addValue(inputId, value);
return;
}
final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer != null) {
valueContainer.dropMergedData();
}
myBackendStorage.addValue(key, inputId, value);
}
@Override
public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
if (myBufferingEnabled) {
getMemValueContainer(key).removeAssociatedValue(inputId);
return;
}
final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer != null) {
valueContainer.dropMergedData();
}
myBackendStorage.removeAllValues(key, inputId);
}
private UpdatableValueContainer<Value> getMemValueContainer(final Key key) {
ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer == null) {
valueContainer = new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer<Value>() {
@Override
public Object getLock() {
return this;
}
@Override
public ValueContainer<Value> compute() {
try {
return myBackendStorage.read(key);
}
catch (StorageException e) {
throw new RuntimeException(e);
}
}
});
myMap.put(key, valueContainer);
}
return valueContainer;
}
@Override
@NotNull
public ValueContainer<Value> read(final Key key) throws StorageException {
final ValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer != null) {
return valueContainer;
}
return myBackendStorage.read(key);
final Processor<Key> uniqueResultProcessor = processCacheFirstProcessor(processor);
return uniqueResultProcessor != null && myBackendStorage.processKeys(uniqueResultProcessor, scope, idFilter);
}
}
platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java
+16 -22
View File
@@ -16,39 +16,33 @@
package com.intellij.util.indexing;
import com.intellij.openapi.util.Computable;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.concurrent.locks.Lock;
/**
* @author Eugene Zhuravlev
* Date: Dec 10, 2007
*/
public interface UpdatableIndex<Key, Value, Input> extends AbstractIndex<Key,Value> {
void clear() throws StorageException;
void flush() throws StorageException;
/**
* @param inputId *positive* id of content.
*/
@NotNull
Computable<Boolean> update(int inputId, @Nullable Input content);
@NotNull
Lock getReadLock();
@NotNull
Lock getWriteLock();
void dispose();
public interface UpdatableIndex<Key, Value, Input> extends AbstractUpdatableIndex<Key,Value, Input> {
boolean processAllKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
void setIndexedStateForFile(int fileId, @NotNull VirtualFile file);
void resetIndexedStateForFile(int fileId);
boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file);
@Override
default boolean processAllKeys(@NotNull Processor<Key> processor) throws StorageException {
return processAllKeys(processor, null, null);
}
@NotNull
@Override
default Class<? extends RuntimeException> getProcessCanceledExceptionClass() {
return ProcessCanceledException.class;
}
}
platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java → platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java
+9 -10
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,21 +13,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* @author Eugene Zhuravlev
* Date: Dec 24, 2007
*/
public interface AbstractIndex<Key, Value> {
@NotNull
ValueContainer<Value> getData(@NotNull Key key) throws StorageException;
public interface VfsAwareIndexStorage<Key, Value> extends IndexStorage<Key, Value> {
boolean processAllKeys(@NotNull Processor<Key> processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
boolean processKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
@Override
default void checkCanceled() {
ProgressManager.checkCanceled();
}
}
platform/util/src/com/intellij/util/indexing/AbstractUpdatableIndex.java
+54
View File
@@ -0,0 +1,54 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.openapi.util.Computable;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.concurrent.locks.Lock;
public interface AbstractUpdatableIndex<Key, Value, Input> {
void clear() throws StorageException;
void flush() throws StorageException;
/**
* @param inputId *positive* id of content.
*/
@NotNull
Computable<Boolean> update(int inputId, @Nullable Input content);
@NotNull
ValueContainer<Value> getData(@NotNull Key key) throws StorageException;
boolean processAllKeys(@NotNull Processor<Key> processor) throws StorageException;
@NotNull
Lock getReadLock();
@NotNull
Lock getWriteLock();
void dispose();
void requestRebuild(@Nullable Exception ex);
@NotNull
Class<? extends RuntimeException> getProcessCanceledExceptionClass();
}
platform/util/src/com/intellij/util/indexing/CancelableCollectProcessor.java
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.util.CommonProcessors;
import org.jetbrains.annotations.NotNull;
import java.util.Collection;
class CancelableCollectProcessor<T> extends CommonProcessors.CollectProcessor<T> {
private final Runnable myCancelChecker;
public CancelableCollectProcessor(@NotNull Collection<T> collection, @NotNull Runnable cancelChecker) {
super(collection);
myCancelChecker = cancelChecker;
}
@Override
public boolean process(T o) {
myCancelChecker.run();
return super.process(o);
}
}
platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java → platform/util/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -54,7 +54,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
merged.addValue(inputId, value);
}
if (myAdded == null) myAdded = new ValueContainerImpl<>();
if (myAdded == null) myAdded = new ValueContainerImpl<Value>();
myAdded.addValue(inputId, value);
}
@@ -131,7 +131,7 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
(newMerged.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD ||
(myAdded != null && myAdded.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD))) {
// Calculate file ids that have Value mapped to avoid O(NumberOfValuesInMerged) during removal
fileId2ValueMapping = new FileId2ValueMapping<>(newMerged);
fileId2ValueMapping = new FileId2ValueMapping<Value>(newMerged);
}
final FileId2ValueMapping<Value> finalFileId2ValueMapping = fileId2ValueMapping;
if (myInvalidated != null) {
platform/indexing-api/src/com/intellij/util/indexing/DataIndexer.java → platform/util/src/com/intellij/util/indexing/DataIndexer.java
View File
platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java → platform/util/src/com/intellij/util/indexing/DebugAssertions.java
+6 -6
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@
*/
package com.intellij.util.indexing;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.SystemProperties;
import com.intellij.util.containers.hash.LinkedHashMap;
@@ -27,9 +26,10 @@ import java.util.Formatter;
public class DebugAssertions {
private static final Logger LOG = Logger.getInstance(DebugAssertions.class);
public static final boolean DEBUG = SystemProperties.getBooleanProperty(
@SuppressWarnings("StaticNonFinalField")
public static volatile boolean DEBUG = SystemProperties.getBooleanProperty(
"intellij.idea.indices.debug",
ApplicationManager.getApplication().isInternal() || ApplicationManager.getApplication().isEAP()
false
);
public static final boolean EXTRA_SANITY_CHECKS = SystemProperties.getBooleanProperty(
@@ -56,9 +56,9 @@ public class DebugAssertions {
static <Key> boolean equals(Collection<Key> keys, Collection<Key> keys2, KeyDescriptor<Key> keyDescriptor) {
if (keys == null && keys2 == null) return true;
if (keys == null || keys2 == null || keys.size() != keys2.size()) return false;
LinkedHashMap<Key, Boolean> map = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor);
LinkedHashMap<Key, Boolean> map = new LinkedHashMap<Key, Boolean>(keys.size(), 0.8f, keyDescriptor);
for(Key key:keys) map.put(key, Boolean.TRUE);
LinkedHashMap<Key, Boolean> map2 = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor);
LinkedHashMap<Key, Boolean> map2 = new LinkedHashMap<Key, Boolean>(keys.size(), 0.8f, keyDescriptor);
for(Key key:keys2) map2.put(key, Boolean.TRUE);
return map.equals(map2);
}
platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java → platform/util/src/com/intellij/util/indexing/FileId2ValueMapping.java
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ class FileId2ValueMapping<Value> {
private boolean myOnePerFileValidationEnabled = true;
FileId2ValueMapping(ValueContainerImpl<Value> _valueContainer) {
id2ValueMap = new TIntObjectHashMap<>();
id2ValueMap = new TIntObjectHashMap<Value>();
valueContainer = _valueContainer;
TIntArrayList removedFileIdList = null;
@@ -45,7 +45,7 @@ class FileId2ValueMapping<Value> {
if (previousValue != null) { // delay removal of duplicated id -> value mapping since it will affect valueIterator we are using
if (removedFileIdList == null) {
removedFileIdList = new TIntArrayList();
removedValueList = new SmartList<>();
removedValueList = new SmartList<Value>();
}
removedFileIdList.add(id);
removedValueList.add(previousValue);
platform/core-api/src/com/intellij/util/indexing/ID.java → platform/util/src/com/intellij/util/indexing/ID.java
View File
platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java → platform/util/src/com/intellij/util/indexing/IndexExtension.java
View File
platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java → platform/util/src/com/intellij/util/indexing/IndexStorage.java
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,10 +16,8 @@
package com.intellij.util.indexing;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.Flushable;
import java.io.IOException;
@@ -40,7 +38,7 @@ public interface IndexStorage<Key, Value> extends Flushable {
@NotNull
ValueContainer<Value> read(Key key) throws StorageException;
boolean processKeys(@NotNull Processor<Key> processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException;
boolean processKeys(@NotNull Processor<Key> processor) throws StorageException;
@NotNull
Collection<Key> getKeys() throws StorageException;
@@ -49,4 +47,6 @@ public interface IndexStorage<Key, Value> extends Flushable {
@Override
void flush() throws IOException;
void checkCanceled();
}
platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java → platform/util/src/com/intellij/util/indexing/InputIndexDataExternalizer.java
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -57,7 +57,7 @@ public class InputIndexDataExternalizer<K> implements DataExternalizer<Collectio
public Collection<K> read(@NotNull DataInput in) throws IOException {
try {
final int size = DataInputOutputUtil.readINT(in);
final List<K> list = new ArrayList<>(size);
final List<K> list = new ArrayList<K>(size);
for (int idx = 0; idx < size; idx++) {
list.add(myKeyDescriptor.read(in));
}
platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java → platform/util/src/com/intellij/util/indexing/MapDiffUpdateData.java
+6 -9
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2015 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@
*/
package com.intellij.util.indexing;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.Ref;
@@ -44,7 +43,7 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
Map<Key, Value> data) throws StorageException {
if (data instanceof THashMap) {
// such map often (from IdIndex) contain 100x (avg ~240) of entries, also THashMap have no Entry inside so we optimize for gc too
final Ref<StorageException> exceptionRef = new Ref<>();
final Ref<StorageException> exceptionRef = new Ref<StorageException>();
final boolean b = ((THashMap<Key, Value>)data).forEachEntry(new TObjectObjectProcedure<Key, Value>() {
@Override
public boolean execute(Key key, Value value) {
@@ -81,8 +80,6 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
iterateRemovedKeys(removedOrChangedKeys.keySet(), inputId, consumer);
}
private static final boolean DO_INFO_DUMP = ApplicationManager.getApplication().isInternal();
private void calcDiff() throws StorageException {
if (removedOrChangedKeys != null) return;
@@ -103,10 +100,10 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
if (!Comparing.equal(newValueForKey, e.getValue()) ||
newValueForKey == null && !newValue.containsKey(e.getKey())
) {
if (removedOrChangedKeys == null) removedOrChangedKeys = new THashMap<>();
if (removedOrChangedKeys == null) removedOrChangedKeys = new THashMap<Key, Value>();
removedOrChangedKeys.put(e.getKey(), e.getValue());
if (newValue.containsKey(e.getKey())) {
if (addedKeys == null) addedKeys = new THashMap<>();
if (addedKeys == null) addedKeys = new THashMap<Key, Value>();
addedKeys.put(e.getKey(), newValueForKey);
}
}
@@ -130,7 +127,7 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
}
for (Map.Entry<Key, Value> e : newValue.entrySet()) {
if (!currentValue.containsKey(e.getKey())) {
if (addedKeys == null) addedKeys = new THashMap<>();
if (addedKeys == null) addedKeys = new THashMap<Key, Value>();
addedKeys.put(e.getKey(), e.getValue());
}
}
@@ -145,7 +142,7 @@ public abstract class MapDiffUpdateData<Key, Value> extends UpdateData<Key, Valu
incrementalAdditions.addAndGet(addedKeys.size());
incrementalRemovals.addAndGet(removedOrChangedKeys.size());
if ((totalRequests & 0xFFF) == 0 && DO_INFO_DUMP) {
if ((totalRequests & 0xFFF) == 0 && DebugAssertions.DEBUG) {
Logger.getInstance(getClass()).info("Incremental index diff update:"+requests +
", removals:" + totalRemovals + "->" + incrementalRemovals +
", additions:" +totalAdditions + "->" +incrementalAdditions);
platform/util/src/com/intellij/util/indexing/MapIndexStorageBase.java
+271
View File
@@ -0,0 +1,271 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.Processor;
import com.intellij.util.containers.SLRUCache;
import com.intellij.util.io.*;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public abstract class MapIndexStorageBase<Key, Value> implements IndexStorage<Key, Value> {
private static final Logger LOG = Logger.getInstance(MapIndexStorageBase.class);
protected PersistentMap<Key, ValueContainer<Value>> myMap;
protected SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
protected final File myBaseStorageFile;
protected final KeyDescriptor<Key> myKeyDescriptor;
private final int myCacheSize;
protected final Lock l = new ReentrantLock();
private final DataExternalizer<Value> myDataExternalizer;
private final boolean myKeyIsUniqueForIndexedFile;
protected MapIndexStorageBase(@NotNull File storageFile,
@NotNull KeyDescriptor<Key> keyDescriptor,
@NotNull DataExternalizer<Value> valueExternalizer,
final int cacheSize,
boolean keyIsUniqueForIndexedFile) throws IOException {
myBaseStorageFile = storageFile;
myKeyDescriptor = keyDescriptor;
myCacheSize = cacheSize;
myDataExternalizer = valueExternalizer;
myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
}
protected void initMapAndCache() throws IOException {
final ValueContainerMap<Key, Value> map;
PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(
new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() {
@Override
public void checkCancellation() {
checkCanceled();
}
});
PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE);
try {
map = new ValueContainerMap<Key, Value>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile);
} finally {
PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null);
PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null);
}
myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
@Override
@NotNull
public ChangeTrackingValueContainer<Value> createValue(final Key key) {
return new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
@NotNull
@Override
public Object getLock() {
return map.getDataAccessLock();
}
@Nullable
@Override
public ValueContainer<Value> compute() {
ValueContainer<Value> value;
try {
value = map.get(key);
if (value == null) {
value = new ValueContainerImpl<Value>();
}
}
catch (IOException e) {
throw new RuntimeException(e);
}
return value;
}
});
}
@Override
protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
if (valueContainer.isDirty()) {
try {
map.put(key, valueContainer);
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
myMap = map;
}
@NotNull
private File getStorageFile() {
return new File(myBaseStorageFile.getPath() + ".storage");
}
@Override
public boolean processKeys(@NotNull Processor<Key> processor) throws StorageException {
l.lock();
try {
myCache.clear(); // this will ensure that all new keys are made into the map
return myMap.processKeys(processor);
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
return unwrapCauseAndRethrow(e);
}
finally {
l.unlock();
}
}
@Override
public void flush() {
l.lock();
try {
if (!myMap.isClosed()) {
myCache.clear();
if (myMap.isDirty()) myMap.force();
}
}
finally {
l.unlock();
}
}
@Override
public void close() throws StorageException {
try {
flush();
myMap.close();
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
unwrapCauseAndRethrow(e);
}
}
@Override
public void clear() throws StorageException{
try {
myMap.close();
}
catch (IOException e) {
LOG.error(e);
}
catch (RuntimeException e) {
LOG.error(e);
}
try {
IOUtil.deleteAllFilesStartingWith(getStorageFile());
initMapAndCache();
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
unwrapCauseAndRethrow(e);
}
}
@NotNull
@Override
public Collection<Key> getKeys() throws StorageException {
List<Key> keys = new ArrayList<Key>();
processKeys(new CancelableCollectProcessor<Key>(keys, new Runnable() {
@Override
public void run() {checkCanceled();}
}));
return keys;
}
@Override
@NotNull
public ChangeTrackingValueContainer<Value> read(final Key key) throws StorageException {
l.lock();
try {
return myCache.get(key);
}
catch (RuntimeException e) {
return unwrapCauseAndRethrow(e);
}
finally {
l.unlock();
}
}
@Override
public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
try {
myMap.markDirty();
if (!myKeyIsUniqueForIndexedFile) {
read(key).addValue(inputId, value);
return;
}
ChangeTrackingValueContainer<Value> cached;
try {
l.lock();
cached = myCache.getIfCached(key);
} finally {
l.unlock();
}
if (cached != null) {
cached.addValue(inputId, value);
return;
}
// do not pollute the cache with keys unique to indexed file
ChangeTrackingValueContainer<Value> valueContainer = new ChangeTrackingValueContainer<Value>(null);
valueContainer.addValue(inputId, value);
myMap.put(key, valueContainer);
}
catch (IOException e) {
throw new StorageException(e);
}
}
@Override
public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
try {
myMap.markDirty();
// important: assuming the key exists in the index
read(key).removeAssociatedValue(inputId);
}
catch (IOException e) {
throw new StorageException(e);
}
}
protected static <T> T unwrapCauseAndRethrow(RuntimeException e) throws StorageException {
final Throwable cause = e.getCause();
if (cause instanceof IOException) {
throw new StorageException(cause);
}
if (cause instanceof StorageException) {
throw (StorageException)cause;
}
throw e;
}
}
platform/util/src/com/intellij/util/indexing/MapReduceIndexBase.java
+441
View File
@@ -0,0 +1,441 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.Computable;
import com.intellij.openapi.util.LowMemoryWatcher;
import com.intellij.openapi.util.NotNullComputable;
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.util.Processor;
import com.intellij.util.io.*;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
public abstract class MapReduceIndexBase<Key,Value, Input> implements AbstractUpdatableIndex<Key, Value, Input> {
private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex");
@Nullable protected final ID<Key, Value> myIndexId;
protected final DataIndexer<Key, Value, Input> myIndexer;
@NotNull protected final IndexStorage<Key, Value> myStorage;
protected final DataExternalizer<Value> myValueExternalizer;
protected final IndexExtension<Key, Value, Input> myExtension;
private final AtomicLong myModificationStamp = new AtomicLong();
@Nullable protected PersistentHashMap<Integer, Collection<Key>> myInputsIndex;
private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock();
private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() {
@Override
public void run() {
try {
if (myStorage instanceof MemoryIndexStorageBase) {
Lock writeLock = getWriteLock();
if (writeLock.tryLock()) {
try {
((MemoryIndexStorageBase<?, ?, ?>)myStorage).clearCaches();
} finally {
writeLock.unlock();
}
}
}
flush();
} catch (StorageException e) {
LOG.info(e);
requestRebuild(e);
}
}
});
protected MapReduceIndexBase(IndexExtension<Key, Value, Input> extension,
@NotNull IndexStorage<Key, Value> storage,
boolean createInputIndex) throws IOException {
myIndexId = extension.getName();
myExtension = extension;
myIndexer = extension.getIndexer();
myStorage = storage;
myValueExternalizer = extension.getValueExternalizer();
if (createInputIndex) {
myInputsIndex = createInputsIndex();
}
}
protected PersistentHashMap<Integer, Collection<Key>> createInputsIndex() throws IOException {
return null;
}
protected <K> DataExternalizer<Collection<K>> createInputsIndexExternalizer(IndexExtension<K, ?, ?> extension,
ID<K, ?> indexId,
KeyDescriptor<K> keyDescriptor) {
DataExternalizer<Collection<K>> externalizer;
externalizer = new InputIndexDataExternalizer<K>(keyDescriptor, indexId);
return externalizer;
}
@NotNull
public IndexStorage<Key, Value> getStorage() {
return myStorage;
}
@Override
public void clear() throws StorageException {
try {
getWriteLock().lock();
myStorage.clear();
if (myInputsIndex != null) {
cleanMapping(myInputsIndex);
myInputsIndex = createInputsIndex();
}
cleanMappings();
}
catch (StorageException e) {
LOG.error(e);
}
catch (IOException e) {
LOG.error(e);
}
finally {
getWriteLock().unlock();
}
}
protected void cleanMappings() throws IOException {
}
protected static void cleanMapping(@NotNull PersistentHashMap<?, ?> index) {
final File baseFile = index.getBaseFile();
try {
index.close();
}
catch (Throwable ignored) {
}
IOUtil.deleteAllFilesStartingWith(baseFile);
}
@Override
public void flush() throws StorageException{
try {
getReadLock().lock();
doForce(myInputsIndex);
doForce();
myStorage.flush();
}
catch (IOException e) {
throw new StorageException(e);
}
catch (RuntimeException e) {
final Throwable cause = e.getCause();
if (cause instanceof StorageException || cause instanceof IOException) {
throw new StorageException(cause);
}
else {
throw e;
}
}
finally {
getReadLock().unlock();
}
}
protected void doForce() {
}
protected static void doForce(PersistentHashMap<?, ?> inputsIndex) {
if (inputsIndex != null && inputsIndex.isDirty()) {
inputsIndex.force();
}
}
@Override
public void dispose() {
myLowMemoryFlusher.stop();
final Lock lock = getWriteLock();
try {
lock.lock();
try {
myStorage.close();
}
finally {
doClose(myInputsIndex);
doClose();
}
}
catch (StorageException e) {
LOG.error(e);
}
finally {
lock.unlock();
}
}
protected void doClose() {
}
protected static void doClose(PersistentHashMap<?, ?> index) {
if (index != null) {
try {
index.close();
}
catch (IOException e) {
LOG.error(e);
}
}
}
@NotNull
@Override
public final Lock getReadLock() {
return myLock.readLock();
}
@NotNull
@Override
public final Lock getWriteLock() {
return myLock.writeLock();
}
@Override
@NotNull
public ValueContainer<Value> getData(@NotNull final Key key) throws StorageException {
final Lock lock = getReadLock();
try {
lock.lock();
ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
return myStorage.read(key);
}
finally {
ValueContainerImpl.ourDebugIndexInfo.set(null);
lock.unlock();
}
}
@Override
public boolean processAllKeys(@NotNull Processor<Key> processor) throws StorageException {
final Lock lock = getReadLock();
try {
lock.lock();
return myStorage.processKeys(processor);
}
finally {
lock.unlock();
}
}
@NotNull
@Override
public Computable<Boolean> update(final int inputId, @Nullable Input content) {
Map<Key, Value> data = content != null ? myIndexer.map(content) : Collections.<Key, Value>emptyMap();
checkValuesHaveProperEqualsAndHashCode(data);
myStorage.checkCanceled();
// do not depend on content!
final UpdateData<Key, Value> updateData = buildUpdateData(data,
createOldKeysGetterByInputIndex(inputId),
inputId);
return new Computable<Boolean>() {
@Override
public Boolean compute() {
try {
updateWithMap(inputId, updateData);
}
catch (StorageException ex) {
LOG.info("Exception during updateWithMap:" + ex);
requestRebuild(ex);
return Boolean.FALSE;
}
catch (RuntimeException ex) {
if (getProcessCanceledExceptionClass().isInstance(ex)) {
LOG.info("Exception during updateWithMap:" + ex);
requestRebuild(ex);
return Boolean.FALSE;
} else {
throw ex;
}
}
return Boolean.TRUE;
}
};
}
@NotNull
protected NotNullComputable<Collection<Key>> createOldKeysGetterByInputIndex(final int inputId) {
return new NotNullComputable<Collection<Key>>() {
@NotNull
@Override
public Collection<Key> compute() {
try {
Collection<Key> oldKeys = MapReduceIndexBase.this.readInputKeys(inputId);
return oldKeys == null ? Collections.<Key>emptyList() : oldKeys;
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
};
}
protected UpdateData<Key, Value> buildUpdateData(Map<Key, Value> data, NotNullComputable<Collection<Key>> oldKeysGetter, int savedInputId) {
return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter);
}
protected Collection<Key> readInputKeys(int inputId) throws IOException {
return myInputsIndex != null ? myInputsIndex.get(inputId) : null;
}
protected void saveInputKeys(int inputId, int savedInputId, Map<Key, Value> newData) throws IOException {
if (myInputsIndex != null) {
if (newData.size() > 0) {
myInputsIndex.put(inputId, newData.keySet());
}
else {
myInputsIndex.remove(inputId);
}
}
}
protected void checkValuesHaveProperEqualsAndHashCode(Map<Key, Value> data) {
if (DebugAssertions.DEBUG) {
for (Map.Entry<Key, Value> e : data.entrySet()) {
final Value value = e.getValue();
if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) {
LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter");
}
if (myValueExternalizer != null) {
try {
final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream();
DataOutputStream outputStream = new DataOutputStream(out);
myValueExternalizer.save(outputStream, value);
outputStream.close();
final Value deserializedValue =
myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size())));
if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) {
LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter");
}
}
catch (IOException ex) {
LOG.error(ex);
}
}
}
}
}
public IndexExtension<Key, Value, Input> getExtension() {
return myExtension;
}
public long getModificationStamp() {
return myModificationStamp.get();
}
public class SimpleUpdateData extends UpdateData<Key, Value> {
private final int savedInputId;
private final @NotNull Map<Key, Value> newData;
protected final @NotNull NotNullComputable<Collection<Key>> oldKeysGetter;
public SimpleUpdateData(ID<Key,Value> indexId, int id, @NotNull Map<Key, Value> data, @NotNull NotNullComputable<Collection<Key>> getter) {
super(indexId);
savedInputId = id;
newData = data;
oldKeysGetter = getter;
}
public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor<Key> consumer) throws StorageException {
MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer);
}
public void iterateAddedKeys(final int inputId, final AddedKeyProcessor<Key, Value> consumer) throws StorageException {
MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData);
}
@Override
public void save(int inputId) throws IOException {
saveInputKeys(inputId, savedInputId, newData);
}
public @NotNull Map<Key, Value> getNewData() {
return newData;
}
}
private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>
myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor<Key>() {
@Override
public void process(Key key, int inputId) throws StorageException {
myModificationStamp.incrementAndGet();
myStorage.removeAllValues(key, inputId);
}
};
private final MapDiffUpdateData.AddedKeyProcessor<Key, Value> myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor<Key, Value>() {
@Override
public void process(Key key, Value value, int inputId) throws StorageException {
myModificationStamp.incrementAndGet();
myStorage.addValue(key, inputId, value);
}
};
protected void updateWithMap(final int inputId,
@NotNull UpdateData<Key, Value> updateData) throws StorageException {
getWriteLock().lock();
try {
try {
ValueContainerImpl.ourDebugIndexInfo.set(myIndexId);
updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation);
updateData.iterateAddedKeys(inputId, myAddedKeyProcessor);
updateData.save(inputId);
}
catch (RuntimeException e) {
if (getProcessCanceledExceptionClass().isInstance(e)) {
throw e;
} else {
throw new StorageException(e);
}
}
catch (Throwable e) { // e.g. IOException, AssertionError
throw new StorageException(e);
}
finally {
ValueContainerImpl.ourDebugIndexInfo.set(null);
}
}
finally {
getWriteLock().unlock();
}
}
}
platform/util/src/com/intellij/util/indexing/MemoryIndexStorageBase.java
+223
View File
@@ -0,0 +1,223 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.indexing;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.Processor;
import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.IOException;
import java.util.*;
public abstract class MemoryIndexStorageBase<Key, Value, Storage extends IndexStorage<Key, Value>> implements IndexStorage<Key, Value> {
private final static Logger LOG = Logger.getInstance(MemoryIndexStorageBase.class);
private final Map<Key, ChangeTrackingValueContainer<Value>> myMap = new HashMap<Key, ChangeTrackingValueContainer<Value>>();
@NotNull
protected final Storage myBackendStorage;
private final List<BufferingStateListener> myListeners = ContainerUtil.createLockFreeCopyOnWriteList();
private final ID<?, ?> myIndexId;
private boolean myBufferingEnabled;
public interface BufferingStateListener {
void bufferingStateChanged(boolean newState);
void memoryStorageCleared();
}
public MemoryIndexStorageBase(@NotNull Storage backend, ID<?, ?> indexId) {
myBackendStorage = backend;
myIndexId = indexId;
}
@NotNull
public IndexStorage<Key, Value> getBackendStorage() {
return myBackendStorage;
}
public void addBufferingStateListener(@NotNull BufferingStateListener listener) {
myListeners.add(listener);
}
public void removeBufferingStateListener(@NotNull BufferingStateListener listener) {
myListeners.remove(listener);
}
public void setBufferingEnabled(boolean enabled) {
final boolean wasEnabled = myBufferingEnabled;
assert wasEnabled != enabled;
myBufferingEnabled = enabled;
for (BufferingStateListener listener : myListeners) {
listener.bufferingStateChanged(enabled);
}
}
public boolean isBufferingEnabled() {
return myBufferingEnabled;
}
public void clearMemoryMap() {
myMap.clear();
}
public void fireMemoryStorageCleared() {
for (BufferingStateListener listener : myListeners) {
listener.memoryStorageCleared();
}
}
void clearCaches() {
if (myMap.size() == 0) return;
if (DebugAssertions.DEBUG) {
String message = "Dropping caches for " + (myIndexId != null ? myIndexId:this) + ", number of items:" + myMap.size();
LOG.info(message);
}
for(ChangeTrackingValueContainer<Value> v:myMap.values()) {
v.dropMergedData();
}
}
@Override
public void close() throws StorageException {
myBackendStorage.close();
}
@Override
public void clear() throws StorageException {
clearMemoryMap();
myBackendStorage.clear();
}
@Override
public void flush() throws IOException {
myBackendStorage.flush();
}
@NotNull
@Override
public Collection<Key> getKeys() throws StorageException {
final Set<Key> keys = new HashSet<Key>();
processKeys(new CancelableCollectProcessor<Key>(keys, new Runnable() {
@Override
public void run() {
checkCanceled();
}
}));
return keys;
}
@Override
public boolean processKeys(@NotNull final Processor<Key> processor) throws StorageException {
final Processor<Key> uniqueResultProcessor = processCacheFirstProcessor(processor);
return uniqueResultProcessor != null && myBackendStorage.processKeys(uniqueResultProcessor);
}
@Nullable
protected Processor<Key> processCacheFirstProcessor(final Processor<Key> initialProcessor) {
if (myMap.isEmpty()) return initialProcessor;
final Set<Key> stopList = new HashSet<Key>();
Processor<Key> decoratingProcessor = new Processor<Key>() {
@Override
public boolean process(Key key) {
if (stopList.contains(key)) return true;
final UpdatableValueContainer<Value> container = myMap.get(key);
if (container != null && container.size() == 0) {
return true;
}
return initialProcessor.process(key);
}
};
for (Key key : myMap.keySet()) {
if (!decoratingProcessor.process(key)) {
return null;
}
stopList.add(key);
}
return decoratingProcessor;
}
@Override
public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
if (myBufferingEnabled) {
getMemValueContainer(key).addValue(inputId, value);
return;
}
final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer != null) {
valueContainer.dropMergedData();
}
myBackendStorage.addValue(key, inputId, value);
}
@Override
public void removeAllValues(@NotNull Key key, int inputId) throws StorageException {
if (myBufferingEnabled) {
getMemValueContainer(key).removeAssociatedValue(inputId);
return;
}
final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer != null) {
valueContainer.dropMergedData();
}
myBackendStorage.removeAllValues(key, inputId);
}
private UpdatableValueContainer<Value> getMemValueContainer(final Key key) {
ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer == null) {
valueContainer = new ChangeTrackingValueContainer<Value>(new ChangeTrackingValueContainer.Initializer<Value>() {
@Override
public Object getLock() {
return this;
}
@Override
public ValueContainer<Value> compute() {
try {
return myBackendStorage.read(key);
}
catch (StorageException e) {
throw new RuntimeException(e);
}
}
});
myMap.put(key, valueContainer);
}
return valueContainer;
}
@Override
@NotNull
public ValueContainer<Value> read(final Key key) throws StorageException {
final ValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer != null) {
return valueContainer;
}
return myBackendStorage.read(key);
}
}
platform/lang-impl/src/com/intellij/util/indexing/StorageException.java → platform/util/src/com/intellij/util/indexing/StorageException.java
View File
platform/lang-impl/src/com/intellij/util/indexing/UpdatableValueContainer.java → platform/util/src/com/intellij/util/indexing/UpdatableValueContainer.java
View File
platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java → platform/util/src/com/intellij/util/indexing/UpdateData.java
View File
platform/lang-impl/src/com/intellij/util/indexing/ValueContainer.java → platform/util/src/com/intellij/util/indexing/ValueContainer.java
View File
platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java → platform/util/src/com/intellij/util/indexing/ValueContainerImpl.java
+10 -10
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -79,20 +79,20 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
return myInputIdMapping != null ? myInputIdMapping instanceof THashMap ? ((THashMap)myInputIdMapping).size(): 1 : 0;
}
static final ThreadLocal<ID> ourDebugIndexInfo = new ThreadLocal<>();
static final ThreadLocal<ID> ourDebugIndexInfo = new ThreadLocal<ID>();
@Override
public void removeAssociatedValue(int inputId) {
if (myInputIdMapping == null) return;
List<Object> fileSetObjects = null;
List<Value> valueObjects = null;
for (final ValueIterator<Value> valueIterator = getValueIterator(); valueIterator.hasNext();) {
for (final ValueContainer.ValueIterator<Value> valueIterator = getValueIterator(); valueIterator.hasNext();) {
final Value value = valueIterator.next();
if (valueIterator.getValueAssociationPredicate().contains(inputId)) {
if (fileSetObjects == null) {
fileSetObjects = new SmartList<>();
valueObjects = new SmartList<>();
fileSetObjects = new SmartList<Object>();
valueObjects = new SmartList<Value>();
}
else if (DebugAssertions.DEBUG) {
LOG.error("Expected only one value per-inputId for " + ourDebugIndexInfo.get(), String.valueOf(fileSetObjects.get(0)), String.valueOf(value));
@@ -263,9 +263,9 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
if (myInputIdMapping == null) {
return Collections.emptyList();
} else if (myInputIdMapping instanceof THashMap) {
return new ArrayList<>(((THashMap<Value, Object>)myInputIdMapping).keySet());
return new ArrayList<Value>(((THashMap<Value, Object>)myInputIdMapping).keySet());
} else {
return new SmartList<>((Value)myInputIdMapping);
return new SmartList<Value>((Value)myInputIdMapping);
}
}
@@ -366,11 +366,11 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
@NotNull
public ValueContainerImpl<Value> copy() {
ValueContainerImpl<Value> container = new ValueContainerImpl<>();
ValueContainerImpl<Value> container = new ValueContainerImpl<Value>();
if (myInputIdMapping instanceof THashMap) {
final THashMap<Value, Object> mapping = (THashMap<Value, Object>)myInputIdMapping;
final THashMap<Value, Object> newMapping = new THashMap<>(mapping.size());
final THashMap<Value, Object> newMapping = new THashMap<Value, Object>(mapping.size());
container.myInputIdMapping = newMapping;
mapping.forEachEntry(new TObjectObjectProcedure<Value, Object>() {
@@ -485,7 +485,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
final int inputId = -valueCount;
if (mapping == null && size() > NUMBER_OF_VALUES_THRESHOLD) { // avoid O(NumberOfValues)
mapping = new FileId2ValueMapping<>(this);
mapping = new FileId2ValueMapping<Value>(this);
}
boolean doCompact;
platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java → platform/util/src/com/intellij/util/indexing/ValueContainerMap.java
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,7 +35,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
@NotNull DataExternalizer<Value> valueExternalizer,
boolean keyIsUniqueForIndexedFile
) throws IOException {
super(file, keyKeyDescriptor, new ValueContainerExternalizer<>(valueExternalizer));
super(file, keyKeyDescriptor, new ValueContainerExternalizer<Value>(valueExternalizer));
myValueExternalizer = valueExternalizer;
myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
}
@@ -48,7 +48,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
@Override
protected void doPut(Key key, ValueContainer<Value> container) throws IOException {
synchronized (myEnumerator) {
ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container;
final ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container;
// try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files
// note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular
@@ -83,7 +83,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
@NotNull
@Override
public ValueContainerImpl<T> read(@NotNull final DataInput in) throws IOException {
final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<>();
final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
valueContainer.readFrom((DataInputStream)in, myValueExternalizer);
return valueContainer;
platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java → platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java → platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java → platform/util/src/com/intellij/util/indexing/containers/IdSet.java
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
platform/lang-impl/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java → platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java
View File
platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java → platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
* Copyright 2000-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
platform/lang-impl/src/com/intellij/util/indexing/containers/SortedIdSet.java → platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java
View File
platform/lang-impl/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java → platform/util/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java
View File
platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java
+1 -1
View File
@@ -144,7 +144,7 @@ public class VcsLogFullDetailsIndex<T> implements Disposable {
}
@Override
protected void requestRebuild(@Nullable Exception ex) {
public void requestRebuild(@Nullable Exception ex) {
myFatalErrorHandler.consume(this, ex != null ? ex : new Exception("Index rebuild requested"));
}
}