diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java index 7dc4c304040a..0d8520ac5ad9 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java @@ -25,12 +25,10 @@ import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.search.ProjectAndLibrariesScope; import com.intellij.psi.search.ProjectScopeImpl; import com.intellij.util.Processor; -import com.intellij.util.Processors; import com.intellij.util.SystemProperties; import com.intellij.util.ThrowableRunnable; import com.intellij.util.containers.ConcurrentIntObjectMap; import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.containers.SLRUCache; import com.intellij.util.io.*; import com.intellij.util.io.DataOutputStream; import gnu.trove.TIntHashSet; @@ -39,31 +37,18 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; /** * @author Eugene Zhuravlev * Date: Dec 20, 2007 */ -public final class MapIndexStorage implements IndexStorage{ +public final class MapIndexStorage extends MapIndexStorageBase implements VfsAwareIndexStorage { private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage"); private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true); private final boolean myBuildKeyHashToVirtualFileMapping; - private PersistentMap> myMap; private AppendableStorageBackedByResizableMappedFile myKeyHashToVirtualFileMapping; - private SLRUCache> myCache; private volatile int myLastScannedId; - private final File myBaseStorageFile; - private final KeyDescriptor myKeyDescriptor; - private final int myCacheSize; - private final Lock l = new ReentrantLock(); - private final DataExternalizer myDataExternalizer; - private final boolean myKeyIsUniqueForIndexedFile; private static final ConcurrentIntObjectMap ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap(); public MapIndexStorage(@NotNull File storageFile, @@ -80,85 +65,18 @@ public final class MapIndexStorage implements IndexStorage map; - PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(ourProgressManagerCheckCancelledIOCanceller); - PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE); - try { - map = new ValueContainerMap<>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile); - } finally { - PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null); - PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null); - } - myCache = new SLRUCache>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) { - @Override - @NotNull - public ChangeTrackingValueContainer createValue(final Key key) { - return new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer() { - @NotNull - @Override - public Object getLock() { - return map.getDataAccessLock(); - } - - @Nullable - @Override - public ValueContainer compute() { - ValueContainer value; - try { - value = map.get(key); - if (value == null) { - value = new ValueContainerImpl<>(); - } - } - catch (IOException e) { - throw new RuntimeException(e); - } - return value; - } - }); - } - - @Override - protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer valueContainer) { - if (valueContainer.isDirty()) { - try { - map.put(key, valueContainer); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - } - }; - - myMap = map; - + @Override + protected void initMapAndCache() throws IOException { + super.initMapAndCache(); myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ? new AppendableStorageBackedByResizableMappedFile(getProjectFile(), 4096, null, PagedFileStorage.MB, true) : null; } - @NotNull - private File getStorageFile() { - return new File(myBaseStorageFile.getPath() + ".storage"); - } - @NotNull private File getProjectFile() { return new File(myBaseStorageFile.getPath() + ".project"); @@ -172,14 +90,12 @@ public final class MapIndexStorage implements IndexStorage myKeyHashToVirtualFileMapping.force()); } @@ -191,57 +107,34 @@ public final class MapIndexStorage implements IndexStorage myKeyHashToVirtualFileMapping.close()); } - myMap.close(); - } - catch (IOException e) { - throw new StorageException(e); } catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw new StorageException(cause); - } - if (cause instanceof StorageException) { - throw (StorageException)cause; - } - throw e; + unwrapCauseAndRethrow(e); } } @Override public void clear() throws StorageException{ try { - myMap.close(); if (myKeyHashToVirtualFileMapping != null) { withLock(() -> myKeyHashToVirtualFileMapping.close()); } } - catch (IOException|RuntimeException e) { + catch (RuntimeException e) { LOG.error(e); } try { - IOUtil.deleteAllFilesStartingWith(getStorageFile()); if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile()); - initMapAndCache(); - } - catch (IOException e) { - throw new StorageException(e); } catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw new StorageException(cause); - } - if (cause instanceof StorageException) { - throw (StorageException)cause; - } - throw e; + unwrapCauseAndRethrow(e); } + super.clear(); } @Override @@ -302,20 +195,13 @@ public final class MapIndexStorage implements IndexStorage implements IndexStorage getKeys() throws StorageException { - List keys = new ArrayList<>(); - processKeys(Processors.cancelableCollectProcessor(keys), null, null); - return keys; - } - - @Override - @NotNull - public ChangeTrackingValueContainer read(final Key key) throws StorageException { - l.lock(); - try { - return myCache.get(key); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw new StorageException(cause); - } - if (cause instanceof StorageException) { - throw (StorageException)cause; - } - throw e; - } - finally { - l.unlock(); - } - } - @Override public void addValue(final Key key, final int inputId, final Value value) throws StorageException { try { @@ -447,41 +303,7 @@ public final class MapIndexStorage implements IndexStorage cached; - try { - l.lock(); - cached = myCache.getIfCached(key); - } finally { - l.unlock(); - } - - if (cached != null) { - cached.addValue(inputId, value); - return; - } - // do not pollute the cache with keys unique to indexed file - ChangeTrackingValueContainer valueContainer = new ChangeTrackingValueContainer<>(null); - valueContainer.addValue(inputId, value); - myMap.put(key, valueContainer); - } - catch (IOException e) { - throw new StorageException(e); - } - } - - @Override - public void removeAllValues(@NotNull Key key, int inputId) throws StorageException { - try { - myMap.markDirty(); - // important: assuming the key exists in the index - read(key).removeAssociatedValue(inputId); + super.addValue(key, inputId, value); } catch (IOException e) { throw new StorageException(e); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java index 44a53d3375be..b1b92efe625b 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java @@ -24,7 +24,6 @@ import com.intellij.openapi.editor.Document; import com.intellij.openapi.fileEditor.FileDocumentManager; import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.progress.ProcessCanceledException; -import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.util.*; import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; import com.intellij.openapi.util.io.ByteSequence; @@ -46,72 +45,43 @@ import java.io.*; import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * @author Eugene Zhuravlev * Date: Dec 10, 2007 */ -public class MapReduceIndex implements UpdatableIndex { +public class MapReduceIndex extends MapReduceIndexBase implements UpdatableIndex{ private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex"); private static final int NULL_MAPPING = 0; - @NotNull private final ID myIndexId; - private final DataIndexer myIndexer; - @NotNull protected final IndexStorage myStorage; private final boolean myHasSnapshotMapping; - private final DataExternalizer myValueExternalizer; private final DataExternalizer> mySnapshotIndexExternalizer; private final boolean myIsPsiBackedIndex; - private final IndexExtension myExtension; private final AtomicBoolean myInMemoryMode = new AtomicBoolean(); - private final AtomicLong myModificationStamp = new AtomicLong(); private final TIntObjectHashMap> myInMemoryKeys = new TIntObjectHashMap<>(); private PersistentHashMap myContents; private PersistentHashMap myInputsSnapshotMapping; - @Nullable protected PersistentHashMap> myInputsIndex; private PersistentHashMap myIndexingTrace; - private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); - - private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() { - @Override - public void run() { - try { - if (myStorage instanceof MemoryIndexStorage) { - Lock writeLock = getWriteLock(); - if (writeLock.tryLock()) { - try { - ((MemoryIndexStorage)myStorage).clearCaches(); - } finally { - writeLock.unlock(); - } - } - } - flush(); - } catch (StorageException e) { - LOG.info(e); - requestRebuild(null); - } + static { + if (!DebugAssertions.DEBUG) { + final Application app = ApplicationManager.getApplication(); + DebugAssertions.DEBUG = app.isEAP() || app.isInternal(); } - }); + } public MapReduceIndex(IndexExtension extension, @NotNull IndexStorage storage) throws IOException { - myIndexId = extension.getName(); - myExtension = extension; + super(extension, storage, false); + SharedIndicesData.registerIndex(myIndexId, extension); - myIndexer = extension.getIndexer(); - myStorage = storage; myHasSnapshotMapping = extension instanceof FileBasedIndexExtension && ((FileBasedIndexExtension)extension).hasSnapshotMapping() && IdIndex.ourSnapshotMappingsEnabled; mySnapshotIndexExternalizer = createInputsIndexExternalizer(extension, myIndexId, extension.getKeyDescriptor()); - myValueExternalizer = extension.getValueExternalizer(); myIsPsiBackedIndex = extension instanceof PsiDependentIndex; myContents = createContentsIndex(); // todo @@ -146,21 +116,9 @@ public class MapReduceIndex implements UpdatableIndex DataExternalizer> createInputsIndexExternalizer(IndexExtension extension, - ID indexId, - KeyDescriptor keyDescriptor) { - DataExternalizer> externalizer; - if (extension instanceof CustomInputsIndexFileBasedIndexExtension) { - externalizer = ((CustomInputsIndexFileBasedIndexExtension)extension).createExternalizer(); - } else { - externalizer = new InputIndexDataExternalizer<>(keyDescriptor, indexId); - } - return externalizer; - } - @NotNull - private static PersistentHashMap> createIdToDataKeysIndex(@NotNull IndexExtension extension, - @NotNull MemoryIndexStorage storage) + private PersistentHashMap> createIdToDataKeysIndex(@NotNull IndexExtension extension, + @NotNull MemoryIndexStorageBase storage) throws IOException { ID indexId = extension.getName(); KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); @@ -186,41 +144,19 @@ public class MapReduceIndex implements UpdatableIndex getStorage() { - return myStorage; - } - @Override - public void clear() throws StorageException { - try { - getWriteLock().lock(); - myStorage.clear(); - if (myInputsIndex != null) { - cleanMapping(myInputsIndex); - myInputsIndex = createInputsIndex(); - } - if (myInputsSnapshotMapping != null) { - cleanMapping(myInputsSnapshotMapping); - myInputsSnapshotMapping = createInputSnapshotMapping(); - } - if (myIndexingTrace != null) { - cleanMapping(myIndexingTrace); - myIndexingTrace = createIndexingTrace(); - } - if (myContents != null) { - cleanMapping(myContents); - myContents = createContentsIndex(); - } + protected void cleanMappings() throws IOException { + if (myInputsSnapshotMapping != null) { + cleanMapping(myInputsSnapshotMapping); + myInputsSnapshotMapping = createInputSnapshotMapping(); } - catch (StorageException e) { - LOG.error(e); + if (myIndexingTrace != null) { + cleanMapping(myIndexingTrace); + myIndexingTrace = createIndexingTrace(); } - catch (IOException e) { - LOG.error(e); - } - finally { - getWriteLock().unlock(); + if (myContents != null) { + cleanMapping(myContents); + myContents = createContentsIndex(); } } @@ -265,72 +201,18 @@ public class MapReduceIndex implements UpdatableIndex index) { - final File baseFile = index.getBaseFile(); - try { - index.close(); - } - catch (Throwable ignored) { - } - - IOUtil.deleteAllFilesStartingWith(baseFile); + @Override + protected void doForce() { + doForce(myInputsSnapshotMapping); + doForce(myIndexingTrace); + doForce(myContents); } @Override - public void flush() throws StorageException{ - try { - getReadLock().lock(); - doForce(myInputsIndex); - doForce(myInputsSnapshotMapping); - doForce(myIndexingTrace); - doForce(myContents); - myStorage.flush(); - } - catch (IOException e) { - throw new StorageException(e); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof StorageException || cause instanceof IOException) { - throw new StorageException(cause); - } - else { - throw e; - } - } - finally { - getReadLock().unlock(); - } - } - - private static void doForce(@Nullable PersistentHashMap inputsIndex) { - if (inputsIndex != null && inputsIndex.isDirty()) { - inputsIndex.force(); - } - } - - @Override - public void dispose() { - myLowMemoryFlusher.stop(); - final Lock lock = getWriteLock(); - try { - lock.lock(); - try { - myStorage.close(); - } - finally { - doClose(myInputsIndex); - doClose(myInputsSnapshotMapping); - doClose(myIndexingTrace); - doClose(myContents); - } - } - catch (StorageException e) { - LOG.error(e); - } - finally { - lock.unlock(); - } + protected void doClose() { + doClose(myInputsSnapshotMapping); + doClose(myIndexingTrace); + doClose(myContents); } @Override @@ -348,35 +230,12 @@ public class MapReduceIndex implements UpdatableIndex index) { - if (index != null) { - try { - index.close(); - } - catch (IOException e) { - LOG.error(e); - } - } - } - - @NotNull - @Override - public final Lock getReadLock() { - return myLock.readLock(); - } - - @NotNull - @Override - public final Lock getWriteLock() { - return myLock.writeLock(); - } - @Override public boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, IdFilter idFilter) throws StorageException { final Lock lock = getReadLock(); try { lock.lock(); - return myStorage.processKeys(processor, scope, idFilter); + return ((VfsAwareIndexStorage)myStorage).processKeys(processor, scope, idFilter); } finally { lock.unlock(); @@ -384,22 +243,8 @@ public class MapReduceIndex implements UpdatableIndex getData(@NotNull final Key key) throws StorageException { - final Lock lock = getReadLock(); - try { - lock.lock(); - ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); - return myStorage.read(key); - } - finally { - ValueContainerImpl.ourDebugIndexInfo.set(null); - lock.unlock(); - } - } - protected PersistentHashMap> createInputsIndex() throws IOException { - return createIdToDataKeysIndex(myExtension, (MemoryIndexStorage)myStorage); + return createIdToDataKeysIndex(myExtension, (MemoryIndexStorageBase)myStorage); } private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true); @@ -457,9 +302,7 @@ public class MapReduceIndex implements UpdatableIndex implements UpdatableIndex optimizedUpdateData = null; final NotNullComputable> oldKeysGetter; @@ -559,15 +402,7 @@ public class MapReduceIndex implements UpdatableIndex { - try { - Collection oldKeys = readInputKeys(inputId); - return oldKeys == null? Collections.emptyList() : oldKeys; - } - catch (IOException e) { - throw new RuntimeException(e); - } - }; + oldKeysGetter = createOldKeysGetterByInputIndex(inputId); savedInputId = inputId; } @@ -580,13 +415,7 @@ public class MapReduceIndex implements UpdatableIndex requestRebuild(ex), ModalityState.any()); - } else { - requestRebuild(ex); - } + requestRebuild(ex); return Boolean.FALSE; } @@ -594,15 +423,35 @@ public class MapReduceIndex implements UpdatableIndex { + if (ex == null) { + FileBasedIndex.getInstance().requestRebuild(myIndexId); + } + else { + FileBasedIndex.getInstance().requestRebuild(myIndexId, ex); + } + }; + Application application = ApplicationManager.getApplication(); + if (application.isUnitTestMode() || application.isHeadlessEnvironment()) { + // avoid deadlock due to synchronous update in DumbServiceImpl#queueTask + application.invokeLater(action, ModalityState.any()); + } else { + action.run(); } } + @Override + protected DataExternalizer> createInputsIndexExternalizer(IndexExtension extension, + ID indexId, + KeyDescriptor keyDescriptor) { + return extension instanceof CustomInputsIndexFileBasedIndexExtension ? + ((CustomInputsIndexFileBasedIndexExtension)extension).createExternalizer() : + super.createInputsIndexExternalizer(extension, indexId, keyDescriptor); + } + + @Override protected UpdateData buildUpdateData(Map data, NotNullComputable> oldKeysGetter, int savedInputId) { return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter); } @@ -678,7 +527,8 @@ public class MapReduceIndex implements UpdatableIndex readInputKeys(int inputId) throws IOException { + @Override + protected Collection readInputKeys(int inputId) throws IOException { if (myInMemoryMode.get()) { synchronized (myInMemoryKeys) { Collection keys = myInMemoryKeys.get(inputId); @@ -710,10 +560,11 @@ public class MapReduceIndex implements UpdatableIndex newData) throws IOException { + @Override + protected void saveInputKeys(int inputId, int savedInputId, Map newData) throws IOException { if (myInMemoryMode.get()) { synchronized (myInMemoryKeys) { myInMemoryKeys.put(inputId, newData.keySet()); @@ -722,14 +573,7 @@ public class MapReduceIndex implements UpdatableIndex 0) { - myInputsIndex.put(inputId, newData.keySet()); - } - else { - myInputsIndex.remove(inputId); - } - } + super.saveInputKeys(inputId, savedInputId, newData); if (SharedIndicesData.ourFileSharedIndicesEnabled) { Set newKeys = newData.keySet(); @@ -740,32 +584,6 @@ public class MapReduceIndex implements UpdatableIndex data) { - for(Map.Entry e: data.entrySet()) { - final Value value = e.getValue(); - if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) { - LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter"); - } - - if (myValueExternalizer != null) { - try { - final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(); - DataOutputStream outputStream = new DataOutputStream(out); - myValueExternalizer.save(outputStream, value); - outputStream.close(); - final Value deserializedValue = - myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size()))); - - if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) { - LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter"); - } - } catch (IOException ex) { - LOG.error(ex); - } - } - } - } - private StringBuilder buildDiff(Map data, Map contentData) { StringBuilder moreInfo = new StringBuilder(); if (contentData.size() != data.size()) { @@ -906,84 +724,4 @@ public class MapReduceIndex implements UpdatableIndex ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id"); private static final com.intellij.openapi.util.Key ourSavedUncommittedHashIdKey = com.intellij.openapi.util.Key.create("saved.uncommitted.hash.id"); - - public IndexExtension getExtension() { - return myExtension; - } - - public long getModificationStamp() { - return myModificationStamp.get(); - } - - public class SimpleUpdateData extends UpdateData { - private final int savedInputId; - private final @NotNull Map newData; - protected final @NotNull NotNullComputable> oldKeysGetter; - - public SimpleUpdateData(ID indexId, int id, @NotNull Map data, @NotNull NotNullComputable> getter) { - super(indexId); - savedInputId = id; - newData = data; - oldKeysGetter = getter; - } - - public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor consumer) throws StorageException { - MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer); - } - - public void iterateAddedKeys(final int inputId, final AddedKeyProcessor consumer) throws StorageException { - MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData); - } - - @Override - public void save(int inputId) throws IOException { - saveInputKeys(inputId, savedInputId, newData); - } - - public @NotNull Map getNewData() { - return newData; - } - } - - private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor - myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor() { - @Override - public void process(Key key, int inputId) throws StorageException { - myModificationStamp.incrementAndGet(); - myStorage.removeAllValues(key, inputId); - } - }; - - private final MapDiffUpdateData.AddedKeyProcessor myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor() { - @Override - public void process(Key key, Value value, int inputId) throws StorageException { - myModificationStamp.incrementAndGet(); - myStorage.addValue(key, inputId, value); - } - }; - - protected void updateWithMap(final int inputId, - @NotNull UpdateData updateData) throws StorageException { - getWriteLock().lock(); - try { - try { - ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); - updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation); - updateData.iterateAddedKeys(inputId, myAddedKeyProcessor); - updateData.save(inputId); - } - catch (ProcessCanceledException pce) { - throw pce; // extra care - } - catch (Throwable e) { // e.g. IOException, AssertionError - throw new StorageException(e); - } - finally { - ValueContainerImpl.ourDebugIndexInfo.set(null); - } - } - finally { - getWriteLock().unlock(); - } - } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java index 858d93c09a4a..cf54797c31ba 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java @@ -18,199 +18,22 @@ package com.intellij.util.indexing; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; -import com.intellij.util.Processors; -import com.intellij.util.containers.ContainerUtil; import org.jetbrains.annotations.NotNull; -import java.io.IOException; -import java.util.*; - /** * This storage is needed for indexing yet unsaved data without saving those changes to 'main' backend storage * * @author Eugene Zhuravlev * Date: Dec 10, 2007 */ -public class MemoryIndexStorage implements IndexStorage { - private final Map> myMap = new HashMap<>(); - @NotNull - private final IndexStorage myBackendStorage; - private final List myListeners = ContainerUtil.createLockFreeCopyOnWriteList(); - private final ID myIndexId; - private boolean myBufferingEnabled; - - public interface BufferingStateListener { - void bufferingStateChanged(boolean newState); - - void memoryStorageCleared(); - } - - public MemoryIndexStorage(@NotNull IndexStorage backend) { - this(backend, null); - } - - public MemoryIndexStorage(@NotNull IndexStorage backend, ID indexId) { - myBackendStorage = backend; - myIndexId = indexId; - } - - @NotNull - public IndexStorage getBackendStorage() { - return myBackendStorage; - } - - public void addBufferingStateListener(@NotNull BufferingStateListener listener) { - myListeners.add(listener); - } - - public void removeBufferingStateListener(@NotNull BufferingStateListener listener) { - myListeners.remove(listener); - } - - public void setBufferingEnabled(boolean enabled) { - final boolean wasEnabled = myBufferingEnabled; - assert wasEnabled != enabled; - - myBufferingEnabled = enabled; - for (BufferingStateListener listener : myListeners) { - listener.bufferingStateChanged(enabled); - } - } - - public boolean isBufferingEnabled() { - return myBufferingEnabled; - } - - public void clearMemoryMap() { - myMap.clear(); - } - - public void fireMemoryStorageCleared() { - for (BufferingStateListener listener : myListeners) { - listener.memoryStorageCleared(); - } - } - - void clearCaches() { - if (myMap.size() == 0) return; - - if (DebugAssertions.DEBUG) { - String message = "Dropping caches for " + (myIndexId != null ? myIndexId:this) + ", number of items:" + myMap.size(); - FileBasedIndexImpl.LOG.info(message); - } - - for(ChangeTrackingValueContainer v:myMap.values()) { - v.dropMergedData(); - } - } - - @Override - public void close() throws StorageException { - myBackendStorage.close(); - } - - @Override - public void clear() throws StorageException { - clearMemoryMap(); - myBackendStorage.clear(); - } - - @Override - public void flush() throws IOException { - myBackendStorage.flush(); - } - - @NotNull - @Override - public Collection getKeys() throws StorageException { - final Set keys = new HashSet<>(); - processKeys(Processors.cancelableCollectProcessor(keys), null, null); - return keys; +public class MemoryIndexStorage extends MemoryIndexStorageBase> implements VfsAwareIndexStorage { + public MemoryIndexStorage(@NotNull VfsAwareIndexStorage backend, ID indexId) { + super(backend, indexId); } @Override public boolean processKeys(@NotNull final Processor processor, GlobalSearchScope scope, IdFilter idFilter) throws StorageException { - final Set stopList = new HashSet<>(); - - Processor decoratingProcessor = key -> { - if (stopList.contains(key)) return true; - - final UpdatableValueContainer container = myMap.get(key); - if (container != null && container.size() == 0) { - return true; - } - return processor.process(key); - }; - - for (Key key : myMap.keySet()) { - if (!decoratingProcessor.process(key)) { - return false; - } - stopList.add(key); - } - return myBackendStorage.processKeys(stopList.isEmpty() && myMap.isEmpty() ? processor : decoratingProcessor, scope, idFilter); - } - - @Override - public void addValue(final Key key, final int inputId, final Value value) throws StorageException { - if (myBufferingEnabled) { - getMemValueContainer(key).addValue(inputId, value); - return; - } - final ChangeTrackingValueContainer valueContainer = myMap.get(key); - if (valueContainer != null) { - valueContainer.dropMergedData(); - } - - myBackendStorage.addValue(key, inputId, value); - } - - @Override - public void removeAllValues(@NotNull Key key, int inputId) throws StorageException { - if (myBufferingEnabled) { - getMemValueContainer(key).removeAssociatedValue(inputId); - return; - } - final ChangeTrackingValueContainer valueContainer = myMap.get(key); - if (valueContainer != null) { - valueContainer.dropMergedData(); - } - - myBackendStorage.removeAllValues(key, inputId); - } - - private UpdatableValueContainer getMemValueContainer(final Key key) { - ChangeTrackingValueContainer valueContainer = myMap.get(key); - if (valueContainer == null) { - valueContainer = new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer() { - @Override - public Object getLock() { - return this; - } - - @Override - public ValueContainer compute() { - try { - return myBackendStorage.read(key); - } - catch (StorageException e) { - throw new RuntimeException(e); - } - } - }); - myMap.put(key, valueContainer); - } - return valueContainer; - } - - @Override - @NotNull - public ValueContainer read(final Key key) throws StorageException { - final ValueContainer valueContainer = myMap.get(key); - if (valueContainer != null) { - return valueContainer; - } - - return myBackendStorage.read(key); + final Processor uniqueResultProcessor = processCacheFirstProcessor(processor); + return uniqueResultProcessor != null && myBackendStorage.processKeys(uniqueResultProcessor, scope, idFilter); } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java index 3d8c5fb15d9b..19facea95a81 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java @@ -16,39 +16,33 @@ package com.intellij.util.indexing; -import com.intellij.openapi.util.Computable; +import com.intellij.openapi.progress.ProcessCanceledException; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.util.Processor; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.concurrent.locks.Lock; - /** * @author Eugene Zhuravlev * Date: Dec 10, 2007 */ -public interface UpdatableIndex extends AbstractIndex { - - void clear() throws StorageException; - - void flush() throws StorageException; - - /** - * @param inputId *positive* id of content. - */ - @NotNull - Computable update(int inputId, @Nullable Input content); - - @NotNull - Lock getReadLock(); - - @NotNull - Lock getWriteLock(); - - void dispose(); +public interface UpdatableIndex extends AbstractUpdatableIndex { + boolean processAllKeys(@NotNull Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException; void setIndexedStateForFile(int fileId, @NotNull VirtualFile file); void resetIndexedStateForFile(int fileId); boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file); + + @Override + default boolean processAllKeys(@NotNull Processor processor) throws StorageException { + return processAllKeys(processor, null, null); + } + + @NotNull + @Override + default Class getProcessCanceledExceptionClass() { + return ProcessCanceledException.class; + } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java similarity index 65% rename from platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java rename to platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java index 34b7937cb7a1..f6209fc03bb2 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,21 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.intellij.util.indexing; +import com.intellij.openapi.progress.ProgressManager; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -/** - * @author Eugene Zhuravlev - * Date: Dec 24, 2007 - */ -public interface AbstractIndex { - @NotNull - ValueContainer getData(@NotNull Key key) throws StorageException; +public interface VfsAwareIndexStorage extends IndexStorage { - boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException; + boolean processKeys(@NotNull Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException; + + @Override + default void checkCanceled() { + ProgressManager.checkCanceled(); + } } diff --git a/platform/util/src/com/intellij/util/indexing/AbstractUpdatableIndex.java b/platform/util/src/com/intellij/util/indexing/AbstractUpdatableIndex.java new file mode 100644 index 000000000000..91aec4ba4a9b --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/AbstractUpdatableIndex.java @@ -0,0 +1,54 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intellij.util.indexing; + +import com.intellij.openapi.util.Computable; +import com.intellij.util.Processor; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.concurrent.locks.Lock; + +public interface AbstractUpdatableIndex { + void clear() throws StorageException; + + void flush() throws StorageException; + + /** + * @param inputId *positive* id of content. + */ + @NotNull + Computable update(int inputId, @Nullable Input content); + + @NotNull + ValueContainer getData(@NotNull Key key) throws StorageException; + + boolean processAllKeys(@NotNull Processor processor) throws StorageException; + + @NotNull + Lock getReadLock(); + + @NotNull + Lock getWriteLock(); + + void dispose(); + + void requestRebuild(@Nullable Exception ex); + + @NotNull + Class getProcessCanceledExceptionClass(); +} diff --git a/platform/util/src/com/intellij/util/indexing/CancelableCollectProcessor.java b/platform/util/src/com/intellij/util/indexing/CancelableCollectProcessor.java new file mode 100644 index 000000000000..326cdd99018c --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/CancelableCollectProcessor.java @@ -0,0 +1,36 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.util.CommonProcessors; +import org.jetbrains.annotations.NotNull; + +import java.util.Collection; + +class CancelableCollectProcessor extends CommonProcessors.CollectProcessor { + private final Runnable myCancelChecker; + + public CancelableCollectProcessor(@NotNull Collection collection, @NotNull Runnable cancelChecker) { + super(collection); + myCancelChecker = cancelChecker; + } + + @Override + public boolean process(T o) { + myCancelChecker.run(); + return super.process(o); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java b/platform/util/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java similarity index 97% rename from platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java rename to platform/util/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java index 467ba0cf3202..2158a10f23ea 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java +++ b/platform/util/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer merged.addValue(inputId, value); } - if (myAdded == null) myAdded = new ValueContainerImpl<>(); + if (myAdded == null) myAdded = new ValueContainerImpl(); myAdded.addValue(inputId, value); } @@ -131,7 +131,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer (newMerged.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD || (myAdded != null && myAdded.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD))) { // Calculate file ids that have Value mapped to avoid O(NumberOfValuesInMerged) during removal - fileId2ValueMapping = new FileId2ValueMapping<>(newMerged); + fileId2ValueMapping = new FileId2ValueMapping(newMerged); } final FileId2ValueMapping finalFileId2ValueMapping = fileId2ValueMapping; if (myInvalidated != null) { diff --git a/platform/indexing-api/src/com/intellij/util/indexing/DataIndexer.java b/platform/util/src/com/intellij/util/indexing/DataIndexer.java similarity index 100% rename from platform/indexing-api/src/com/intellij/util/indexing/DataIndexer.java rename to platform/util/src/com/intellij/util/indexing/DataIndexer.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java b/platform/util/src/com/intellij/util/indexing/DebugAssertions.java similarity index 80% rename from platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java rename to platform/util/src/com/intellij/util/indexing/DebugAssertions.java index e5b124880032..428464fb8436 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java +++ b/platform/util/src/com/intellij/util/indexing/DebugAssertions.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ */ package com.intellij.util.indexing; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; import com.intellij.util.SystemProperties; import com.intellij.util.containers.hash.LinkedHashMap; @@ -27,9 +26,10 @@ import java.util.Formatter; public class DebugAssertions { private static final Logger LOG = Logger.getInstance(DebugAssertions.class); - public static final boolean DEBUG = SystemProperties.getBooleanProperty( + @SuppressWarnings("StaticNonFinalField") + public static volatile boolean DEBUG = SystemProperties.getBooleanProperty( "intellij.idea.indices.debug", - ApplicationManager.getApplication().isInternal() || ApplicationManager.getApplication().isEAP() + false ); public static final boolean EXTRA_SANITY_CHECKS = SystemProperties.getBooleanProperty( @@ -56,9 +56,9 @@ public class DebugAssertions { static boolean equals(Collection keys, Collection keys2, KeyDescriptor keyDescriptor) { if (keys == null && keys2 == null) return true; if (keys == null || keys2 == null || keys.size() != keys2.size()) return false; - LinkedHashMap map = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor); + LinkedHashMap map = new LinkedHashMap(keys.size(), 0.8f, keyDescriptor); for(Key key:keys) map.put(key, Boolean.TRUE); - LinkedHashMap map2 = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor); + LinkedHashMap map2 = new LinkedHashMap(keys.size(), 0.8f, keyDescriptor); for(Key key:keys2) map2.put(key, Boolean.TRUE); return map.equals(map2); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java b/platform/util/src/com/intellij/util/indexing/FileId2ValueMapping.java similarity index 95% rename from platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java rename to platform/util/src/com/intellij/util/indexing/FileId2ValueMapping.java index 5a6b3357bc99..3527503b464d 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java +++ b/platform/util/src/com/intellij/util/indexing/FileId2ValueMapping.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ class FileId2ValueMapping { private boolean myOnePerFileValidationEnabled = true; FileId2ValueMapping(ValueContainerImpl _valueContainer) { - id2ValueMap = new TIntObjectHashMap<>(); + id2ValueMap = new TIntObjectHashMap(); valueContainer = _valueContainer; TIntArrayList removedFileIdList = null; @@ -45,7 +45,7 @@ class FileId2ValueMapping { if (previousValue != null) { // delay removal of duplicated id -> value mapping since it will affect valueIterator we are using if (removedFileIdList == null) { removedFileIdList = new TIntArrayList(); - removedValueList = new SmartList<>(); + removedValueList = new SmartList(); } removedFileIdList.add(id); removedValueList.add(previousValue); diff --git a/platform/core-api/src/com/intellij/util/indexing/ID.java b/platform/util/src/com/intellij/util/indexing/ID.java similarity index 100% rename from platform/core-api/src/com/intellij/util/indexing/ID.java rename to platform/util/src/com/intellij/util/indexing/ID.java diff --git a/platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java b/platform/util/src/com/intellij/util/indexing/IndexExtension.java similarity index 100% rename from platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java rename to platform/util/src/com/intellij/util/indexing/IndexExtension.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java b/platform/util/src/com/intellij/util/indexing/IndexStorage.java similarity index 83% rename from platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java rename to platform/util/src/com/intellij/util/indexing/IndexStorage.java index 1020876ca2e4..74d1cb845118 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java +++ b/platform/util/src/com/intellij/util/indexing/IndexStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,8 @@ package com.intellij.util.indexing; -import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.Flushable; import java.io.IOException; @@ -40,7 +38,7 @@ public interface IndexStorage extends Flushable { @NotNull ValueContainer read(Key key) throws StorageException; - boolean processKeys(@NotNull Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException; + boolean processKeys(@NotNull Processor processor) throws StorageException; @NotNull Collection getKeys() throws StorageException; @@ -49,4 +47,6 @@ public interface IndexStorage extends Flushable { @Override void flush() throws IOException; + + void checkCanceled(); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java b/platform/util/src/com/intellij/util/indexing/InputIndexDataExternalizer.java similarity index 95% rename from platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java rename to platform/util/src/com/intellij/util/indexing/InputIndexDataExternalizer.java index b33074a2433a..2f5686bd3852 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java +++ b/platform/util/src/com/intellij/util/indexing/InputIndexDataExternalizer.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ public class InputIndexDataExternalizer implements DataExternalizer read(@NotNull DataInput in) throws IOException { try { final int size = DataInputOutputUtil.readINT(in); - final List list = new ArrayList<>(size); + final List list = new ArrayList(size); for (int idx = 0; idx < size; idx++) { list.add(myKeyDescriptor.read(in)); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java b/platform/util/src/com/intellij/util/indexing/MapDiffUpdateData.java similarity index 93% rename from platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java rename to platform/util/src/com/intellij/util/indexing/MapDiffUpdateData.java index e4dcbfddfe0b..fd0f2f472daa 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java +++ b/platform/util/src/com/intellij/util/indexing/MapDiffUpdateData.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2015 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ */ package com.intellij.util.indexing; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Comparing; import com.intellij.openapi.util.Ref; @@ -44,7 +43,7 @@ public abstract class MapDiffUpdateData extends UpdateData data) throws StorageException { if (data instanceof THashMap) { // such map often (from IdIndex) contain 100x (avg ~240) of entries, also THashMap have no Entry inside so we optimize for gc too - final Ref exceptionRef = new Ref<>(); + final Ref exceptionRef = new Ref(); final boolean b = ((THashMap)data).forEachEntry(new TObjectObjectProcedure() { @Override public boolean execute(Key key, Value value) { @@ -81,8 +80,6 @@ public abstract class MapDiffUpdateData extends UpdateData extends UpdateData(); + if (removedOrChangedKeys == null) removedOrChangedKeys = new THashMap(); removedOrChangedKeys.put(e.getKey(), e.getValue()); if (newValue.containsKey(e.getKey())) { - if (addedKeys == null) addedKeys = new THashMap<>(); + if (addedKeys == null) addedKeys = new THashMap(); addedKeys.put(e.getKey(), newValueForKey); } } @@ -130,7 +127,7 @@ public abstract class MapDiffUpdateData extends UpdateData e : newValue.entrySet()) { if (!currentValue.containsKey(e.getKey())) { - if (addedKeys == null) addedKeys = new THashMap<>(); + if (addedKeys == null) addedKeys = new THashMap(); addedKeys.put(e.getKey(), e.getValue()); } } @@ -145,7 +142,7 @@ public abstract class MapDiffUpdateData extends UpdateData" + incrementalRemovals + ", additions:" +totalAdditions + "->" +incrementalAdditions); diff --git a/platform/util/src/com/intellij/util/indexing/MapIndexStorageBase.java b/platform/util/src/com/intellij/util/indexing/MapIndexStorageBase.java new file mode 100644 index 000000000000..3d61e0661e2c --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/MapIndexStorageBase.java @@ -0,0 +1,271 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.util.Processor; +import com.intellij.util.containers.SLRUCache; +import com.intellij.util.io.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public abstract class MapIndexStorageBase implements IndexStorage { + private static final Logger LOG = Logger.getInstance(MapIndexStorageBase.class); + protected PersistentMap> myMap; + protected SLRUCache> myCache; + protected final File myBaseStorageFile; + protected final KeyDescriptor myKeyDescriptor; + private final int myCacheSize; + + protected final Lock l = new ReentrantLock(); + private final DataExternalizer myDataExternalizer; + private final boolean myKeyIsUniqueForIndexedFile; + + protected MapIndexStorageBase(@NotNull File storageFile, + @NotNull KeyDescriptor keyDescriptor, + @NotNull DataExternalizer valueExternalizer, + final int cacheSize, + boolean keyIsUniqueForIndexedFile) throws IOException { + myBaseStorageFile = storageFile; + myKeyDescriptor = keyDescriptor; + myCacheSize = cacheSize; + myDataExternalizer = valueExternalizer; + myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile; + } + + protected void initMapAndCache() throws IOException { + final ValueContainerMap map; + PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set( + new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() { + @Override + public void checkCancellation() { + checkCanceled(); + } + }); + PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE); + try { + map = new ValueContainerMap(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile); + } finally { + PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null); + PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null); + } + myCache = new SLRUCache>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) { + @Override + @NotNull + public ChangeTrackingValueContainer createValue(final Key key) { + return new ChangeTrackingValueContainer(new ChangeTrackingValueContainer.Initializer() { + @NotNull + @Override + public Object getLock() { + return map.getDataAccessLock(); + } + + @Nullable + @Override + public ValueContainer compute() { + ValueContainer value; + try { + value = map.get(key); + if (value == null) { + value = new ValueContainerImpl(); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + return value; + } + }); + } + + @Override + protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer valueContainer) { + if (valueContainer.isDirty()) { + try { + map.put(key, valueContainer); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + } + }; + + myMap = map; + } + + @NotNull + private File getStorageFile() { + return new File(myBaseStorageFile.getPath() + ".storage"); + } + + @Override + public boolean processKeys(@NotNull Processor processor) throws StorageException { + l.lock(); + try { + myCache.clear(); // this will ensure that all new keys are made into the map + return myMap.processKeys(processor); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + return unwrapCauseAndRethrow(e); + } + finally { + l.unlock(); + } + } + + @Override + public void flush() { + l.lock(); + try { + if (!myMap.isClosed()) { + myCache.clear(); + if (myMap.isDirty()) myMap.force(); + } + } + finally { + l.unlock(); + } + } + + @Override + public void close() throws StorageException { + try { + flush(); + myMap.close(); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + unwrapCauseAndRethrow(e); + } + } + + @Override + public void clear() throws StorageException{ + try { + myMap.close(); + } + catch (IOException e) { + LOG.error(e); + } + catch (RuntimeException e) { + LOG.error(e); + } + try { + IOUtil.deleteAllFilesStartingWith(getStorageFile()); + initMapAndCache(); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + unwrapCauseAndRethrow(e); + } + } + + @NotNull + @Override + public Collection getKeys() throws StorageException { + List keys = new ArrayList(); + processKeys(new CancelableCollectProcessor(keys, new Runnable() { + @Override + public void run() {checkCanceled();} + })); + return keys; + } + + @Override + @NotNull + public ChangeTrackingValueContainer read(final Key key) throws StorageException { + l.lock(); + try { + return myCache.get(key); + } + catch (RuntimeException e) { + return unwrapCauseAndRethrow(e); + } + finally { + l.unlock(); + } + } + + @Override + public void addValue(final Key key, final int inputId, final Value value) throws StorageException { + try { + myMap.markDirty(); + if (!myKeyIsUniqueForIndexedFile) { + read(key).addValue(inputId, value); + return; + } + + ChangeTrackingValueContainer cached; + try { + l.lock(); + cached = myCache.getIfCached(key); + } finally { + l.unlock(); + } + + if (cached != null) { + cached.addValue(inputId, value); + return; + } + // do not pollute the cache with keys unique to indexed file + ChangeTrackingValueContainer valueContainer = new ChangeTrackingValueContainer(null); + valueContainer.addValue(inputId, value); + myMap.put(key, valueContainer); + } + catch (IOException e) { + throw new StorageException(e); + } + } + + @Override + public void removeAllValues(@NotNull Key key, int inputId) throws StorageException { + try { + myMap.markDirty(); + // important: assuming the key exists in the index + read(key).removeAssociatedValue(inputId); + } + catch (IOException e) { + throw new StorageException(e); + } + } + + protected static T unwrapCauseAndRethrow(RuntimeException e) throws StorageException { + final Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw new StorageException(cause); + } + if (cause instanceof StorageException) { + throw (StorageException)cause; + } + throw e; + } +} diff --git a/platform/util/src/com/intellij/util/indexing/MapReduceIndexBase.java b/platform/util/src/com/intellij/util/indexing/MapReduceIndexBase.java new file mode 100644 index 000000000000..8abc28886b38 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/MapReduceIndexBase.java @@ -0,0 +1,441 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Comparing; +import com.intellij.openapi.util.Computable; +import com.intellij.openapi.util.LowMemoryWatcher; +import com.intellij.openapi.util.NotNullComputable; +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; +import com.intellij.util.Processor; +import com.intellij.util.io.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public abstract class MapReduceIndexBase implements AbstractUpdatableIndex { + private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex"); + @Nullable protected final ID myIndexId; + protected final DataIndexer myIndexer; + @NotNull protected final IndexStorage myStorage; + + protected final DataExternalizer myValueExternalizer; + protected final IndexExtension myExtension; + private final AtomicLong myModificationStamp = new AtomicLong(); + + @Nullable protected PersistentHashMap> myInputsIndex; + + private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); + + private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() { + @Override + public void run() { + try { + if (myStorage instanceof MemoryIndexStorageBase) { + Lock writeLock = getWriteLock(); + if (writeLock.tryLock()) { + try { + ((MemoryIndexStorageBase)myStorage).clearCaches(); + } finally { + writeLock.unlock(); + } + } + } + flush(); + } catch (StorageException e) { + LOG.info(e); + requestRebuild(e); + } + } + }); + + protected MapReduceIndexBase(IndexExtension extension, + @NotNull IndexStorage storage, + boolean createInputIndex) throws IOException { + myIndexId = extension.getName(); + myExtension = extension; + myIndexer = extension.getIndexer(); + myStorage = storage; + myValueExternalizer = extension.getValueExternalizer(); + + if (createInputIndex) { + myInputsIndex = createInputsIndex(); + } + + } + + protected PersistentHashMap> createInputsIndex() throws IOException { + return null; + } + + protected DataExternalizer> createInputsIndexExternalizer(IndexExtension extension, + ID indexId, + KeyDescriptor keyDescriptor) { + DataExternalizer> externalizer; + externalizer = new InputIndexDataExternalizer(keyDescriptor, indexId); + return externalizer; + } + + @NotNull + public IndexStorage getStorage() { + return myStorage; + } + + @Override + public void clear() throws StorageException { + try { + getWriteLock().lock(); + myStorage.clear(); + if (myInputsIndex != null) { + cleanMapping(myInputsIndex); + myInputsIndex = createInputsIndex(); + } + cleanMappings(); + } + catch (StorageException e) { + LOG.error(e); + } + catch (IOException e) { + LOG.error(e); + } + finally { + getWriteLock().unlock(); + } + } + + protected void cleanMappings() throws IOException { + + } + + protected static void cleanMapping(@NotNull PersistentHashMap index) { + final File baseFile = index.getBaseFile(); + try { + index.close(); + } + catch (Throwable ignored) { + } + + IOUtil.deleteAllFilesStartingWith(baseFile); + } + + @Override + public void flush() throws StorageException{ + try { + getReadLock().lock(); + doForce(myInputsIndex); + doForce(); + myStorage.flush(); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof StorageException || cause instanceof IOException) { + throw new StorageException(cause); + } + else { + throw e; + } + } + finally { + getReadLock().unlock(); + } + } + + protected void doForce() { + } + + protected static void doForce(PersistentHashMap inputsIndex) { + if (inputsIndex != null && inputsIndex.isDirty()) { + inputsIndex.force(); + } + } + + @Override + public void dispose() { + myLowMemoryFlusher.stop(); + final Lock lock = getWriteLock(); + try { + lock.lock(); + try { + myStorage.close(); + } + finally { + doClose(myInputsIndex); + doClose(); + } + } + catch (StorageException e) { + LOG.error(e); + } + finally { + lock.unlock(); + } + } + + protected void doClose() { + } + + protected static void doClose(PersistentHashMap index) { + if (index != null) { + try { + index.close(); + } + catch (IOException e) { + LOG.error(e); + } + } + } + + @NotNull + @Override + public final Lock getReadLock() { + return myLock.readLock(); + } + + @NotNull + @Override + public final Lock getWriteLock() { + return myLock.writeLock(); + } + + @Override + @NotNull + public ValueContainer getData(@NotNull final Key key) throws StorageException { + final Lock lock = getReadLock(); + try { + lock.lock(); + ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); + return myStorage.read(key); + } + finally { + ValueContainerImpl.ourDebugIndexInfo.set(null); + lock.unlock(); + } + } + + @Override + public boolean processAllKeys(@NotNull Processor processor) throws StorageException { + final Lock lock = getReadLock(); + try { + lock.lock(); + return myStorage.processKeys(processor); + } + finally { + lock.unlock(); + } + } + + @NotNull + @Override + public Computable update(final int inputId, @Nullable Input content) { + Map data = content != null ? myIndexer.map(content) : Collections.emptyMap(); + + checkValuesHaveProperEqualsAndHashCode(data); + + myStorage.checkCanceled(); + + // do not depend on content! + final UpdateData updateData = buildUpdateData(data, + createOldKeysGetterByInputIndex(inputId), + inputId); + return new Computable() { + @Override + public Boolean compute() { + try { + updateWithMap(inputId, updateData); + } + catch (StorageException ex) { + LOG.info("Exception during updateWithMap:" + ex); + requestRebuild(ex); + return Boolean.FALSE; + } + catch (RuntimeException ex) { + if (getProcessCanceledExceptionClass().isInstance(ex)) { + LOG.info("Exception during updateWithMap:" + ex); + requestRebuild(ex); + return Boolean.FALSE; + } else { + throw ex; + } + } + + return Boolean.TRUE; + } + }; + } + + @NotNull + protected NotNullComputable> createOldKeysGetterByInputIndex(final int inputId) { + return new NotNullComputable>() { + @NotNull + @Override + public Collection compute() { + try { + Collection oldKeys = MapReduceIndexBase.this.readInputKeys(inputId); + return oldKeys == null ? Collections.emptyList() : oldKeys; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + } + + protected UpdateData buildUpdateData(Map data, NotNullComputable> oldKeysGetter, int savedInputId) { + return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter); + } + + protected Collection readInputKeys(int inputId) throws IOException { + return myInputsIndex != null ? myInputsIndex.get(inputId) : null; + } + + protected void saveInputKeys(int inputId, int savedInputId, Map newData) throws IOException { + if (myInputsIndex != null) { + if (newData.size() > 0) { + myInputsIndex.put(inputId, newData.keySet()); + } + else { + myInputsIndex.remove(inputId); + } + } + } + + protected void checkValuesHaveProperEqualsAndHashCode(Map data) { + if (DebugAssertions.DEBUG) { + for (Map.Entry e : data.entrySet()) { + final Value value = e.getValue(); + if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) { + LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter"); + } + + if (myValueExternalizer != null) { + try { + final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(out); + myValueExternalizer.save(outputStream, value); + outputStream.close(); + final Value deserializedValue = + myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size()))); + + if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) { + LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter"); + } + } + catch (IOException ex) { + LOG.error(ex); + } + } + } + } + } + + public IndexExtension getExtension() { + return myExtension; + } + + public long getModificationStamp() { + return myModificationStamp.get(); + } + + public class SimpleUpdateData extends UpdateData { + private final int savedInputId; + private final @NotNull Map newData; + protected final @NotNull NotNullComputable> oldKeysGetter; + + public SimpleUpdateData(ID indexId, int id, @NotNull Map data, @NotNull NotNullComputable> getter) { + super(indexId); + savedInputId = id; + newData = data; + oldKeysGetter = getter; + } + + public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor consumer) throws StorageException { + MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer); + } + + public void iterateAddedKeys(final int inputId, final AddedKeyProcessor consumer) throws StorageException { + MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData); + } + + @Override + public void save(int inputId) throws IOException { + saveInputKeys(inputId, savedInputId, newData); + } + + public @NotNull Map getNewData() { + return newData; + } + } + + private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor + myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor() { + @Override + public void process(Key key, int inputId) throws StorageException { + myModificationStamp.incrementAndGet(); + myStorage.removeAllValues(key, inputId); + } + }; + + private final MapDiffUpdateData.AddedKeyProcessor myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor() { + @Override + public void process(Key key, Value value, int inputId) throws StorageException { + myModificationStamp.incrementAndGet(); + myStorage.addValue(key, inputId, value); + } + }; + + protected void updateWithMap(final int inputId, + @NotNull UpdateData updateData) throws StorageException { + getWriteLock().lock(); + try { + try { + ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); + updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation); + updateData.iterateAddedKeys(inputId, myAddedKeyProcessor); + updateData.save(inputId); + } + catch (RuntimeException e) { + if (getProcessCanceledExceptionClass().isInstance(e)) { + throw e; + } else { + throw new StorageException(e); + } + } + catch (Throwable e) { // e.g. IOException, AssertionError + throw new StorageException(e); + } + finally { + ValueContainerImpl.ourDebugIndexInfo.set(null); + } + } + finally { + getWriteLock().unlock(); + } + } +} + diff --git a/platform/util/src/com/intellij/util/indexing/MemoryIndexStorageBase.java b/platform/util/src/com/intellij/util/indexing/MemoryIndexStorageBase.java new file mode 100644 index 000000000000..21a756ef2a86 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/MemoryIndexStorageBase.java @@ -0,0 +1,223 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.util.Processor; +import com.intellij.util.containers.ContainerUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.*; + +public abstract class MemoryIndexStorageBase> implements IndexStorage { + private final static Logger LOG = Logger.getInstance(MemoryIndexStorageBase.class); + + private final Map> myMap = new HashMap>(); + @NotNull + protected final Storage myBackendStorage; + private final List myListeners = ContainerUtil.createLockFreeCopyOnWriteList(); + private final ID myIndexId; + private boolean myBufferingEnabled; + + public interface BufferingStateListener { + void bufferingStateChanged(boolean newState); + + void memoryStorageCleared(); + } + + public MemoryIndexStorageBase(@NotNull Storage backend, ID indexId) { + myBackendStorage = backend; + myIndexId = indexId; + } + + @NotNull + public IndexStorage getBackendStorage() { + return myBackendStorage; + } + + public void addBufferingStateListener(@NotNull BufferingStateListener listener) { + myListeners.add(listener); + } + + public void removeBufferingStateListener(@NotNull BufferingStateListener listener) { + myListeners.remove(listener); + } + + public void setBufferingEnabled(boolean enabled) { + final boolean wasEnabled = myBufferingEnabled; + assert wasEnabled != enabled; + + myBufferingEnabled = enabled; + for (BufferingStateListener listener : myListeners) { + listener.bufferingStateChanged(enabled); + } + } + + public boolean isBufferingEnabled() { + return myBufferingEnabled; + } + + public void clearMemoryMap() { + myMap.clear(); + } + + public void fireMemoryStorageCleared() { + for (BufferingStateListener listener : myListeners) { + listener.memoryStorageCleared(); + } + } + + void clearCaches() { + if (myMap.size() == 0) return; + + if (DebugAssertions.DEBUG) { + String message = "Dropping caches for " + (myIndexId != null ? myIndexId:this) + ", number of items:" + myMap.size(); + LOG.info(message); + } + + for(ChangeTrackingValueContainer v:myMap.values()) { + v.dropMergedData(); + } + } + + @Override + public void close() throws StorageException { + myBackendStorage.close(); + } + + @Override + public void clear() throws StorageException { + clearMemoryMap(); + myBackendStorage.clear(); + } + + @Override + public void flush() throws IOException { + myBackendStorage.flush(); + } + + @NotNull + @Override + public Collection getKeys() throws StorageException { + final Set keys = new HashSet(); + processKeys(new CancelableCollectProcessor(keys, new Runnable() { + @Override + public void run() { + checkCanceled(); + } + })); + return keys; + } + + @Override + public boolean processKeys(@NotNull final Processor processor) throws StorageException { + final Processor uniqueResultProcessor = processCacheFirstProcessor(processor); + return uniqueResultProcessor != null && myBackendStorage.processKeys(uniqueResultProcessor); + } + + @Nullable + protected Processor processCacheFirstProcessor(final Processor initialProcessor) { + if (myMap.isEmpty()) return initialProcessor; + final Set stopList = new HashSet(); + + Processor decoratingProcessor = new Processor() { + @Override + public boolean process(Key key) { + if (stopList.contains(key)) return true; + + final UpdatableValueContainer container = myMap.get(key); + if (container != null && container.size() == 0) { + return true; + } + return initialProcessor.process(key); + } + }; + + for (Key key : myMap.keySet()) { + if (!decoratingProcessor.process(key)) { + return null; + } + stopList.add(key); + } + + return decoratingProcessor; + } + + @Override + public void addValue(final Key key, final int inputId, final Value value) throws StorageException { + if (myBufferingEnabled) { + getMemValueContainer(key).addValue(inputId, value); + return; + } + final ChangeTrackingValueContainer valueContainer = myMap.get(key); + if (valueContainer != null) { + valueContainer.dropMergedData(); + } + + myBackendStorage.addValue(key, inputId, value); + } + + @Override + public void removeAllValues(@NotNull Key key, int inputId) throws StorageException { + if (myBufferingEnabled) { + getMemValueContainer(key).removeAssociatedValue(inputId); + return; + } + final ChangeTrackingValueContainer valueContainer = myMap.get(key); + if (valueContainer != null) { + valueContainer.dropMergedData(); + } + + myBackendStorage.removeAllValues(key, inputId); + } + + private UpdatableValueContainer getMemValueContainer(final Key key) { + ChangeTrackingValueContainer valueContainer = myMap.get(key); + if (valueContainer == null) { + valueContainer = new ChangeTrackingValueContainer(new ChangeTrackingValueContainer.Initializer() { + @Override + public Object getLock() { + return this; + } + + @Override + public ValueContainer compute() { + try { + return myBackendStorage.read(key); + } + catch (StorageException e) { + throw new RuntimeException(e); + } + } + }); + myMap.put(key, valueContainer); + } + return valueContainer; + } + + @Override + @NotNull + public ValueContainer read(final Key key) throws StorageException { + final ValueContainer valueContainer = myMap.get(key); + if (valueContainer != null) { + return valueContainer; + } + + return myBackendStorage.read(key); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/StorageException.java b/platform/util/src/com/intellij/util/indexing/StorageException.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/StorageException.java rename to platform/util/src/com/intellij/util/indexing/StorageException.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableValueContainer.java b/platform/util/src/com/intellij/util/indexing/UpdatableValueContainer.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/UpdatableValueContainer.java rename to platform/util/src/com/intellij/util/indexing/UpdatableValueContainer.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java b/platform/util/src/com/intellij/util/indexing/UpdateData.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java rename to platform/util/src/com/intellij/util/indexing/UpdateData.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainer.java b/platform/util/src/com/intellij/util/indexing/ValueContainer.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/ValueContainer.java rename to platform/util/src/com/intellij/util/indexing/ValueContainer.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java b/platform/util/src/com/intellij/util/indexing/ValueContainerImpl.java similarity index 96% rename from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java rename to platform/util/src/com/intellij/util/indexing/ValueContainerImpl.java index 591a4a0d5b6e..4f1d4fc43239 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java +++ b/platform/util/src/com/intellij/util/indexing/ValueContainerImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -79,20 +79,20 @@ class ValueContainerImpl extends UpdatableValueContainer implement return myInputIdMapping != null ? myInputIdMapping instanceof THashMap ? ((THashMap)myInputIdMapping).size(): 1 : 0; } - static final ThreadLocal ourDebugIndexInfo = new ThreadLocal<>(); + static final ThreadLocal ourDebugIndexInfo = new ThreadLocal(); @Override public void removeAssociatedValue(int inputId) { if (myInputIdMapping == null) return; List fileSetObjects = null; List valueObjects = null; - for (final ValueIterator valueIterator = getValueIterator(); valueIterator.hasNext();) { + for (final ValueContainer.ValueIterator valueIterator = getValueIterator(); valueIterator.hasNext();) { final Value value = valueIterator.next(); if (valueIterator.getValueAssociationPredicate().contains(inputId)) { if (fileSetObjects == null) { - fileSetObjects = new SmartList<>(); - valueObjects = new SmartList<>(); + fileSetObjects = new SmartList(); + valueObjects = new SmartList(); } else if (DebugAssertions.DEBUG) { LOG.error("Expected only one value per-inputId for " + ourDebugIndexInfo.get(), String.valueOf(fileSetObjects.get(0)), String.valueOf(value)); @@ -263,9 +263,9 @@ class ValueContainerImpl extends UpdatableValueContainer implement if (myInputIdMapping == null) { return Collections.emptyList(); } else if (myInputIdMapping instanceof THashMap) { - return new ArrayList<>(((THashMap)myInputIdMapping).keySet()); + return new ArrayList(((THashMap)myInputIdMapping).keySet()); } else { - return new SmartList<>((Value)myInputIdMapping); + return new SmartList((Value)myInputIdMapping); } } @@ -366,11 +366,11 @@ class ValueContainerImpl extends UpdatableValueContainer implement @NotNull public ValueContainerImpl copy() { - ValueContainerImpl container = new ValueContainerImpl<>(); + ValueContainerImpl container = new ValueContainerImpl(); if (myInputIdMapping instanceof THashMap) { final THashMap mapping = (THashMap)myInputIdMapping; - final THashMap newMapping = new THashMap<>(mapping.size()); + final THashMap newMapping = new THashMap(mapping.size()); container.myInputIdMapping = newMapping; mapping.forEachEntry(new TObjectObjectProcedure() { @@ -485,7 +485,7 @@ class ValueContainerImpl extends UpdatableValueContainer implement final int inputId = -valueCount; if (mapping == null && size() > NUMBER_OF_VALUES_THRESHOLD) { // avoid O(NumberOfValues) - mapping = new FileId2ValueMapping<>(this); + mapping = new FileId2ValueMapping(this); } boolean doCompact; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java b/platform/util/src/com/intellij/util/indexing/ValueContainerMap.java similarity index 94% rename from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java rename to platform/util/src/com/intellij/util/indexing/ValueContainerMap.java index e6789513ee35..c50745debf58 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java +++ b/platform/util/src/com/intellij/util/indexing/ValueContainerMap.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ class ValueContainerMap extends PersistentHashMap valueExternalizer, boolean keyIsUniqueForIndexedFile ) throws IOException { - super(file, keyKeyDescriptor, new ValueContainerExternalizer<>(valueExternalizer)); + super(file, keyKeyDescriptor, new ValueContainerExternalizer(valueExternalizer)); myValueExternalizer = valueExternalizer; myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile; } @@ -48,7 +48,7 @@ class ValueContainerMap extends PersistentHashMap container) throws IOException { synchronized (myEnumerator) { - ChangeTrackingValueContainer valueContainer = (ChangeTrackingValueContainer)container; + final ChangeTrackingValueContainer valueContainer = (ChangeTrackingValueContainer)container; // try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files // note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular @@ -83,7 +83,7 @@ class ValueContainerMap extends PersistentHashMap read(@NotNull final DataInput in) throws IOException { - final ValueContainerImpl valueContainer = new ValueContainerImpl<>(); + final ValueContainerImpl valueContainer = new ValueContainerImpl(); valueContainer.readFrom((DataInputStream)in, myValueExternalizer); return valueContainer; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java b/platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java similarity index 99% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java rename to platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java index 7e9076282b8e..7ca2968b20d2 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java +++ b/platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java b/platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java similarity index 99% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java rename to platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java index 357c919f7162..d82f0e223f1b 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java +++ b/platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java b/platform/util/src/com/intellij/util/indexing/containers/IdSet.java similarity index 97% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java rename to platform/util/src/com/intellij/util/indexing/containers/IdSet.java index bbe7ff7891e7..9afcd7d6b48a 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java +++ b/platform/util/src/com/intellij/util/indexing/containers/IdSet.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java b/platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java rename to platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java b/platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java similarity index 98% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java rename to platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java index f42373c139c7..af2a200cf559 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java +++ b/platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/SortedIdSet.java b/platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/SortedIdSet.java rename to platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java b/platform/util/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java rename to platform/util/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java index 389b2a068330..f5d3d22966ea 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java @@ -144,7 +144,7 @@ public class VcsLogFullDetailsIndex implements Disposable { } @Override - protected void requestRebuild(@Nullable Exception ex) { + public void requestRebuild(@Nullable Exception ex) { myFatalErrorHandler.consume(this, ex != null ? ex : new Exception("Index rebuild requested")); } }