From 72125c450f389bf8baf3825903fda0c9b063dfdf Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Sat, 5 Mar 2016 22:33:07 +0100 Subject: [PATCH] refactor MapReduceIndex to have simple constructor arguments and all index implementation inside --- .../com/intellij/index/IndexTest.groovy | 20 +-- .../com/intellij/index/StringIndex.java | 49 ++++-- .../indexing/FileBasedIndexExtension.java | 18 +-- .../util/indexing/IndexExtension.java | 42 +++++ .../com/intellij/psi/stubs/StubIndexImpl.java | 39 ++++- .../intellij/psi/stubs/StubUpdatingIndex.java | 9 +- ...ImplementationFileBasedIndexExtension.java | 3 +- .../util/indexing/FileBasedIndexImpl.java | 106 +----------- .../util/indexing/MapReduceIndex.java | 152 ++++++++++++------ 9 files changed, 240 insertions(+), 198 deletions(-) create mode 100644 platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java diff --git a/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy b/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy index 6e6976950520..c5924c4f63a4 100644 --- a/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy +++ b/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy @@ -24,7 +24,6 @@ import com.intellij.openapi.fileEditor.FileDocumentManager import com.intellij.openapi.fileEditor.FileEditor import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.fileTypes.PlainTextFileType -import com.intellij.openapi.util.Factory import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.vfs.VfsUtil import com.intellij.openapi.vfs.VirtualFile @@ -71,7 +70,7 @@ public class IndexTest extends JavaCodeInsightFixtureTestCase { } public void testUpdate() throws StorageException, IOException { - StringIndex index = createIndex(new EnumeratorStringDescriptor()) + StringIndex index = createIndex(getTestName(false), new EnumeratorStringDescriptor()) try { // build index @@ -118,7 +117,7 @@ public class IndexTest extends JavaCodeInsightFixtureTestCase { } public void testUpdateWithCustomEqualityPolicy() { - def index = createIndex(new CaseInsensitiveEnumeratorStringDescriptor()) + def index = createIndex(getTestName(false), new CaseInsensitiveEnumeratorStringDescriptor()) try { index.update("a.java", "x", null) assertDataEquals(index.getFilesByWord("x"), "a.java") @@ -136,21 +135,12 @@ public class IndexTest extends JavaCodeInsightFixtureTestCase { } } - private static StringIndex createIndex(EnumeratorStringDescriptor keyDescriptor) { + private static StringIndex createIndex(String testName, EnumeratorStringDescriptor keyDescriptor) { final File storageFile = FileUtil.createTempFile("index_test", "storage"); final File metaIndexFile = FileUtil.createTempFile("index_test_inputs", "storage"); + PersistentHashMap> index = createMetaIndex(metaIndexFile); final MapIndexStorage indexStorage = new MapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024); - return new StringIndex(indexStorage, new Factory>>() { - @Override - public PersistentHashMap> create() { - try { - return createMetaIndex(metaIndexFile); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - }); + return new StringIndex(testName, indexStorage, index); } private static PersistentHashMap> createMetaIndex(File metaIndexFile) throws IOException { diff --git a/java/java-tests/testSrc/com/intellij/index/StringIndex.java b/java/java-tests/testSrc/com/intellij/index/StringIndex.java index b4e816e49144..f83d83c299e9 100644 --- a/java/java-tests/testSrc/com/intellij/index/StringIndex.java +++ b/java/java-tests/testSrc/com/intellij/index/StringIndex.java @@ -15,11 +15,10 @@ */ package com.intellij.index; -import com.intellij.openapi.util.Factory; -import com.intellij.util.indexing.DataIndexer; -import com.intellij.util.indexing.IndexStorage; -import com.intellij.util.indexing.MapReduceIndex; -import com.intellij.util.indexing.StorageException; +import com.intellij.util.indexing.*; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.KeyDescriptor; import com.intellij.util.io.PersistentHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -37,12 +36,44 @@ import java.util.Map; public class StringIndex { private final MapReduceIndex myIndex; - public StringIndex(final IndexStorage storage, final Factory>> factory) + public StringIndex(String testName, final IndexStorage storage, final PersistentHashMap> inputIndex) throws IOException { - myIndex = new MapReduceIndex(null, new Indexer(), storage); - myIndex.setInputIdToDataKeysIndex(factory); + myIndex = new MapReduceIndex(new IndexExtension() { + @NotNull + @Override + public ID getName() { + return new ID(testName + "string_index") {}; + } + + @NotNull + @Override + public DataIndexer getIndexer() { + return new Indexer(); + } + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return new EnumeratorStringDescriptor(); + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return new EnumeratorStringDescriptor(); + } + + @Override + public int getVersion() { + return 0; + } + }, storage) { + protected PersistentHashMap> createInputsIndex() throws IOException { + return inputIndex; + } + }; } - + public List getFilesByWord(@NotNull String word) throws StorageException { return myIndex.getData(word).toValueList(); } diff --git a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java index 6726e56fad7e..ea63f3e59d52 100644 --- a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java +++ b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java @@ -18,8 +18,6 @@ package com.intellij.util.indexing; import com.intellij.openapi.extensions.ExtensionPointName; import com.intellij.openapi.fileTypes.FileType; -import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.KeyDescriptor; import org.jetbrains.annotations.NotNull; import java.util.Collection; @@ -30,22 +28,10 @@ import java.util.Collections; * Date: Dec 26, 2007 * V class MUST have equals / hashcode properly defined!!! */ -public abstract class FileBasedIndexExtension { +public abstract class FileBasedIndexExtension extends IndexExtension{ public static final ExtensionPointName EXTENSION_POINT_NAME = ExtensionPointName.create("com.intellij.fileBasedIndex"); public static final int DEFAULT_CACHE_SIZE = 1024; - @NotNull - public abstract ID getName(); - - @NotNull - public abstract DataIndexer getIndexer(); - - @NotNull - public abstract KeyDescriptor getKeyDescriptor(); - - @NotNull - public abstract DataExternalizer getValueExternalizer(); - @NotNull public abstract FileBasedIndex.InputFilter getInputFilter(); @@ -55,8 +41,6 @@ public abstract class FileBasedIndexExtension { return false; } - public abstract int getVersion(); - /** * @see FileBasedIndexExtension#DEFAULT_CACHE_SIZE */ diff --git a/platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java b/platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java new file mode 100644 index 000000000000..8b9b0081a9e1 --- /dev/null +++ b/platform/indexing-api/src/com/intellij/util/indexing/IndexExtension.java @@ -0,0 +1,42 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intellij.util.indexing; + +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.KeyDescriptor; +import org.jetbrains.annotations.NotNull; + +/** + * @author Eugene Zhuravlev + * Date: Dec 26, 2007 + * V class MUST have equals / hashcode properly defined!!! + */ +public abstract class IndexExtension { + @NotNull + public abstract ID getName(); + + @NotNull + public abstract DataIndexer getIndexer(); + + @NotNull + public abstract KeyDescriptor getKeyDescriptor(); + + @NotNull + public abstract DataExternalizer getValueExternalizer(); + + public abstract int getVersion(); +} diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 7421bda26859..81395db27733 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -51,6 +51,7 @@ import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.DataInputOutputUtil; +import com.intellij.util.io.KeyDescriptor; import gnu.trove.THashMap; import gnu.trove.TIntArrayList; import gnu.trove.TObjectIntHashMap; @@ -112,17 +113,47 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe for (int attempt = 0; attempt < 2; attempt++) { try { + StubIdExternalizer externalizer = new StubIdExternalizer(); final MapIndexStorage storage = new MapIndexStorage( IndexInfrastructure.getStorageFile(indexKey), extension.getKeyDescriptor(), - new StubIdExternalizer(), + externalizer, extension.getCacheSize(), false, extension instanceof StringStubIndexExtension && ((StringStubIndexExtension)extension).traceKeyHashToVirtualFileMapping() ); final MemoryIndexStorage memStorage = new MemoryIndexStorage(storage); - myIndices.put(indexKey, new MyIndex(memStorage)); + myIndices.put(indexKey, new MyIndex(new IndexExtension() { + @NotNull + @Override + public ID getName() { + return (ID)indexKey; + } + + @NotNull + @Override + public DataIndexer getIndexer() { + return inputData -> Collections.emptyMap(); + } + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return extension.getKeyDescriptor(); + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return externalizer; + } + + @Override + public int getVersion() { + return extension.getVersion(); + } + }, memStorage)); break; } catch (IOException e) { @@ -512,8 +543,8 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } private static class MyIndex extends MapReduceIndex { - public MyIndex(final IndexStorage storage) throws IOException { - super(null, null, storage); + public MyIndex(IndexExtension extension, IndexStorage storage) throws IOException { + super(extension, storage); } @Override diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 21fd7b45b372..58104a8558c8 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -230,7 +230,8 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi @NotNull @Override - public UpdatableIndex createIndexImplementation(@NotNull final ID indexId, @NotNull final FileBasedIndex owner, @NotNull IndexStorage storage) + public UpdatableIndex createIndexImplementation(@NotNull final FileBasedIndexExtension extension, + @NotNull IndexStorage storage) throws StorageException, IOException { if (storage instanceof MemoryIndexStorage) { final MemoryIndexStorage memStorage = (MemoryIndexStorage)storage; @@ -246,7 +247,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } }); } - return new MyIndex(indexId, storage, getIndexer()); + return new MyIndex(extension, storage); } private static void updateStubIndices(@NotNull final Collection indexKeys, @@ -277,9 +278,9 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi private static class MyIndex extends MapReduceIndex { private StubIndexImpl myStubIndex; - public MyIndex(final ID indexId, final IndexStorage storage, final DataIndexer indexer) + public MyIndex(FileBasedIndexExtension extension, IndexStorage storage) throws StorageException, IOException { - super(indexId, indexer, storage); + super(extension, storage); checkNameStorage(); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java index e0e5d8e79db8..bc18fa17f00d 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java @@ -25,6 +25,7 @@ import java.io.IOException; public abstract class CustomImplementationFileBasedIndexExtension extends FileBasedIndexExtension { @NotNull - public abstract UpdatableIndex createIndexImplementation(@NotNull ID indexId, @NotNull FileBasedIndex owner, @NotNull IndexStorage storage) + public abstract UpdatableIndex createIndexImplementation(@NotNull FileBasedIndexExtension extension, + @NotNull IndexStorage storage) throws StorageException, IOException; } \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 8e3ee5ee64a0..ceef97a90a55 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -73,8 +73,8 @@ import com.intellij.psi.stubs.SerializationManagerEx; import com.intellij.util.*; import com.intellij.util.containers.ConcurrentIntObjectMap; import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.io.*; import com.intellij.util.io.DataOutputStream; +import com.intellij.util.io.IOUtil; import com.intellij.util.io.storage.HeavyProcessLatch; import com.intellij.util.messages.MessageBus; import com.intellij.util.messages.MessageBusConnection; @@ -536,124 +536,26 @@ public class FileBasedIndexImpl extends FileBasedIndex { } @NotNull - private UpdatableIndex createIndex(@NotNull final ID indexId, + private static UpdatableIndex createIndex(@NotNull final ID indexId, @NotNull final FileBasedIndexExtension extension, @NotNull final MemoryIndexStorage storage) throws StorageException, IOException { final MapReduceIndex index; if (extension instanceof CustomImplementationFileBasedIndexExtension) { final UpdatableIndex custom = - ((CustomImplementationFileBasedIndexExtension)extension).createIndexImplementation(indexId, this, storage); + ((CustomImplementationFileBasedIndexExtension)extension).createIndexImplementation(extension, storage); if (!(custom instanceof MapReduceIndex)) { return custom; } index = (MapReduceIndex)custom; } else { - DataExternalizer> externalizer = - extension.hasSnapshotMapping() && IdIndex.ourSnapshotMappingsEnabled - ? createInputsIndexExternalizer(extension, indexId, extension.getKeyDescriptor()) - : null; - index = new MapReduceIndex( - indexId, extension.getIndexer(), storage, externalizer, extension.getValueExternalizer(), extension instanceof PsiDependentIndex); + index = new MapReduceIndex(extension, storage); } - index.setInputIdToDataKeysIndex(new Factory>>() { - @Override - public PersistentHashMap> create() { - try { - return createIdToDataKeysIndex(extension, storage); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - }); return index; } - @NotNull - public static PersistentHashMap> createIdToDataKeysIndex(@NotNull FileBasedIndexExtension extension, - @NotNull MemoryIndexStorage storage) - throws IOException { - ID indexId = extension.getName(); - KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); - final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(indexId); - final AtomicBoolean isBufferingMode = new AtomicBoolean(); - final TIntObjectHashMap> tempMap = new TIntObjectHashMap>(); - - // Important! Update IdToDataKeysIndex depending on the sate of "buffering" flag from the MemoryStorage. - // If buffering is on, all changes should be done in memory (similar to the way it is done in memory storage). - // Otherwise data in IdToDataKeysIndex will not be in sync with the 'main' data in the index on disk and index updates will be based on the - // wrong sets of keys for the given file. This will lead to unpredictable results in main index because it will not be - // cleared properly before updating (removed data will still be present on disk). See IDEA-52223 for illustration of possible effects. - - final PersistentHashMap> map = new PersistentHashMap>( - indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, createInputsIndexExternalizer(extension, indexId, keyDescriptor) - ) { - - @Override - protected Collection doGet(Integer integer) throws IOException { - if (isBufferingMode.get()) { - final Collection collection = tempMap.get(integer); - if (collection != null) { - return collection; - } - } - return super.doGet(integer); - } - - @Override - protected void doPut(Integer integer, @Nullable Collection ks) throws IOException { - if (isBufferingMode.get()) { - tempMap.put(integer, ks == null ? Collections.emptySet() : ks); - } - else { - super.doPut(integer, ks); - } - } - - @Override - protected void doRemove(Integer integer) throws IOException { - if (isBufferingMode.get()) { - tempMap.put(integer, Collections.emptySet()); - } - else { - super.doRemove(integer); - } - } - }; - - storage.addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() { - @Override - public void bufferingStateChanged(boolean newState) { - synchronized (map) { - isBufferingMode.set(newState); - } - } - - @Override - public void memoryStorageCleared() { - synchronized (map) { - tempMap.clear(); - } - } - }); - return map; - } - - private static DataExternalizer> createInputsIndexExternalizer(FileBasedIndexExtension extension, - ID indexId, - KeyDescriptor keyDescriptor) { - DataExternalizer> externalizer; - if (extension instanceof CustomInputsIndexFileBasedIndexExtension) { - externalizer = ((CustomInputsIndexFileBasedIndexExtension)extension).createExternalizer(); - } else { - externalizer = new InputIndexDataExternalizer(keyDescriptor, indexId); - } - return externalizer; - } - @Override public void disposeComponent() { performShutdown(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java index 1de0393acde6..562e73bddc79 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java @@ -26,17 +26,20 @@ import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; import com.intellij.openapi.util.io.ByteSequence; import com.intellij.psi.PsiDocumentManager; import com.intellij.psi.PsiFile; +import com.intellij.psi.impl.cache.impl.id.IdIndex; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.*; import com.intellij.util.io.*; import com.intellij.util.io.DataOutputStream; import gnu.trove.THashMap; +import gnu.trove.TIntObjectHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.*; import java.nio.charset.Charset; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -55,6 +58,7 @@ public class MapReduceIndex implements UpdatableIndex myValueExternalizer; private final DataExternalizer> mySnapshotIndexExternalizer; private final boolean myIsPsiBackedIndex; + private final IndexExtension myExtension; private PersistentHashMap> myInputsIndex; private PersistentHashMap myContents; @@ -63,8 +67,6 @@ public class MapReduceIndex implements UpdatableIndex>> myInputsIndexFactory; - private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() { @Override public void run() { @@ -77,28 +79,110 @@ public class MapReduceIndex implements UpdatableIndex indexId, - DataIndexer indexer, + public MapReduceIndex(IndexExtension extension, @NotNull IndexStorage storage) throws IOException { - this(indexId, indexer, storage, null, null, false); + myIndexId = extension.getName(); + myExtension = extension; + myIndexer = extension.getIndexer(); + myStorage = storage; + myHasSnapshotMapping = extension instanceof FileBasedIndexExtension && + ((FileBasedIndexExtension)extension).hasSnapshotMapping() && + IdIndex.ourSnapshotMappingsEnabled; + + mySnapshotIndexExternalizer = createInputsIndexExternalizer(extension, myIndexId, extension.getKeyDescriptor()); + myValueExternalizer = extension.getValueExternalizer(); + myContents = createContentsIndex(); + myIsPsiBackedIndex = extension instanceof PsiDependentIndex; + + if (myHasSnapshotMapping) { + myInputsSnapshotMapping = createInputSnapshotMapping(); + } + myInputsIndex = createInputsIndex(); + if (DebugAssertions.EXTRA_SANITY_CHECKS && myHasSnapshotMapping && myIndexId != null) { + myIndexingTrace = createIndexingTrace(); + } } - public MapReduceIndex(@Nullable final ID indexId, - DataIndexer indexer, - @NotNull IndexStorage storage, - DataExternalizer> snapshotIndexExternalizer, - DataExternalizer valueDataExternalizer, - boolean psiBasedIndex - ) throws IOException { - myIndexId = indexId; - myIndexer = indexer; - myStorage = storage; - myHasSnapshotMapping = snapshotIndexExternalizer != null; + private static DataExternalizer> createInputsIndexExternalizer(IndexExtension extension, + ID indexId, + KeyDescriptor keyDescriptor) { + DataExternalizer> externalizer; + if (extension instanceof CustomInputsIndexFileBasedIndexExtension) { + externalizer = ((CustomInputsIndexFileBasedIndexExtension)extension).createExternalizer(); + } else { + externalizer = new InputIndexDataExternalizer(keyDescriptor, indexId); + } + return externalizer; + } - mySnapshotIndexExternalizer = snapshotIndexExternalizer; - myValueExternalizer = valueDataExternalizer; - myContents = createContentsIndex(); - myIsPsiBackedIndex = psiBasedIndex; + @NotNull + private static PersistentHashMap> createIdToDataKeysIndex(@NotNull IndexExtension extension, + @NotNull MemoryIndexStorage storage) + throws IOException { + ID indexId = extension.getName(); + KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); + final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(indexId); + final AtomicBoolean isBufferingMode = new AtomicBoolean(); + final TIntObjectHashMap> tempMap = new TIntObjectHashMap>(); + + // Important! Update IdToDataKeysIndex depending on the sate of "buffering" flag from the MemoryStorage. + // If buffering is on, all changes should be done in memory (similar to the way it is done in memory storage). + // Otherwise data in IdToDataKeysIndex will not be in sync with the 'main' data in the index on disk and index updates will be based on the + // wrong sets of keys for the given file. This will lead to unpredictable results in main index because it will not be + // cleared properly before updating (removed data will still be present on disk). See IDEA-52223 for illustration of possible effects. + + final PersistentHashMap> map = new PersistentHashMap>( + indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, createInputsIndexExternalizer(extension, indexId, keyDescriptor) + ) { + + @Override + protected Collection doGet(Integer integer) throws IOException { + if (isBufferingMode.get()) { + final Collection collection = tempMap.get(integer); + if (collection != null) { + return collection; + } + } + return super.doGet(integer); + } + + @Override + protected void doPut(Integer integer, @Nullable Collection ks) throws IOException { + if (isBufferingMode.get()) { + tempMap.put(integer, ks == null ? Collections.emptySet() : ks); + } + else { + super.doPut(integer, ks); + } + } + + @Override + protected void doRemove(Integer integer) throws IOException { + if (isBufferingMode.get()) { + tempMap.put(integer, Collections.emptySet()); + } + else { + super.doRemove(integer); + } + } + }; + + storage.addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() { + @Override + public void bufferingStateChanged(boolean newState) { + synchronized (map) { + isBufferingMode.set(newState); + } + } + + @Override + public void memoryStorageCleared() { + synchronized (map) { + tempMap.clear(); + } + } + }); + return map; } private PersistentHashMap createContentsIndex() throws IOException { @@ -315,32 +399,8 @@ public class MapReduceIndex implements UpdatableIndex>> factory) throws IOException { - myInputsIndexFactory = factory; - if (myHasSnapshotMapping) { - myInputsSnapshotMapping = createInputSnapshotMapping(); - } - myInputsIndex = createInputsIndex(); - if (DebugAssertions.EXTRA_SANITY_CHECKS && myHasSnapshotMapping && myIndexId != null) { - myIndexingTrace = createIndexingTrace(); - } - } - - @Nullable - private PersistentHashMap> createInputsIndex() throws IOException { - Factory>> factory = myInputsIndexFactory; - if (factory != null) { - try { - return factory.create(); - } - catch (RuntimeException e) { - if (e.getCause() instanceof IOException) { - throw (IOException)e.getCause(); - } - throw e; - } - } - return null; + protected PersistentHashMap> createInputsIndex() throws IOException { + return createIdToDataKeysIndex(myExtension, (MemoryIndexStorage)myStorage); } private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true);