diff --git a/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy b/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy index 5c01c01c3611..db7f6efb0e61 100644 --- a/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy +++ b/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy @@ -154,7 +154,7 @@ class IndexTest extends JavaCodeInsightFixtureTestCase { final File storageFile = FileUtil.createTempFile("index_test", "storage") final File metaIndexFile = FileUtil.createTempFile("index_test_inputs", "storage") PersistentHashMap> index = createMetaIndex(metaIndexFile) - final MapIndexStorage indexStorage = new MapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024) + final VfsAwareMapIndexStorage indexStorage = new VfsAwareMapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024) return new StringIndex(testName, indexStorage, index) } diff --git a/java/java-tests/testSrc/com/intellij/index/StringIndex.java b/java/java-tests/testSrc/com/intellij/index/StringIndex.java index 379a0e2ae236..460044d3cfc2 100644 --- a/java/java-tests/testSrc/com/intellij/index/StringIndex.java +++ b/java/java-tests/testSrc/com/intellij/index/StringIndex.java @@ -17,12 +17,16 @@ package com.intellij.index; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; +import com.intellij.util.indexing.impl.IndexStorage; +import com.intellij.util.indexing.impl.MapBasedForwardIndex; +import com.intellij.util.indexing.impl.MapReduceIndex; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.EnumeratorStringDescriptor; import com.intellij.util.io.KeyDescriptor; import com.intellij.util.io.PersistentHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.junit.Assert; import java.io.IOException; import java.util.Collection; @@ -39,11 +43,12 @@ public class StringIndex { public StringIndex(String testName, final IndexStorage storage, final PersistentHashMap> inputIndex) throws IOException { - myIndex = new MapReduceIndex(new IndexExtension() { + ID id = ID.create(testName + "string_index"); + IndexExtension extension = new IndexExtension() { @NotNull @Override public ID getName() { - return new ID(testName + "string_index") {}; + return id; } @NotNull @@ -68,10 +73,19 @@ public class StringIndex { public int getVersion() { return 0; } - }, storage) { - protected PersistentHashMap> createInputsIndex() throws IOException { + }; + myIndex = new VfsAwareMapReduceIndex(extension, storage, new MapBasedForwardIndex(extension) { + @NotNull + @Override + public PersistentHashMap> createMap() throws IOException { return inputIndex; } + }) { + @Override + public void requestRebuild(@NotNull Exception ex) { + Assert.fail(); + } + }; } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 4f2b2014fc2c..4d80ed09dc5f 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -31,6 +31,7 @@ import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.fileTypes.LanguageFileType; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.ThrowableComputable; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; @@ -49,6 +50,7 @@ import com.intellij.util.Processors; import com.intellij.util.SmartList; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; +import com.intellij.util.indexing.impl.*; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.DataInputOutputUtil; import com.intellij.util.io.KeyDescriptor; @@ -134,7 +136,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe for (int attempt = 0; attempt < 2; attempt++) { try { - final MapIndexStorage storage = new MapIndexStorage<>( + final VfsAwareMapIndexStorage storage = new VfsAwareMapIndexStorage<>( IndexInfrastructure.getStorageFile(indexKey), extension.getKeyDescriptor(), StubIdExternalizer.INSTANCE, @@ -503,14 +505,14 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe public void setDataBufferingEnabled(final boolean enabled) { for (UpdatableIndex index : getAsyncState().myIndices.values()) { - final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage(); + final IndexStorage indexStorage = ((VfsAwareMapReduceIndex)index).getStorage(); ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled); } } public void cleanupMemoryStorage() { for (UpdatableIndex index : getAsyncState().myIndices.values()) { - final IndexStorage indexStorage = ((MapReduceIndex)index).getStorage(); + final IndexStorage indexStorage = ((VfsAwareMapReduceIndex)index).getStorage(); index.getWriteLock().lock(); try { ((MemoryIndexStorage)indexStorage).clearMemoryMap(); @@ -573,32 +575,18 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe return Collections.unmodifiableCollection(getAsyncState().myIndices.keySet()); } - public void updateIndex(@NotNull StubIndexKey key, int fileId, @NotNull final Map oldValues, @NotNull final Map newValues) { + public void updateIndex(@NotNull StubIndexKey key, + int fileId, + @NotNull final Map oldValues, + @NotNull final Map newValues) { try { final MyIndex index = (MyIndex)getAsyncState().myIndices.get(key); - UpdateData updateData; - - if (MapDiffUpdateData.ourDiffUpdateEnabled) { - updateData = new MapDiffUpdateData(key) { - @Override - public void save(int inputId) throws IOException { - } - - @Override - protected Map getNewValue() { - return newValues; - } - - @Override - protected Map getCurrentValue() throws IOException { - return oldValues; - } - }; - } - else { - updateData = index.new SimpleUpdateData(key, fileId, newValues, oldValues::keySet); - } - index.updateWithMap(fileId, updateData); + final ThrowableComputable, IOException> + oldMapGetter = () -> new MapInputKeyIterator<>(oldValues); + index.updateWithMap(fileId, + DiffUpdateData.ourDiffUpdateEnabled + ? new DiffUpdateData<>(newValues, oldMapGetter, key, null) + : new SimpleUpdateData<>(newValues, oldMapGetter, key, null)); } catch (StorageException e) { LOG.info(e); @@ -606,7 +594,8 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } } - private static class MyIndex extends MapReduceIndex { + private static class MyIndex extends VfsAwareMapReduceIndex { + public MyIndex(IndexExtension extension, IndexStorage storage) throws IOException { super(extension, storage); } @@ -616,6 +605,10 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe @NotNull UpdateData updateData) throws StorageException { super.updateWithMap(inputId, updateData); } + + public IndexExtension getExtension() { + return myExtension; + } } @Override diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 2f5ed7af88b2..63dcb2555ffc 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -23,7 +23,7 @@ import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.fileTypes.FileType; import com.intellij.openapi.fileTypes.LanguageFileType; import com.intellij.openapi.progress.ProcessCanceledException; -import com.intellij.openapi.util.NotNullComputable; +import com.intellij.openapi.util.ThrowableComputable; import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.newvfs.FileAttribute; @@ -32,8 +32,10 @@ import com.intellij.openapi.vfs.newvfs.persistent.FSRecords; import com.intellij.psi.tree.IFileElementType; import com.intellij.psi.tree.IStubFileElementType; import com.intellij.util.ExceptionUtil; +import com.intellij.util.ThrowableRunnable; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.*; +import com.intellij.util.indexing.impl.*; import com.intellij.util.io.*; import gnu.trove.THashMap; import org.jetbrains.annotations.NotNull; @@ -391,7 +393,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } } - private static class MyIndex extends MapReduceIndex { + private static class MyIndex extends VfsAwareMapReduceIndex { private StubIndexImpl myStubIndex; private final StubVersionMap myStubVersionMap = new StubVersionMap(); @@ -401,37 +403,32 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi checkNameStorage(); } + @NotNull @Override - public void flush() throws StorageException { - final StubIndexImpl stubIndex = getStubIndex(); - try { - stubIndex.flush(); - } - finally { - super.flush(); - } + protected UpdateData createUpdateData(Map data, + ThrowableComputable, IOException> oldKeys, + ThrowableRunnable forwardIndexUpdate) { + return new StubUpdatingData(data, oldKeys, forwardIndexUpdate); } - @Override - protected UpdateData buildUpdateData(Map data, - NotNullComputable> oldKeysGetter, - int savedInputId) { - return new StubUpdatingData(savedInputId, data, oldKeysGetter); - } - - class StubUpdatingData extends SimpleUpdateData { + static class StubUpdatingData extends SimpleUpdateData { private Collection oldStubIndexKeys; - public StubUpdatingData(int id, - @NotNull Map data, - @NotNull NotNullComputable> getter) { - super(INDEX_ID, id, data, getter); + public StubUpdatingData(@NotNull Map newData, + @NotNull ThrowableComputable, IOException> iterator, + ThrowableRunnable forwardIndexUpdate) { + super(newData, iterator, INDEX_ID, forwardIndexUpdate); } @Override - public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor consumer) throws StorageException { - oldStubIndexKeys = oldKeysGetter.compute(); - MapDiffUpdateData.iterateRemovedKeys(oldStubIndexKeys, inputId, consumer); + protected void iterateKeys(int inputId, + KeyValueUpdateProcessor addProcessor, + RemovedKeyProcessor removeProcessor, + ForwardIndex.InputKeyIterator currentData) throws StorageException { + if (currentData instanceof CollectionInputKeyIterator) { + oldStubIndexKeys = ((CollectionInputKeyIterator)currentData).getCollection(); + } + super.iterateKeys(inputId, addProcessor, removeProcessor, currentData); } public Map> getOldStubIndicesValueMap() { @@ -451,10 +448,20 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } @Override - protected void updateWithMap(final int inputId, - @NotNull UpdateData updateData) - throws StorageException { + public void flush() throws StorageException { + final StubIndexImpl stubIndex = getStubIndex(); + try { + stubIndex.flush(); + } + finally { + super.flush(); + } + } + + @Override + protected void updateWithMap(int inputId, + @NotNull UpdateData updateData) throws StorageException { checkNameStorage(); StubUpdatingData stubUpdatingData = (StubUpdatingData)updateData; final Map> newStubIndicesValueMap = stubUpdatingData.getNewStubIndicesValueMap(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java b/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java index bc18fa17f00d..c8272ae2fcec 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/CustomImplementationFileBasedIndexExtension.java @@ -19,6 +19,7 @@ */ package com.intellij.util.indexing; +import com.intellij.util.indexing.impl.IndexStorage; import org.jetbrains.annotations.NotNull; import java.io.IOException; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 46d914f9f48d..add80c869961 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -76,13 +76,17 @@ import com.intellij.util.containers.ContainerUtil; import com.intellij.util.containers.JBIterable; import com.intellij.util.gist.GistManager; import com.intellij.util.gist.GistManagerImpl; -import com.intellij.util.indexing.containers.TroveSetIntIterator; +import com.intellij.util.indexing.impl.InvertedIndexValueIterator; +import com.intellij.util.indexing.impl.MapReduceIndex; import com.intellij.util.io.DataOutputStream; import com.intellij.util.io.IOUtil; import com.intellij.util.io.storage.HeavyProcessLatch; import com.intellij.util.messages.MessageBus; import com.intellij.util.messages.MessageBusConnection; -import gnu.trove.*; +import gnu.trove.THashMap; +import gnu.trove.THashSet; +import gnu.trove.TIntArrayList; +import gnu.trove.TIntHashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; @@ -350,7 +354,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { private static void initIndexStorage(@NotNull FileBasedIndexExtension extension, int version, @NotNull File versionFile, IndexConfiguration state) throws IOException { - MapIndexStorage storage = null; + VfsAwareMapIndexStorage storage = null; final ID name = extension.getName(); boolean contentHashesEnumeratorOk = false; @@ -360,7 +364,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { ContentHashesSupport.initContentHashesEnumerator(); contentHashesEnumeratorOk = true; } - storage = new MapIndexStorage<>( + storage = new VfsAwareMapIndexStorage<>( IndexInfrastructure.getStorageFile(name), extension.getKeyDescriptor(), extension.getValueExternalizer(), @@ -444,17 +448,17 @@ public class FileBasedIndexImpl extends FileBasedIndex { private static UpdatableIndex createIndex(@NotNull final FileBasedIndexExtension extension, @NotNull final MemoryIndexStorage storage) throws StorageException, IOException { - final MapReduceIndex index; + final VfsAwareMapReduceIndex index; if (extension instanceof CustomImplementationFileBasedIndexExtension) { final UpdatableIndex custom = ((CustomImplementationFileBasedIndexExtension)extension).createIndexImplementation(extension, storage); - if (!(custom instanceof MapReduceIndex)) { + if (!(custom instanceof VfsAwareMapReduceIndex)) { return custom; } - index = (MapReduceIndex)custom; + index = (VfsAwareMapReduceIndex)custom; } else { - index = new MapReduceIndex<>(extension, storage); + index = new VfsAwareMapReduceIndex<>(extension, storage); } return index; @@ -1046,75 +1050,15 @@ public class FileBasedIndexImpl extends FileBasedIndex { @Nullable final Condition valueChecker, @Nullable final ProjectIndexableFilesFilter projectFilesFilter) { ThrowableConvertor, TIntHashSet, StorageException> convertor = - index -> collectInputIdsContainingAllKeys(index, dataKeys, valueChecker, + index -> InvertedIndexUtil.collectInputIdsContainingAllKeys(index, dataKeys, (k) -> { + ProgressManager.checkCanceled(); + return true; + }, valueChecker, projectFilesFilter == null ? null : projectFilesFilter::containsFileId); return processExceptions(indexId, null, filter, convertor); } - @Nullable - private static TIntHashSet collectInputIdsContainingAllKeys(@NotNull InvertedIndex index, - @NotNull Collection dataKeys, - @Nullable Condition valueChecker, - @Nullable IntPredicate idChecker) - throws StorageException { - TIntHashSet mainIntersection = null; - - for (K dataKey : dataKeys) { - ProgressManager.checkCanceled(); - final TIntHashSet copy = new TIntHashSet(); - final ValueContainer container = index.getData(dataKey); - - for (InvertedIndexValueIterator valueIt = (InvertedIndexValueIterator)container.getValueIterator(); valueIt.hasNext(); ) { - final V value = valueIt.next(); - if (valueChecker != null && !valueChecker.value(value)) { - continue; - } - - ValueContainer.IntIterator iterator = valueIt.getInputIdsIterator(); - - if (mainIntersection == null || iterator.size() < mainIntersection.size()) { - while (iterator.hasNext()) { - final int id = iterator.next(); - if (mainIntersection == null && (idChecker == null || idChecker.contains(id)) || - mainIntersection != null && mainIntersection.contains(id) - ) { - copy.add(id); - } - } - } - else { - mainIntersection.forEach(new TIntProcedure() { - final IntPredicate predicate = valueIt.getValueAssociationPredicate(); - - @Override - public boolean execute(int id) { - if (predicate.contains(id)) copy.add(id); - return true; - } - }); - } - } - - mainIntersection = copy; - if (mainIntersection.isEmpty()) { - return new TIntHashSet(); - } - } - - return mainIntersection; - } - - - @NotNull - public static ValueContainer.IntIterator collectInputIdsContainingAllKeys(@NotNull InvertedIndex index, - @NotNull Collection dataKeys) - throws StorageException { - TIntHashSet result = collectInputIdsContainingAllKeys(index, dataKeys, null, null); - if (result == null) return TroveSetIntIterator.EMPTY; - return new TroveSetIntIterator(result); - } - private static boolean processVirtualFiles(@NotNull TIntHashSet ids, @NotNull final GlobalSearchScope filter, @NotNull final Processor processor) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java b/platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java deleted file mode 100644 index e4dcbfddfe0b..000000000000 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapDiffUpdateData.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2000-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.indexing; - -import com.intellij.openapi.application.ApplicationManager; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.Comparing; -import com.intellij.openapi.util.Ref; -import com.intellij.util.SystemProperties; -import gnu.trove.THashMap; -import gnu.trove.TObjectObjectProcedure; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -public abstract class MapDiffUpdateData extends UpdateData { - public static boolean ourDiffUpdateEnabled = SystemProperties.getBooleanProperty("idea.disable.diff.index.update", true); - - private Map removedOrChangedKeys; - private Map addedKeys; - - public MapDiffUpdateData(ID indexId) { - super(indexId); - } - - public static void iterateAddedKeyAndValues(final int inputId, - final AddedKeyProcessor consumer, - Map data) throws StorageException { - if (data instanceof THashMap) { - // such map often (from IdIndex) contain 100x (avg ~240) of entries, also THashMap have no Entry inside so we optimize for gc too - final Ref exceptionRef = new Ref<>(); - final boolean b = ((THashMap)data).forEachEntry(new TObjectObjectProcedure() { - @Override - public boolean execute(Key key, Value value) { - try { - consumer.process(key, value, inputId); - } - catch (StorageException ex) { - exceptionRef.set(ex); - return false; - } - return true; - } - }); - if (!b) throw exceptionRef.get(); - } - else { - for (Map.Entry entry : data.entrySet()) { - consumer.process(entry.getKey(), entry.getValue(), inputId); - } - } - } - - public static void iterateRemovedKeys(Collection keyCollection, int inputId, - RemovedOrUpdatedKeyProcessor consumer) throws StorageException { - for (Key key : keyCollection) { - consumer.process(key, inputId); - } - } - - @Override - public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor consumer) - throws StorageException { - calcDiff(); - iterateRemovedKeys(removedOrChangedKeys.keySet(), inputId, consumer); - } - - private static final boolean DO_INFO_DUMP = ApplicationManager.getApplication().isInternal(); - - private void calcDiff() throws StorageException { - if (removedOrChangedKeys != null) return; - - try { - Map currentValue = getCurrentValue(); - Map newValue = getNewValue(); - - if (!currentValue.isEmpty()) { - if (newValue.isEmpty()) { - // removal from index - addedKeys = newValue; - removedOrChangedKeys = currentValue; - return; - } - for (Map.Entry e : currentValue.entrySet()) { - Value newValueForKey = newValue.get(e.getKey()); - - if (!Comparing.equal(newValueForKey, e.getValue()) || - newValueForKey == null && !newValue.containsKey(e.getKey()) - ) { - if (removedOrChangedKeys == null) removedOrChangedKeys = new THashMap<>(); - removedOrChangedKeys.put(e.getKey(), e.getValue()); - if (newValue.containsKey(e.getKey())) { - if (addedKeys == null) addedKeys = new THashMap<>(); - addedKeys.put(e.getKey(), newValueForKey); - } - } - } - } - else { - if (newValue.isEmpty()) { - // before and after map are empty - addedKeys = newValue; - removedOrChangedKeys = currentValue; - return; - } - } - - if (!newValue.isEmpty()) { - if (currentValue.isEmpty()) { - // initial indexing - addedKeys = newValue; - removedOrChangedKeys = currentValue; - return; - } - for (Map.Entry e : newValue.entrySet()) { - if (!currentValue.containsKey(e.getKey())) { - if (addedKeys == null) addedKeys = new THashMap<>(); - addedKeys.put(e.getKey(), e.getValue()); - } - } - } - - if (removedOrChangedKeys == null) removedOrChangedKeys = Collections.emptyMap(); - if (addedKeys == null) addedKeys = Collections.emptyMap(); - - int totalRequests = requests.incrementAndGet(); - totalRemovals.addAndGet(currentValue.size()); - totalAdditions.addAndGet(newValue.size()); - incrementalAdditions.addAndGet(addedKeys.size()); - incrementalRemovals.addAndGet(removedOrChangedKeys.size()); - - if ((totalRequests & 0xFFF) == 0 && DO_INFO_DUMP) { - Logger.getInstance(getClass()).info("Incremental index diff update:"+requests + - ", removals:" + totalRemovals + "->" + incrementalRemovals + - ", additions:" +totalAdditions + "->" +incrementalAdditions); - } - //if (removedOrChangedKeys.size() != currentValue.size() || - // addedKeys.size() != newValue.size() - // ) { - // int a = 1; // your breakpoint can be here - //} - } - catch (IOException e) { - throw new StorageException(e); - } - } - - private static final AtomicInteger requests = new AtomicInteger(); - private static final AtomicInteger totalRemovals = new AtomicInteger(); - private static final AtomicInteger totalAdditions = new AtomicInteger(); - private static final AtomicInteger incrementalRemovals = new AtomicInteger(); - private static final AtomicInteger incrementalAdditions = new AtomicInteger(); - - protected abstract Map getNewValue(); - - protected abstract Map getCurrentValue() throws IOException; - - @Override - public void iterateAddedKeys(int inputId, AddedKeyProcessor consumer) throws StorageException { - calcDiff(); - iterateAddedKeyAndValues(inputId, consumer, addedKeys); - } -} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java deleted file mode 100644 index 186897bf1aeb..000000000000 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ /dev/null @@ -1,994 +0,0 @@ -/* - * Copyright 2000-2014 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.intellij.util.indexing; - -import com.intellij.openapi.application.Application; -import com.intellij.openapi.application.ApplicationManager; -import com.intellij.openapi.application.ModalityState; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.editor.Document; -import com.intellij.openapi.fileEditor.FileDocumentManager; -import com.intellij.openapi.fileTypes.FileType; -import com.intellij.openapi.progress.ProcessCanceledException; -import com.intellij.openapi.progress.ProgressManager; -import com.intellij.openapi.util.*; -import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; -import com.intellij.openapi.util.io.ByteSequence; -import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; -import com.intellij.psi.PsiDocumentManager; -import com.intellij.psi.PsiFile; -import com.intellij.psi.impl.cache.impl.id.IdIndex; -import com.intellij.psi.search.GlobalSearchScope; -import com.intellij.util.*; -import com.intellij.util.io.*; -import com.intellij.util.io.DataOutputStream; -import gnu.trove.THashMap; -import gnu.trove.TIntObjectHashMap; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.io.*; -import java.nio.charset.Charset; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * @author Eugene Zhuravlev - * Date: Dec 10, 2007 - */ -public class MapReduceIndex implements UpdatableIndex { - private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex"); - private static final int NULL_MAPPING = 0; - @NotNull private final ID myIndexId; - private final DataIndexer myIndexer; - @NotNull protected final IndexStorage myStorage; - private final boolean myHasSnapshotMapping; - - private final DataExternalizer myValueExternalizer; - private final DataExternalizer> mySnapshotIndexExternalizer; - private final boolean myIsPsiBackedIndex; - private final IndexExtension myExtension; - private final AtomicBoolean myInMemoryMode = new AtomicBoolean(); - private final AtomicLong myModificationStamp = new AtomicLong(); - private final TIntObjectHashMap> myInMemoryKeys = new TIntObjectHashMap<>(); - - private PersistentHashMap myContents; - private PersistentHashMap myInputsSnapshotMapping; - @Nullable protected PersistentHashMap> myInputsIndex; - private PersistentHashMap myIndexingTrace; - private volatile boolean myDisposed; - - private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); - - private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() { - @Override - public void run() { - try { - if (myStorage instanceof MemoryIndexStorage) { - Lock writeLock = getWriteLock(); - if (writeLock.tryLock()) { - try { - ((MemoryIndexStorage)myStorage).clearCaches(); - } finally { - writeLock.unlock(); - } - } - } - flush(); - } catch (StorageException e) { - LOG.info(e); - requestRebuild(null); - } - } - }); - - public MapReduceIndex(IndexExtension extension, - @NotNull IndexStorage storage) throws IOException { - myIndexId = extension.getName(); - myExtension = extension; - SharedIndicesData.registerIndex(myIndexId, extension); - myIndexer = extension.getIndexer(); - myStorage = storage; - myHasSnapshotMapping = extension instanceof FileBasedIndexExtension && - ((FileBasedIndexExtension)extension).hasSnapshotMapping() && - IdIndex.ourSnapshotMappingsEnabled; - - mySnapshotIndexExternalizer = createInputsIndexExternalizer(extension, myIndexId, extension.getKeyDescriptor()); - myValueExternalizer = extension.getValueExternalizer(); - myIsPsiBackedIndex = extension instanceof PsiDependentIndex; - - myContents = createContentsIndex(); // todo - - if (!SharedIndicesData.ourFileSharedIndicesEnabled || SharedIndicesData.DO_CHECKS) { - if (myHasSnapshotMapping) { - myInputsSnapshotMapping = createInputSnapshotMapping(); - } - else { - myInputsIndex = createInputsIndex(); - } - } - - if (DebugAssertions.EXTRA_SANITY_CHECKS && myHasSnapshotMapping) { - myIndexingTrace = createIndexingTrace(); - } - - if (storage instanceof MemoryIndexStorage) { - ((MemoryIndexStorage)storage).addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() { - @Override - public void bufferingStateChanged(boolean newState) { - myInMemoryMode.set(newState); - } - - @Override - public void memoryStorageCleared() { - synchronized (myInMemoryKeys) { - myInMemoryKeys.clear(); - } - } - }); - } - } - - private static DataExternalizer> createInputsIndexExternalizer(IndexExtension extension, - ID indexId, - KeyDescriptor keyDescriptor) { - DataExternalizer> externalizer; - if (extension instanceof CustomInputsIndexFileBasedIndexExtension) { - externalizer = ((CustomInputsIndexFileBasedIndexExtension)extension).createExternalizer(); - } else { - externalizer = new InputIndexDataExternalizer<>(keyDescriptor, indexId); - } - return externalizer; - } - - @NotNull - private static PersistentHashMap> createIdToDataKeysIndex(@NotNull IndexExtension extension, - @NotNull MemoryIndexStorage storage) - throws IOException { - ID indexId = extension.getName(); - KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); - final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(indexId); - - return new PersistentHashMap<>( - indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, createInputsIndexExternalizer(extension, indexId, keyDescriptor) - ); - } - - private PersistentHashMap createContentsIndex() throws IOException { - final File saved = myHasSnapshotMapping ? new File(IndexInfrastructure.getPersistentIndexRootDir(myIndexId), "values") : null; - - if (saved != null) { - try { - return new PersistentHashMap<>(saved, EnumeratorIntegerDescriptor.INSTANCE, ByteSequenceDataExternalizer.INSTANCE); - } catch (IOException ex) { - IOUtil.deleteAllFilesStartingWith(saved); - throw ex; - } - } else { - return null; - } - } - - @NotNull - public IndexStorage getStorage() { - return myStorage; - } - - @Override - public void clear() throws StorageException { - try { - getWriteLock().lock(); - myStorage.clear(); - if (myInputsIndex != null) { - cleanMapping(myInputsIndex); - myInputsIndex = createInputsIndex(); - } - if (myInputsSnapshotMapping != null) { - cleanMapping(myInputsSnapshotMapping); - myInputsSnapshotMapping = createInputSnapshotMapping(); - } - if (myIndexingTrace != null) { - cleanMapping(myIndexingTrace); - myIndexingTrace = createIndexingTrace(); - } - if (myContents != null) { - cleanMapping(myContents); - myContents = createContentsIndex(); - } - } - catch (StorageException e) { - LOG.error(e); - } - catch (IOException e) { - LOG.error(e); - } - finally { - getWriteLock().unlock(); - } - } - - private PersistentHashMap createInputSnapshotMapping() throws IOException { - final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId"); - try { - return new PersistentHashMap(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE, - EnumeratorIntegerDescriptor.INSTANCE, 4096) { - @Override - protected boolean wantNonnegativeIntegralValues() { - return true; - } - }; - } - catch (IOException ex) { - IOUtil.deleteAllFilesStartingWith(fileIdToHashIdFile); - throw ex; - } - } - - private PersistentHashMap createIndexingTrace() throws IOException { - final File mapFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "indextrace"); - try { - return new PersistentHashMap<>(mapFile, EnumeratorIntegerDescriptor.INSTANCE, - new DataExternalizer() { - @Override - public void save(@NotNull DataOutput out, String value) throws IOException { - out.write((byte[])CompressionUtil.compressCharSequence(value, Charset.defaultCharset())); - } - - @Override - public String read(@NotNull DataInput in) throws IOException { - byte[] b = new byte[((InputStream)in).available()]; - in.readFully(b); - return (String)CompressionUtil.uncompressCharSequence(b, Charset.defaultCharset()); - } - }, 4096); - } - catch (IOException ex) { - IOUtil.deleteAllFilesStartingWith(mapFile); - throw ex; - } - } - - private static void cleanMapping(@NotNull PersistentHashMap index) { - final File baseFile = index.getBaseFile(); - try { - index.close(); - } - catch (Throwable ignored) { - } - - IOUtil.deleteAllFilesStartingWith(baseFile); - } - - @Override - public void flush() throws StorageException{ - try { - getReadLock().lock(); - doForce(myInputsIndex); - doForce(myInputsSnapshotMapping); - doForce(myIndexingTrace); - doForce(myContents); - myStorage.flush(); - } - catch (IOException e) { - throw new StorageException(e); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof StorageException || cause instanceof IOException) { - throw new StorageException(cause); - } - else { - throw e; - } - } - finally { - getReadLock().unlock(); - } - } - - private static void doForce(@Nullable PersistentHashMap inputsIndex) { - if (inputsIndex != null && inputsIndex.isDirty()) { - inputsIndex.force(); - } - } - - @Override - public void dispose() { - myLowMemoryFlusher.stop(); - final Lock lock = getWriteLock(); - try { - lock.lock(); - try { - myStorage.close(); - } - finally { - doClose(myInputsIndex); - doClose(myInputsSnapshotMapping); - doClose(myIndexingTrace); - doClose(myContents); - } - } - catch (StorageException e) { - LOG.error(e); - } - finally { - myDisposed = true; - lock.unlock(); - } - } - - @Override - public void setIndexedStateForFile(int fileId, @NotNull VirtualFile file) { - IndexingStamp.setFileIndexedStateCurrent(fileId, myIndexId); - } - - @Override - public void resetIndexedStateForFile(int fileId) { - IndexingStamp.setFileIndexedStateOutdated(fileId, myIndexId); - } - - @Override - public boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file) { - return IndexingStamp.isFileIndexedStateCurrent(fileId, myIndexId); - } - - private static void doClose(@Nullable PersistentHashMap index) { - if (index != null) { - try { - index.close(); - } - catch (IOException e) { - LOG.error(e); - } - } - } - - @NotNull - @Override - public final Lock getReadLock() { - return myLock.readLock(); - } - - @NotNull - @Override - public final Lock getWriteLock() { - return myLock.writeLock(); - } - - @Override - public boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, IdFilter idFilter) throws StorageException { - final Lock lock = getReadLock(); - try { - lock.lock(); - return myStorage.processKeys(processor, scope, idFilter); - } - finally { - lock.unlock(); - } - } - - @Override - @NotNull - public ValueContainer getData(@NotNull final Key key) throws StorageException { - final Lock lock = getReadLock(); - try { - lock.lock(); - if (myDisposed) { - return new ValueContainerImpl<>(); - } - ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); - return myStorage.read(key); - } - finally { - ValueContainerImpl.ourDebugIndexInfo.set(null); - lock.unlock(); - } - } - - protected PersistentHashMap> createInputsIndex() throws IOException { - return createIdToDataKeysIndex(myExtension, (MemoryIndexStorage)myStorage); - } - - private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true); - - @NotNull - @Override - public final Computable update(final int inputId, @Nullable Input content) { - final boolean weProcessPhysicalContent = content == null || - (content instanceof UserDataHolder && - FileBasedIndexImpl.ourPhysicalContentKey.get((UserDataHolder)content, Boolean.FALSE)); - - Map data = null; - boolean havePersistentData = false; - Integer hashId = null; - boolean skippedReadingPersistentDataButMayHaveIt = false; - - if (myContents != null && weProcessPhysicalContent && content != null) { - try { - FileContent fileContent = (FileContent)content; - hashId = getHashOfContent(fileContent); - if (doReadSavedPersistentData) { - if (!myContents.isBusyReading() || DebugAssertions.EXTRA_SANITY_CHECKS) { // avoid blocking read, we can calculate index value - ByteSequence bytes = readContents(hashId); - - if (bytes != null) { - data = deserializeSavedPersistentData(bytes); - havePersistentData = true; - if (DebugAssertions.EXTRA_SANITY_CHECKS) { - Map contentData = myIndexer.map(content); - boolean sameValueForSavedIndexedResultAndCurrentOne = contentData.equals(data); - if (!sameValueForSavedIndexedResultAndCurrentOne) { - DebugAssertions.error( - "Unexpected difference in indexing of %s by index %s, file type %s, charset %s\ndiff %s\nprevious indexed info %s", - fileContent.getFile(), - myIndexId, - fileContent.getFileType().getName(), - ((FileContentImpl)fileContent).getCharset(), - buildDiff(data, contentData), - myIndexingTrace.get(hashId) - ); - } - } - } - } else { - skippedReadingPersistentDataButMayHaveIt = true; - } - } else { - havePersistentData = myContents.containsMapping(hashId); - } - } catch (IOException ex) { - // todo: - throw new RuntimeException(ex); - } - } - - if (data == null) { - data = content != null ? myIndexer.map(content) : Collections.emptyMap(); - if (DebugAssertions.DEBUG) { - checkValuesHaveProperEqualsAndHashCode(data); - } - } - - if (hashId != null && !havePersistentData) { - boolean saved = savePersistentData(data, hashId, skippedReadingPersistentDataButMayHaveIt); - if (DebugAssertions.EXTRA_SANITY_CHECKS) { - if (saved) { - - FileContent fileContent = (FileContent)content; - try { - myIndexingTrace.put(hashId, ((FileContentImpl)fileContent).getCharset() + "," + fileContent.getFileType().getName()+"," + fileContent.getFile().getPath() + "," + - ExceptionUtil.getThrowableText(new Throwable())); - } catch (IOException ex) { - LOG.error(ex); - } - } - } - } - ProgressManager.checkCanceled(); - - UpdateData optimizedUpdateData = null; - final NotNullComputable> oldKeysGetter; - final int savedInputId; - if (myHasSnapshotMapping) { - try { - final NotNullComputable> keysForGivenInputId = () -> { - try { - Integer currentHashId = readInputHashId(inputId); - Collection currentKeys; - if (currentHashId != null) { - ByteSequence byteSequence = readContents(currentHashId); - currentKeys = byteSequence != null ? deserializeSavedPersistentData(byteSequence).keySet() : Collections.emptyList(); - } - else { - currentKeys = Collections.emptyList(); - } - - return currentKeys; - } - catch (IOException e) { - throw new RuntimeException(e); - } - }; - - if (weProcessPhysicalContent) { - if (content instanceof FileContent) { - savedInputId = getHashOfContent((FileContent)content); - } - else { - savedInputId = NULL_MAPPING; - } - oldKeysGetter = keysForGivenInputId; - - if (MapDiffUpdateData.ourDiffUpdateEnabled) { - final Map newValue = data; - optimizedUpdateData = new MapDiffUpdateData(myIndexId) { - @Override - protected Map getNewValue() { - return newValue; - } - - @Override - protected Map getCurrentValue() throws IOException { - Integer currentHashId = readInputHashId(inputId); - Map currentValue; - if (currentHashId != null) { - ByteSequence byteSequence = readContents(currentHashId); - currentValue = byteSequence != null ? deserializeSavedPersistentData(byteSequence) : Collections.emptyMap(); - } - else { - currentValue = Collections.emptyMap(); - } - return currentValue; - } - - @Override - public void save(int inputId) throws IOException { - saveInputHashId(inputId, savedInputId); - } - }; - } - } else { - oldKeysGetter = () -> { - try { - Collection oldKeys = readInputKeys(inputId); - if (oldKeys == null) { - return keysForGivenInputId.compute(); - } - return oldKeys; - } - catch (IOException e) { - throw new RuntimeException(e); - } - }; - savedInputId = NULL_MAPPING; - } - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } else { - oldKeysGetter = () -> { - try { - Collection oldKeys = readInputKeys(inputId); - return oldKeys == null? Collections.emptyList() : oldKeys; - } - catch (IOException e) { - throw new RuntimeException(e); - } - }; - savedInputId = inputId; - } - - // do not depend on content! - final UpdateData updateData = optimizedUpdateData != null ? optimizedUpdateData : buildUpdateData(data, oldKeysGetter, savedInputId); - return () -> { - - try { - updateWithMap(inputId, updateData); - } - catch (StorageException|ProcessCanceledException ex) { - LOG.info("Exception during updateWithMap:" + ex); - Application application = ApplicationManager.getApplication(); - if (application.isUnitTestMode() || application.isHeadlessEnvironment()) { - // avoid deadlock due to synchronous update in DumbServiceImpl#queueTask - application.invokeLater(() -> requestRebuild(ex), ModalityState.any()); - } else { - requestRebuild(ex); - } - return Boolean.FALSE; - } - - return Boolean.TRUE; - }; - } - - protected void requestRebuild(@Nullable Exception ex) { - if (ex == null) { - FileBasedIndex.getInstance().requestRebuild(myIndexId); - } - else { - FileBasedIndex.getInstance().requestRebuild(myIndexId, ex); - } - } - - protected UpdateData buildUpdateData(Map data, NotNullComputable> oldKeysGetter, int savedInputId) { - return new SimpleUpdateData(myIndexId, savedInputId, data, oldKeysGetter); - } - - private ByteSequence readContents(Integer hashId) throws IOException { - if (SharedIndicesData.ourFileSharedIndicesEnabled) { - if (SharedIndicesData.DO_CHECKS) { - synchronized (myContents) { - ByteSequence contentBytes = SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE); - ByteSequence contentBytesFromContents = myContents.get(hashId); - - if ((contentBytes == null && contentBytesFromContents != null) || - !Comparing.equal(contentBytesFromContents, contentBytes)) { - SharedIndicesData.associateContentData(hashId, myIndexId, contentBytesFromContents, ByteSequenceDataExternalizer.INSTANCE); - if (contentBytes != null) { - LOG.error("Unexpected indexing diff with hashid " + myIndexId + "," + hashId); - } - contentBytes = contentBytesFromContents; - } - return contentBytes; - } - } else { - return SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE); - } - } - - return myContents.get(hashId); - } - - private void saveContents(int id, BufferExposingByteArrayOutputStream out) throws IOException { - ByteSequence byteSequence = new ByteSequence(out.getInternalBuffer(), 0, out.size()); - if (SharedIndicesData.ourFileSharedIndicesEnabled) { - if (SharedIndicesData.DO_CHECKS) { - synchronized (myContents) { - myContents.put(id, byteSequence); - SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE); - } - } else { - SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE); - } - } else { - myContents.put(id, byteSequence); - } - } - - private Integer readInputHashId(int inputId) throws IOException { - if (SharedIndicesData.ourFileSharedIndicesEnabled) { - Integer hashId = SharedIndicesData.recallFileData(inputId, myIndexId, EnumeratorIntegerDescriptor.INSTANCE); - if (hashId == null) hashId = 0; - if (myInputsSnapshotMapping == null) return hashId; - - Integer hashIdFromInputSnapshotMapping = myInputsSnapshotMapping.get(inputId); - if ((hashId == 0 && hashIdFromInputSnapshotMapping != 0) || - !Comparing.equal(hashIdFromInputSnapshotMapping, hashId)) { - SharedIndicesData.associateFileData(inputId, myIndexId, hashIdFromInputSnapshotMapping, - EnumeratorIntegerDescriptor.INSTANCE); - if (hashId != 0) { - LOG.error("Unexpected indexing diff with hashid " + myIndexId + ", file:" + IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId) - + "," + hashIdFromInputSnapshotMapping + "," + hashId); - } - hashId = hashIdFromInputSnapshotMapping; - } - return hashId; - } - return myInputsSnapshotMapping.get(inputId); - } - - private void saveInputHashId(int inputId, int savedInputId) throws IOException { - if (SharedIndicesData.ourFileSharedIndicesEnabled) { - SharedIndicesData.associateFileData(inputId, myIndexId, savedInputId, EnumeratorIntegerDescriptor.INSTANCE); - } - - if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.put(inputId, savedInputId); - } - - private Collection readInputKeys(int inputId) throws IOException { - if (myInMemoryMode.get()) { - synchronized (myInMemoryKeys) { - Collection keys = myInMemoryKeys.get(inputId); - if (keys != null) { - return keys; - } - } - } - if (myHasSnapshotMapping) { - return null; - } - - if (SharedIndicesData.ourFileSharedIndicesEnabled) { - Collection keys = SharedIndicesData.recallFileData(inputId, myIndexId, mySnapshotIndexExternalizer); - if (myInputsIndex != null) { - Collection keysFromInputsIndex = myInputsIndex.get(inputId); - - if ((keys == null && keysFromInputsIndex != null) || - !DebugAssertions.equals(keysFromInputsIndex, keys, myExtension.getKeyDescriptor()) - ) { - SharedIndicesData.associateFileData(inputId, myIndexId, keysFromInputsIndex, mySnapshotIndexExternalizer); - if (keys != null) { - DebugAssertions.error( - "Unexpected indexing diff " + myIndexId + ", file:" + IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId) - + "," + keysFromInputsIndex + "," + keys); - } - keys = keysFromInputsIndex; - } - } - return keys; - } - return myInputsIndex != null ? myInputsIndex.get(inputId) : null; - } - - private void saveInputKeys(int inputId, int savedInputId, Map newData) throws IOException { - if (myInMemoryMode.get()) { - synchronized (myInMemoryKeys) { - myInMemoryKeys.put(inputId, newData.keySet()); - } - } else { - if (myHasSnapshotMapping) { - saveInputHashId(inputId, savedInputId); - } else { - if (myInputsIndex != null) { - if (newData.size() > 0) { - myInputsIndex.put(inputId, newData.keySet()); - } - else { - myInputsIndex.remove(inputId); - } - } - - if (SharedIndicesData.ourFileSharedIndicesEnabled) { - Set newKeys = newData.keySet(); - if (newKeys.size() == 0) newKeys = null; - SharedIndicesData.associateFileData(inputId, myIndexId, newKeys, mySnapshotIndexExternalizer); - } - } - } - } - - private void checkValuesHaveProperEqualsAndHashCode(Map data) { - for(Map.Entry e: data.entrySet()) { - final Value value = e.getValue(); - if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) { - LOG.error("Index " + myIndexId.toString() + " violates equals / hashCode contract for Value parameter"); - } - - if (myValueExternalizer != null) { - try { - final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(); - DataOutputStream outputStream = new DataOutputStream(out); - myValueExternalizer.save(outputStream, value); - outputStream.close(); - final Value deserializedValue = - myValueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size()))); - - if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) { - LOG.error("Index " + myIndexId.toString() + " deserialization violates equals / hashCode contract for Value parameter"); - } - } catch (IOException ex) { - LOG.error(ex); - } - } - } - } - - private StringBuilder buildDiff(Map data, Map contentData) { - StringBuilder moreInfo = new StringBuilder(); - if (contentData.size() != data.size()) { - moreInfo.append("Indexer has different number of elements, previously ").append(data.size()).append(" after ") - .append(contentData.size()).append("\n"); - } else { - moreInfo.append("total ").append(contentData.size()).append(" entries\n"); - } - - for(Map.Entry keyValueEntry:contentData.entrySet()) { - if (!data.containsKey(keyValueEntry.getKey())) { - moreInfo.append("Previous data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n"); - } - else { - Value value = data.get(keyValueEntry.getKey()); - if (!Comparing.equal(keyValueEntry.getValue(), value)) { - moreInfo.append("Previous data has different value for key:").append(keyValueEntry.getKey()).append( ", new value ").append(keyValueEntry.getValue()).append( ", oldValue:").append(value).append("\n"); - } - } - } - - for(Map.Entry keyValueEntry:data.entrySet()) { - if (!contentData.containsKey(keyValueEntry.getKey())) { - moreInfo.append("New data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n"); - } - else { - Value value = contentData.get(keyValueEntry.getKey()); - if (!Comparing.equal(keyValueEntry.getValue(), value)) { - moreInfo.append("New data has different value for key:").append(keyValueEntry.getKey()).append( " new value ").append(value).append( ", oldValue:").append(keyValueEntry.getValue()).append("\n"); - } - } - } - return moreInfo; - } - - private Map deserializeSavedPersistentData(ByteSequence bytes) throws IOException { - DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes.getBytes(), bytes.getOffset(), bytes.getLength())); - int pairs = DataInputOutputUtil.readINT(stream); - if (pairs == 0) return Collections.emptyMap(); - Map result = new THashMap<>(pairs); - while (stream.available() > 0) { - Value value = myValueExternalizer.read(stream); - Collection keys = mySnapshotIndexExternalizer.read(stream); - for(Key k:keys) result.put(k, value); - } - return result; - } - - private Integer getHashOfContent(FileContent content) throws IOException { - FileType fileType = content.getFileType(); - if (myIsPsiBackedIndex && myHasSnapshotMapping && content instanceof FileContentImpl) { - // psi backed index should use existing psi to build index value (FileContentImpl.getPsiFileForPsiDependentIndex()) - // so we should use different bytes to calculate hash(Id) - Integer previouslyCalculatedUncommittedHashId = content.getUserData(ourSavedUncommittedHashIdKey); - - if (previouslyCalculatedUncommittedHashId == null) { - Document document = FileDocumentManager.getInstance().getCachedDocument(content.getFile()); - - if (document != null) { // if document is not committed - PsiDocumentManager psiDocumentManager = PsiDocumentManager.getInstance(content.getProject()); - - if (psiDocumentManager.isUncommited(document)) { - PsiFile file = psiDocumentManager.getCachedPsiFile(document); - Charset charset = ((FileContentImpl)content).getCharset(); - - if (file != null) { - previouslyCalculatedUncommittedHashId = ContentHashesSupport - .calcContentHashIdWithFileType(file.getText().getBytes(charset), charset, - fileType); - content.putUserData(ourSavedUncommittedHashIdKey, previouslyCalculatedUncommittedHashId); - } - } - } - } - if (previouslyCalculatedUncommittedHashId != null) return previouslyCalculatedUncommittedHashId; - } - - Integer previouslyCalculatedContentHashId = content.getUserData(ourSavedContentHashIdKey); - if (previouslyCalculatedContentHashId == null) { - byte[] hash = content instanceof FileContentImpl ? ((FileContentImpl)content).getHash():null; - if (hash == null) { - if (fileType.isBinary()) { - previouslyCalculatedContentHashId = ContentHashesSupport.calcContentHashId(content.getContent(), fileType); - } else { - Charset charset = content instanceof FileContentImpl ? ((FileContentImpl)content).getCharset() : null; - previouslyCalculatedContentHashId = ContentHashesSupport - .calcContentHashIdWithFileType(content.getContent(), charset, fileType); - } - } else { - previouslyCalculatedContentHashId = ContentHashesSupport.enumerateHash(hash); - } - content.putUserData(ourSavedContentHashIdKey, previouslyCalculatedContentHashId); - } - return previouslyCalculatedContentHashId; - } - - private static final ThreadLocalCachedByteArray ourSpareByteArray = new ThreadLocalCachedByteArray(); - - private boolean savePersistentData(Map data, int id, boolean delayedReading) { - try { - if (delayedReading && myContents.containsMapping(id)) return false; - BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(ourSpareByteArray.getBuffer(4 * data.size())); - DataOutputStream stream = new DataOutputStream(out); - int size = data.size(); - DataInputOutputUtil.writeINT(stream, size); - - if (size > 0) { - THashMap> values = new THashMap<>(); - List keysForNullValue = null; - for (Map.Entry e : data.entrySet()) { - Value value = e.getValue(); - - List keys = value != null ? values.get(value):keysForNullValue; - if (keys == null) { - if (value != null) values.put(value, keys = new SmartList<>()); - else keys = keysForNullValue = new SmartList<>(); - } - keys.add(e.getKey()); - } - - if (keysForNullValue != null) { - myValueExternalizer.save(stream, null); - mySnapshotIndexExternalizer.save(stream, keysForNullValue); - } - - for(Value value:values.keySet()) { - myValueExternalizer.save(stream, value); - mySnapshotIndexExternalizer.save(stream, values.get(value)); - } - } - - saveContents(id, out); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - return true; - } - - private static final com.intellij.openapi.util.Key ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id"); - private static final com.intellij.openapi.util.Key ourSavedUncommittedHashIdKey = com.intellij.openapi.util.Key.create("saved.uncommitted.hash.id"); - - public IndexExtension getExtension() { - return myExtension; - } - - public long getModificationStamp() { - return myModificationStamp.get(); - } - - public class SimpleUpdateData extends UpdateData { - private final int savedInputId; - private final @NotNull Map newData; - protected final @NotNull NotNullComputable> oldKeysGetter; - - public SimpleUpdateData(ID indexId, int id, @NotNull Map data, @NotNull NotNullComputable> getter) { - super(indexId); - savedInputId = id; - newData = data; - oldKeysGetter = getter; - } - - public void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor consumer) throws StorageException { - MapDiffUpdateData.iterateRemovedKeys(oldKeysGetter.compute(), inputId, consumer); - } - - public void iterateAddedKeys(final int inputId, final AddedKeyProcessor consumer) throws StorageException { - MapDiffUpdateData.iterateAddedKeyAndValues(inputId, consumer, newData); - } - - @Override - public void save(int inputId) throws IOException { - saveInputKeys(inputId, savedInputId, newData); - } - - public @NotNull Map getNewData() { - return newData; - } - } - - private final MapDiffUpdateData.RemovedOrUpdatedKeyProcessor - myRemoveStaleKeyOperation = new MapDiffUpdateData.RemovedOrUpdatedKeyProcessor() { - @Override - public void process(Key key, int inputId) throws StorageException { - myModificationStamp.incrementAndGet(); - myStorage.removeAllValues(key, inputId); - } - }; - - private final MapDiffUpdateData.AddedKeyProcessor myAddedKeyProcessor = new MapDiffUpdateData.AddedKeyProcessor() { - @Override - public void process(Key key, Value value, int inputId) throws StorageException { - myModificationStamp.incrementAndGet(); - myStorage.addValue(key, inputId, value); - } - }; - - protected void updateWithMap(final int inputId, - @NotNull UpdateData updateData) throws StorageException { - getWriteLock().lock(); - try { - try { - ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); - updateData.iterateRemovedOrUpdatedKeys(inputId, myRemoveStaleKeyOperation); - updateData.iterateAddedKeys(inputId, myAddedKeyProcessor); - updateData.save(inputId); - } - catch (ProcessCanceledException pce) { - throw pce; // extra care - } - catch (Throwable e) { // e.g. IOException, AssertionError - throw new StorageException(e); - } - finally { - ValueContainerImpl.ourDebugIndexInfo.set(null); - } - } - finally { - getWriteLock().unlock(); - } - } -} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java index 858d93c09a4a..8ebced5856b4 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java @@ -18,8 +18,11 @@ package com.intellij.util.indexing; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; -import com.intellij.util.Processors; import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.impl.ChangeTrackingValueContainer; +import com.intellij.util.indexing.impl.DebugAssertions; +import com.intellij.util.indexing.impl.IndexStorage; +import com.intellij.util.indexing.impl.UpdatableValueContainer; import org.jetbrains.annotations.NotNull; import java.io.IOException; @@ -31,7 +34,7 @@ import java.util.*; * @author Eugene Zhuravlev * Date: Dec 10, 2007 */ -public class MemoryIndexStorage implements IndexStorage { +public class MemoryIndexStorage implements VfsAwareIndexStorage { private final Map> myMap = new HashMap<>(); @NotNull private final IndexStorage myBackendStorage; @@ -91,7 +94,8 @@ public class MemoryIndexStorage implements IndexStorage } } - void clearCaches() { + @Override + public void clearCaches() { if (myMap.size() == 0) return; if (DebugAssertions.DEBUG) { @@ -120,14 +124,6 @@ public class MemoryIndexStorage implements IndexStorage myBackendStorage.flush(); } - @NotNull - @Override - public Collection getKeys() throws StorageException { - final Set keys = new HashSet<>(); - processKeys(Processors.cancelableCollectProcessor(keys), null, null); - return keys; - } - @Override public boolean processKeys(@NotNull final Processor processor, GlobalSearchScope scope, IdFilter idFilter) throws StorageException { final Set stopList = new HashSet<>(); @@ -148,7 +144,7 @@ public class MemoryIndexStorage implements IndexStorage } stopList.add(key); } - return myBackendStorage.processKeys(stopList.isEmpty() && myMap.isEmpty() ? processor : decoratingProcessor, scope, idFilter); + return ((VfsAwareIndexStorage) myBackendStorage).processKeys(stopList.isEmpty() && myMap.isEmpty() ? processor : decoratingProcessor, scope, idFilter); } @Override diff --git a/platform/lang-impl/src/com/intellij/util/indexing/SharedIndicesData.java b/platform/lang-impl/src/com/intellij/util/indexing/SharedIndicesData.java index f5756d4e97c7..c39de012dfe4 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/SharedIndicesData.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/SharedIndicesData.java @@ -39,7 +39,7 @@ public class SharedIndicesData { static final boolean ourFileSharedIndicesEnabled = SystemProperties.getBooleanProperty("idea.shared.input.index.enabled", false); static final boolean DO_CHECKS = ourFileSharedIndicesEnabled && SystemProperties.getBooleanProperty("idea.shared.input.index.checked", false); - private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapReduceIndex"); + private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.MapReduceIndex"); static { if (ourFileSharedIndicesEnabled) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java new file mode 100644 index 000000000000..4207206e8b19 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java @@ -0,0 +1,89 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; +import com.intellij.util.indexing.impl.AbstractForwardIndex; +import com.intellij.util.indexing.impl.CollectionInputKeyIterator; +import com.intellij.util.indexing.impl.DebugAssertions; +import com.intellij.util.indexing.impl.MapBasedForwardIndex; +import com.intellij.util.io.DataExternalizer; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +class SharedMapBasedForwardIndex extends AbstractForwardIndex { + private final DataExternalizer> mySnapshotIndexExternalizer; + private MapBasedForwardIndex myUnderlying; + + public SharedMapBasedForwardIndex(MapBasedForwardIndex underlying) { + super(underlying.getIndexExtension()); + myUnderlying = underlying; + mySnapshotIndexExternalizer = VfsAwareMapReduceIndex.createInputsIndexExternalizer(underlying.getIndexExtension()); + } + + @NotNull + @Override + public InputKeyIterator getInputKeys(int inputId) throws IOException { + Collection keys; + if (SharedIndicesData.ourFileSharedIndicesEnabled) { + keys = SharedIndicesData.recallFileData(inputId, myIndexId, mySnapshotIndexExternalizer); + Collection keysFromInputsIndex = myUnderlying.getInputsIndex().get(inputId); + + if ((keys == null && keysFromInputsIndex != null) || + !DebugAssertions.equals(keysFromInputsIndex, keys, myKeyDescriptor) + ) { + SharedIndicesData.associateFileData(inputId, myIndexId, keysFromInputsIndex, mySnapshotIndexExternalizer); + if (keys != null) { + DebugAssertions.error( + "Unexpected indexing diff " + myIndexId + ", file:" + IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId) + + "," + keysFromInputsIndex + "," + keys); + } + keys = keysFromInputsIndex; + } + return new CollectionInputKeyIterator<>(keys); + } + return new CollectionInputKeyIterator<>(myUnderlying.getInputsIndex().get(inputId)); + } + + @Override + public void putInputData(int inputId, @NotNull Map data) + throws IOException { + Collection keySeq = data.keySet(); + myUnderlying.putData(inputId, keySeq); + if (SharedIndicesData.ourFileSharedIndicesEnabled) { + if (keySeq.size() == 0) keySeq = null; + SharedIndicesData.associateFileData(inputId, myIndexId, keySeq, mySnapshotIndexExternalizer); + } + } + + @Override + public void flush() { + myUnderlying.flush(); + } + + @Override + public void clear() throws IOException { + myUnderlying.clear(); + } + + @Override + public void close() throws IOException { + myUnderlying.close(); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java b/platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java new file mode 100644 index 000000000000..d7beac03326c --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java @@ -0,0 +1,489 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.editor.Document; +import com.intellij.openapi.fileEditor.FileDocumentManager; +import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.util.Comparing; +import com.intellij.openapi.util.ThreadLocalCachedByteArray; +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; +import com.intellij.openapi.util.io.ByteSequence; +import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; +import com.intellij.psi.PsiDocumentManager; +import com.intellij.psi.PsiFile; +import com.intellij.util.CompressionUtil; +import com.intellij.util.ExceptionUtil; +import com.intellij.util.SmartList; +import com.intellij.util.SystemProperties; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.impl.DebugAssertions; +import com.intellij.util.indexing.impl.MapReduceIndex; +import com.intellij.util.io.*; +import com.intellij.util.io.DataOutputStream; +import gnu.trove.THashMap; +import org.jetbrains.annotations.NotNull; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.*; +import java.util.stream.Collectors; + +public class SnapshotInputMappings { + private static final Logger LOG = Logger.getInstance(SnapshotInputMappings.class); + private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true); + + private final ID myIndexId; + private final DataExternalizer myValueExternalizer; + private final IndexExtension myIndexExtension; + private final DataIndexer myIndexer; + private volatile PersistentHashMap myContents; + private volatile PersistentHashMap myInputsSnapshotMapping; + private volatile PersistentHashMap myIndexingTrace; + + private final DataExternalizer> mySnapshotIndexExternalizer; + private boolean myIsPsiBackedIndex; + + public SnapshotInputMappings(IndexExtension indexExtension) throws IOException { + myIndexId = indexExtension.getName(); + myIsPsiBackedIndex = indexExtension instanceof PsiDependentIndex; + mySnapshotIndexExternalizer = VfsAwareMapReduceIndex.createInputsIndexExternalizer(indexExtension); + myValueExternalizer = indexExtension.getValueExternalizer(); + myIndexer = indexExtension.getIndexer(); + myIndexExtension = indexExtension; + createMaps(); + } + + @NotNull + public Map readInputKeys(int inputId) throws IOException { + Integer currentHashId = readInputHashId(inputId); + if (currentHashId != null) { + ByteSequence byteSequence = readContents(currentHashId); + if (byteSequence != null) { + return deserializeSavedPersistentData(byteSequence); + } + } + return Collections.emptyMap(); + } + + static class Snapshot { + private final Map myData; + private final int hashId; + + private Snapshot(Map data, int id) { + myData = data; + hashId = id; + } + + public Map getData() { + return myData; + } + + public int getHashId() { + return hashId; + } + } + + @NotNull + Snapshot readPersistentDataOrMap(@NotNull Input content) { + Map data = null; + boolean havePersistentData = false; + int hashId; + boolean skippedReadingPersistentDataButMayHaveIt = false; + + try { + FileContent fileContent = (FileContent)content; + hashId = getHashOfContent(fileContent); + if (doReadSavedPersistentData) { + if (!myContents.isBusyReading() || DebugAssertions.EXTRA_SANITY_CHECKS) { // avoid blocking read, we can calculate index value + ByteSequence bytes = readContents(hashId); + + if (bytes != null) { + data = deserializeSavedPersistentData(bytes); + havePersistentData = true; + if (DebugAssertions.EXTRA_SANITY_CHECKS) { + Map contentData = myIndexer.map(content); + boolean sameValueForSavedIndexedResultAndCurrentOne = contentData.equals(data); + if (!sameValueForSavedIndexedResultAndCurrentOne) { + DebugAssertions.error( + "Unexpected difference in indexing of %s by index %s, file type %s, charset %s\ndiff %s\nprevious indexed info %s", + fileContent.getFile(), + myIndexId, + fileContent.getFileType().getName(), + ((FileContentImpl)fileContent).getCharset(), + buildDiff(data, contentData), + myIndexingTrace.get(hashId) + ); + } + } + } + } + else { + skippedReadingPersistentDataButMayHaveIt = true; + } + } + else { + havePersistentData = myContents.containsMapping(hashId); + } + } + catch (IOException ex) { + // todo: + throw new RuntimeException(ex); + } + + if (data == null) { + data = myIndexer.map(content); + if (DebugAssertions.DEBUG) { + MapReduceIndex.checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer); + } + } + + if (!havePersistentData) { + boolean saved = savePersistentData(data, hashId, skippedReadingPersistentDataButMayHaveIt); + if (DebugAssertions.EXTRA_SANITY_CHECKS) { + if (saved) { + + FileContent fileContent = (FileContent)content; + try { + myIndexingTrace.put(hashId, ((FileContentImpl)fileContent).getCharset() + + "," + + fileContent.getFileType().getName() + + "," + + fileContent.getFile().getPath() + + "," + + ExceptionUtil.getThrowableText(new Throwable())); + } + catch (IOException ex) { + LOG.error(ex); + } + } + } + } + + return new Snapshot<>(data, hashId); + } + + public void putInputHash(int inputId, int hashId) + throws IOException { + try { + if (SharedIndicesData.ourFileSharedIndicesEnabled) { + SharedIndicesData.associateFileData(inputId, myIndexId, hashId, EnumeratorIntegerDescriptor.INSTANCE); + } + if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.put(inputId, hashId); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + public void flush() { + if (myContents != null) myContents.force(); + if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.force(); + if (myIndexingTrace != null) myIndexingTrace.force(); + } + + public void clear() throws IOException { + List baseDirs = ContainerUtil.list(myContents, myIndexingTrace, myInputsSnapshotMapping) + .stream() + .filter(Objects::nonNull) + .map(PersistentHashMap::getBaseFile) + .collect(Collectors.toList()); + try { + close(); + } + catch (Exception e) { + LOG.error(e); + } + baseDirs.forEach(PersistentHashMap::deleteFilesStartingWith); + createMaps(); + } + + public void close() throws IOException { + if (myContents != null) myContents.close(); + if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.close(); + if (myIndexingTrace != null) myIndexingTrace.close(); + } + + private void createMaps() throws IOException { + myContents = createContentsIndex(); + myIndexingTrace = DebugAssertions.EXTRA_SANITY_CHECKS ? createIndexingTrace() : null; + myInputsSnapshotMapping = + !SharedIndicesData.ourFileSharedIndicesEnabled || SharedIndicesData.DO_CHECKS ? createInputSnapshotMapping() : null; + } + + private PersistentHashMap createContentsIndex() throws IOException { + final File saved = new File(IndexInfrastructure.getPersistentIndexRootDir(myIndexId), "values"); + try { + return new PersistentHashMap<>(saved, EnumeratorIntegerDescriptor.INSTANCE, ByteSequenceDataExternalizer.INSTANCE); + } + catch (IOException ex) { + IOUtil.deleteAllFilesStartingWith(saved); + throw ex; + } + } + + private PersistentHashMap createInputSnapshotMapping() throws IOException { + final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId"); + try { + return new PersistentHashMap(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE, + EnumeratorIntegerDescriptor.INSTANCE, 4096) { + @Override + protected boolean wantNonnegativeIntegralValues() { + return true; + } + }; + } + catch (IOException ex) { + IOUtil.deleteAllFilesStartingWith(fileIdToHashIdFile); + throw ex; + } + } + + private PersistentHashMap createIndexingTrace() throws IOException { + final File mapFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "indextrace"); + try { + return new PersistentHashMap<>(mapFile, EnumeratorIntegerDescriptor.INSTANCE, + new DataExternalizer() { + @Override + public void save(@NotNull DataOutput out, String value) throws IOException { + out.write((byte[])CompressionUtil.compressCharSequence(value, Charset.defaultCharset())); + } + + @Override + public String read(@NotNull DataInput in) throws IOException { + byte[] b = new byte[((InputStream)in).available()]; + in.readFully(b); + return (String)CompressionUtil.uncompressCharSequence(b, Charset.defaultCharset()); + } + }, 4096); + } + catch (IOException ex) { + IOUtil.deleteAllFilesStartingWith(mapFile); + throw ex; + } + } + + private Integer readInputHashId(int inputId) throws IOException { + if (SharedIndicesData.ourFileSharedIndicesEnabled) { + Integer hashId = SharedIndicesData.recallFileData(inputId, myIndexId, EnumeratorIntegerDescriptor.INSTANCE); + if (hashId == null) hashId = 0; + if (myInputsSnapshotMapping == null) return hashId; + + Integer hashIdFromInputSnapshotMapping = myInputsSnapshotMapping.get(inputId); + if ((hashId == 0 && hashIdFromInputSnapshotMapping != 0) || + !Comparing.equal(hashIdFromInputSnapshotMapping, hashId)) { + SharedIndicesData.associateFileData(inputId, myIndexId, hashIdFromInputSnapshotMapping, + EnumeratorIntegerDescriptor.INSTANCE); + if (hashId != 0) { + LOG.error("Unexpected indexing diff with hashid " + + myIndexId + + ", file:" + + IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId) + + + "," + + hashIdFromInputSnapshotMapping + + "," + + hashId); + } + hashId = hashIdFromInputSnapshotMapping; + } + return hashId; + } + return myInputsSnapshotMapping.get(inputId); + } + + private ByteSequence readContents(Integer hashId) throws IOException { + if (SharedIndicesData.ourFileSharedIndicesEnabled) { + if (SharedIndicesData.DO_CHECKS) { + synchronized (myContents) { + ByteSequence contentBytes = SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE); + ByteSequence contentBytesFromContents = myContents.get(hashId); + + if ((contentBytes == null && contentBytesFromContents != null) || + !Comparing.equal(contentBytesFromContents, contentBytes)) { + SharedIndicesData.associateContentData(hashId, myIndexId, contentBytesFromContents, ByteSequenceDataExternalizer.INSTANCE); + if (contentBytes != null) { + LOG.error("Unexpected indexing diff with hashid " + myIndexId + "," + hashId); + } + contentBytes = contentBytesFromContents; + } + return contentBytes; + } + } else { + return SharedIndicesData.recallContentData(hashId, myIndexId, ByteSequenceDataExternalizer.INSTANCE); + } + } + + return myContents.get(hashId); + } + + private Map deserializeSavedPersistentData(ByteSequence bytes) throws IOException { + DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes.getBytes(), bytes.getOffset(), bytes.getLength())); + int pairs = DataInputOutputUtil.readINT(stream); + if (pairs == 0) return Collections.emptyMap(); + Map result = new THashMap<>(pairs); + while (stream.available() > 0) { + Value value = myIndexExtension.getValueExternalizer().read(stream); + Collection keys = mySnapshotIndexExternalizer.read(stream); + for(Key k:keys) result.put(k, value); + } + return result; + } + + private Integer getHashOfContent(FileContent content) throws IOException { + FileType fileType = content.getFileType(); + if (myIsPsiBackedIndex && content instanceof FileContentImpl) { + // psi backed index should use existing psi to build index value (FileContentImpl.getPsiFileForPsiDependentIndex()) + // so we should use different bytes to calculate hash(Id) + Integer previouslyCalculatedUncommittedHashId = content.getUserData(ourSavedUncommittedHashIdKey); + + if (previouslyCalculatedUncommittedHashId == null) { + Document document = FileDocumentManager.getInstance().getCachedDocument(content.getFile()); + + if (document != null) { // if document is not committed + PsiDocumentManager psiDocumentManager = PsiDocumentManager.getInstance(content.getProject()); + + if (psiDocumentManager.isUncommited(document)) { + PsiFile file = psiDocumentManager.getCachedPsiFile(document); + Charset charset = ((FileContentImpl)content).getCharset(); + + if (file != null) { + previouslyCalculatedUncommittedHashId = ContentHashesSupport + .calcContentHashIdWithFileType(file.getText().getBytes(charset), charset, + fileType); + content.putUserData(ourSavedUncommittedHashIdKey, previouslyCalculatedUncommittedHashId); + } + } + } + } + if (previouslyCalculatedUncommittedHashId != null) return previouslyCalculatedUncommittedHashId; + } + + Integer previouslyCalculatedContentHashId = content.getUserData(ourSavedContentHashIdKey); + if (previouslyCalculatedContentHashId == null) { + byte[] hash = content instanceof FileContentImpl ? ((FileContentImpl)content).getHash():null; + if (hash == null) { + if (fileType.isBinary()) { + previouslyCalculatedContentHashId = ContentHashesSupport.calcContentHashId(content.getContent(), fileType); + } else { + Charset charset = content instanceof FileContentImpl ? ((FileContentImpl)content).getCharset() : null; + previouslyCalculatedContentHashId = ContentHashesSupport + .calcContentHashIdWithFileType(content.getContent(), charset, fileType); + } + } else { + previouslyCalculatedContentHashId = ContentHashesSupport.enumerateHash(hash); + } + content.putUserData(ourSavedContentHashIdKey, previouslyCalculatedContentHashId); + } + return previouslyCalculatedContentHashId; + } + private static final com.intellij.openapi.util.Key ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id"); + private static final com.intellij.openapi.util.Key ourSavedUncommittedHashIdKey = com.intellij.openapi.util.Key.create("saved.uncommitted.hash.id"); + + + private StringBuilder buildDiff(Map data, Map contentData) { + StringBuilder moreInfo = new StringBuilder(); + if (contentData.size() != data.size()) { + moreInfo.append("Indexer has different number of elements, previously ").append(data.size()).append(" after ") + .append(contentData.size()).append("\n"); + } else { + moreInfo.append("total ").append(contentData.size()).append(" entries\n"); + } + + for(Map.Entry keyValueEntry:contentData.entrySet()) { + if (!data.containsKey(keyValueEntry.getKey())) { + moreInfo.append("Previous data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n"); + } + else { + Value value = data.get(keyValueEntry.getKey()); + if (!Comparing.equal(keyValueEntry.getValue(), value)) { + moreInfo.append("Previous data has different value for key:").append(keyValueEntry.getKey()).append( ", new value ").append(keyValueEntry.getValue()).append( ", oldValue:").append(value).append("\n"); + } + } + } + + for(Map.Entry keyValueEntry:data.entrySet()) { + if (!contentData.containsKey(keyValueEntry.getKey())) { + moreInfo.append("New data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n"); + } + else { + Value value = contentData.get(keyValueEntry.getKey()); + if (!Comparing.equal(keyValueEntry.getValue(), value)) { + moreInfo.append("New data has different value for key:").append(keyValueEntry.getKey()).append( " new value ").append(value).append( ", oldValue:").append(keyValueEntry.getValue()).append("\n"); + } + } + } + return moreInfo; + } + + private static final ThreadLocalCachedByteArray ourSpareByteArray = new ThreadLocalCachedByteArray(); + private boolean savePersistentData(Map data, int id, boolean delayedReading) { + try { + if (delayedReading && myContents.containsMapping(id)) return false; + BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(ourSpareByteArray.getBuffer(4 * data.size())); + DataOutputStream stream = new DataOutputStream(out); + int size = data.size(); + DataInputOutputUtil.writeINT(stream, size); + + if (size > 0) { + THashMap> values = new THashMap<>(); + List keysForNullValue = null; + for (Map.Entry e : data.entrySet()) { + Value value = e.getValue(); + + List keys = value != null ? values.get(value):keysForNullValue; + if (keys == null) { + if (value != null) values.put(value, keys = new SmartList<>()); + else keys = keysForNullValue = new SmartList<>(); + } + keys.add(e.getKey()); + } + + if (keysForNullValue != null) { + myValueExternalizer.save(stream, null); + mySnapshotIndexExternalizer.save(stream, keysForNullValue); + } + + for(Value value:values.keySet()) { + myValueExternalizer.save(stream, value); + mySnapshotIndexExternalizer.save(stream, values.get(value)); + } + } + + saveContents(id, out); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + return true; + } + + private void saveContents(int id, BufferExposingByteArrayOutputStream out) throws IOException { + ByteSequence byteSequence = new ByteSequence(out.getInternalBuffer(), 0, out.size()); + if (SharedIndicesData.ourFileSharedIndicesEnabled) { + if (SharedIndicesData.DO_CHECKS) { + synchronized (myContents) { + myContents.put(id, byteSequence); + SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE); + } + } else { + SharedIndicesData.associateContentData(id, myIndexId, byteSequence, ByteSequenceDataExternalizer.INSTANCE); + } + } else { + myContents.put(id, byteSequence); + } + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java b/platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java deleted file mode 100644 index 400b9f79f455..000000000000 --- a/platform/lang-impl/src/com/intellij/util/indexing/UpdateData.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2000-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.indexing; - -import java.io.IOException; - -public abstract class UpdateData { - private final ID myIndexId; - - protected UpdateData(ID indexId) { - myIndexId = indexId; - } - - public abstract void iterateRemovedOrUpdatedKeys(int inputId, RemovedOrUpdatedKeyProcessor consumer) - throws StorageException; - - public abstract void iterateAddedKeys(final int inputId, final AddedKeyProcessor consumer) throws StorageException; - - public abstract void save(int inputId) throws IOException; - - public interface AddedKeyProcessor { - void process(Key key, Value value, int inputId) throws StorageException; - } - - public interface RemovedOrUpdatedKeyProcessor { - void process(Key key, int inputId) throws StorageException; - } - - @Override - public String toString() { - return myIndexId + "," + getClass().getName(); - } -} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IntPredicate.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java similarity index 58% rename from platform/lang-impl/src/com/intellij/util/indexing/IntPredicate.java rename to platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java index a0d2e86bcc35..cc8a72eecc5c 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IntPredicate.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareIndexStorage.java @@ -15,9 +15,12 @@ */ package com.intellij.util.indexing; -/** - * Created by Maxim.Mossienko on 11/22/2016. - */ -public interface IntPredicate { - boolean contains(int id); +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.util.Processor; +import com.intellij.util.indexing.impl.IndexStorage; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public interface VfsAwareIndexStorage extends IndexStorage { + boolean processKeys(@NotNull Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException; } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java similarity index 60% rename from platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java rename to platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java index e0fdf9f73472..77aecf010694 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java @@ -25,12 +25,11 @@ import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.search.ProjectAndLibrariesScope; import com.intellij.psi.search.ProjectScopeImpl; import com.intellij.util.Processor; -import com.intellij.util.Processors; import com.intellij.util.SystemProperties; import com.intellij.util.ThrowableRunnable; import com.intellij.util.containers.ConcurrentIntObjectMap; import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.containers.SLRUCache; +import com.intellij.util.indexing.impl.MapIndexStorage; import com.intellij.util.io.*; import com.intellij.util.io.DataOutputStream; import gnu.trove.TIntHashSet; @@ -39,124 +38,49 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; /** * @author Eugene Zhuravlev * Date: Dec 20, 2007 */ -public final class MapIndexStorage implements IndexStorage{ - private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage"); +public final class VfsAwareMapIndexStorage extends MapIndexStorage implements VfsAwareIndexStorage { + private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.MapIndexStorage"); private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true); private final boolean myBuildKeyHashToVirtualFileMapping; - private PersistentMap> myMap; private AppendableStorageBackedByResizableMappedFile myKeyHashToVirtualFileMapping; - private SLRUCache> myCache; private volatile int myLastScannedId; - private final File myBaseStorageFile; - private final KeyDescriptor myKeyDescriptor; - private final int myCacheSize; - private final Lock l = new ReentrantLock(); - private final DataExternalizer myDataExternalizer; - private final boolean myKeyIsUniqueForIndexedFile; private static final ConcurrentIntObjectMap ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap(); - public MapIndexStorage(@NotNull File storageFile, - @NotNull KeyDescriptor keyDescriptor, - @NotNull DataExternalizer valueExternalizer, - final int cacheSize + public VfsAwareMapIndexStorage(@NotNull File storageFile, + @NotNull KeyDescriptor keyDescriptor, + @NotNull DataExternalizer valueExternalizer, + final int cacheSize ) throws IOException { this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false); } - public MapIndexStorage(@NotNull File storageFile, - @NotNull KeyDescriptor keyDescriptor, - @NotNull DataExternalizer valueExternalizer, - final int cacheSize, - boolean keyIsUniqueForIndexedFile, - boolean buildKeyHashToVirtualFileMapping) throws IOException { - myBaseStorageFile = storageFile; - myKeyDescriptor = keyDescriptor; - myCacheSize = cacheSize; - myDataExternalizer = valueExternalizer; - myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile; + public VfsAwareMapIndexStorage(@NotNull File storageFile, + @NotNull KeyDescriptor keyDescriptor, + @NotNull DataExternalizer valueExternalizer, + final int cacheSize, + boolean keyIsUniqueForIndexedFile, + boolean buildKeyHashToVirtualFileMapping) throws IOException { + super(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, false); myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping; initMapAndCache(); } - private static final PersistentHashMapValueStorage.ExceptionalIOCancellationCallback ourProgressManagerCheckCancelledIOCanceller = - new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() { - @Override - public void checkCancellation() { - ProgressManager.checkCanceled(); - } - }; - private void initMapAndCache() throws IOException { - final ValueContainerMap map; - PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(ourProgressManagerCheckCancelledIOCanceller); - PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE); - try { - map = new ValueContainerMap<>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile); - } finally { - PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null); - PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null); - } - myCache = new SLRUCache>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) { - @Override - @NotNull - public ChangeTrackingValueContainer createValue(final Key key) { - return new ChangeTrackingValueContainer<>(new ChangeTrackingValueContainer.Initializer() { - @NotNull - @Override - public Object getLock() { - return map.getDataAccessLock(); - } - - @Nullable - @Override - public ValueContainer compute() { - ValueContainer value; - try { - value = map.get(key); - if (value == null) { - value = new ValueContainerImpl<>(); - } - } - catch (IOException e) { - throw new RuntimeException(e); - } - return value; - } - }); - } - - @Override - protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer valueContainer) { - if (valueContainer.isDirty()) { - try { - map.put(key, valueContainer); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - } - }; - - myMap = map; - + @Override + protected void initMapAndCache() throws IOException { + super.initMapAndCache(); myKeyHashToVirtualFileMapping = myBuildKeyHashToVirtualFileMapping ? new AppendableStorageBackedByResizableMappedFile(getProjectFile(), 4096, null, PagedFileStorage.MB, true) : null; } - @NotNull - private File getStorageFile() { - return new File(myBaseStorageFile.getPath() + ".storage"); + @Override + protected void checkCanceled() { + ProgressManager.checkCanceled(); } @NotNull @@ -172,14 +96,12 @@ public final class MapIndexStorage implements IndexStorage myKeyHashToVirtualFileMapping.force()); } @@ -191,57 +113,34 @@ public final class MapIndexStorage implements IndexStorage myKeyHashToVirtualFileMapping.close()); } - myMap.close(); - } - catch (IOException e) { - throw new StorageException(e); } catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw new StorageException(cause); - } - if (cause instanceof StorageException) { - throw (StorageException)cause; - } - throw e; + unwrapCauseAndRethrow(e); } } @Override public void clear() throws StorageException{ try { - myMap.close(); if (myKeyHashToVirtualFileMapping != null) { withLock(() -> myKeyHashToVirtualFileMapping.close()); } } - catch (IOException|RuntimeException e) { + catch (RuntimeException e) { LOG.error(e); } try { - IOUtil.deleteAllFilesStartingWith(getStorageFile()); if (myKeyHashToVirtualFileMapping != null) IOUtil.deleteAllFilesStartingWith(getProjectFile()); - initMapAndCache(); - } - catch (IOException e) { - throw new StorageException(e); } catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw new StorageException(cause); - } - if (cause instanceof StorageException) { - throw (StorageException)cause; - } - throw e; + unwrapCauseAndRethrow(e); } + super.clear(); } @Override @@ -308,14 +207,7 @@ public final class MapIndexStorage implements IndexStorage implements IndexStorage implements IndexStorage getKeys() throws StorageException { - List keys = new ArrayList<>(); - processKeys(Processors.cancelableCollectProcessor(keys), null, null); - return keys; - } - - @Override - @NotNull - public ChangeTrackingValueContainer read(final Key key) throws StorageException { - l.lock(); - try { - return myCache.get(key); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw new StorageException(cause); - } - if (cause instanceof StorageException) { - throw (StorageException)cause; - } - throw e; - } - finally { - l.unlock(); - } - } - @Override public void addValue(final Key key, final int inputId, final Value value) throws StorageException { try { @@ -447,41 +309,7 @@ public final class MapIndexStorage implements IndexStorage cached; - try { - l.lock(); - cached = myCache.getIfCached(key); - } finally { - l.unlock(); - } - - if (cached != null) { - cached.addValue(inputId, value); - return; - } - // do not pollute the cache with keys unique to indexed file - ChangeTrackingValueContainer valueContainer = new ChangeTrackingValueContainer<>(null); - valueContainer.addValue(inputId, value); - myMap.put(key, valueContainer); - } - catch (IOException e) { - throw new StorageException(e); - } - } - - @Override - public void removeAllValues(@NotNull Key key, int inputId) throws StorageException { - try { - myMap.markDirty(); - // important: assuming the key exists in the index - read(key).removeAssociatedValue(inputId); + super.addValue(key, inputId, value); } catch (IOException e) { throw new StorageException(e); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java new file mode 100644 index 000000000000..ac32f4b11e1f --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java @@ -0,0 +1,260 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intellij.util.indexing; + +import com.intellij.openapi.application.Application; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.application.ModalityState; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.util.UserDataHolder; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.impl.cache.impl.id.IdIndex; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.util.Processor; +import com.intellij.util.indexing.impl.*; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.EnumeratorIntegerDescriptor; +import com.intellij.util.io.PersistentHashMap; +import gnu.trove.TIntObjectHashMap; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; + +/** + * @author Eugene Zhuravlev + * Date: Dec 10, 2007 + */ +public class VfsAwareMapReduceIndex extends MapReduceIndex implements UpdatableIndex{ + private static final Logger LOG = Logger.getInstance(VfsAwareMapReduceIndex.class); + + static { + if (!DebugAssertions.DEBUG) { + final Application app = ApplicationManager.getApplication(); + DebugAssertions.DEBUG = app.isEAP() || app.isInternal(); + } + } + + private final AtomicBoolean myInMemoryMode = new AtomicBoolean(); + private final TIntObjectHashMap> myInMemoryKeys = new TIntObjectHashMap>(); + private final SnapshotInputMappings mySnapshotInputMappings; + + public VfsAwareMapReduceIndex(@NotNull IndexExtension extension, + @NotNull IndexStorage storage) throws IOException { + super(extension, storage, getForwardIndex(extension)); + SharedIndicesData.registerIndex(myIndexId, extension); + mySnapshotInputMappings = myForwardIndex == null ? + new SnapshotInputMappings<>(extension) : + null; + installMemoryModeListener(); + } + + @TestOnly + public VfsAwareMapReduceIndex(@NotNull IndexExtension extension, + @NotNull IndexStorage storage, + @NotNull ForwardIndex forwardIndex) throws IOException { + super(extension, storage, forwardIndex); + SharedIndicesData.registerIndex(myIndexId, extension); + mySnapshotInputMappings = myForwardIndex == null ? + new SnapshotInputMappings<>(extension) : + null; + installMemoryModeListener(); + } + + @NotNull + @Override + protected UpdateData calculateUpdateData(int inputId, @Nullable Input content) { + Map data; + int hashId; + final boolean isContentPhysical = isContentPhysical(content); + if (mySnapshotInputMappings != null && content != null && isContentPhysical) { + final SnapshotInputMappings.Snapshot snapshot = mySnapshotInputMappings.readPersistentDataOrMap(content); + data = snapshot.getData(); + hashId = snapshot.getHashId(); + } else { + data = mapInput(content); + hashId = 0; + } + return createUpdateData(data, () -> { + if (mySnapshotInputMappings != null && isContentPhysical) { + return new MapInputKeyIterator<>(mySnapshotInputMappings.readInputKeys(inputId)); + } + if (myInMemoryMode.get()) { + synchronized (myInMemoryKeys) { + Collection keys = myInMemoryKeys.get(inputId); + if (keys != null) { + return new CollectionInputKeyIterator<>(keys); + } + } + } + if (myForwardIndex != null) { + return readInputKeys(inputId); + } + return EmptyInputKeyIterator.getInstance(); + }, () -> { + if (myInMemoryMode.get()) { + synchronized (myInMemoryKeys) { + myInMemoryKeys.put(inputId, data.keySet()); + } + } else { + if (mySnapshotInputMappings != null ) { + mySnapshotInputMappings.putInputHash(inputId, hashId); + } else { + myForwardIndex.putInputData(inputId, data); + } + } + }); + } + + @Override + public void setIndexedStateForFile(int fileId, @NotNull VirtualFile file) { + IndexingStamp.setFileIndexedStateCurrent(fileId, myIndexId); + } + + @Override + public void resetIndexedStateForFile(int fileId) { + IndexingStamp.setFileIndexedStateOutdated(fileId, myIndexId); + } + + @Override + public boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file) { + return IndexingStamp.isFileIndexedStateCurrent(fileId, myIndexId); + } + + @Override + public boolean processAllKeys(@NotNull Processor processor, @NotNull GlobalSearchScope scope, IdFilter idFilter) throws StorageException { + final Lock lock = getReadLock(); + try { + lock.lock(); + return ((VfsAwareIndexStorage)myStorage).processKeys(processor, scope, idFilter); + } + finally { + lock.unlock(); + } + } + + @Override + public void checkCanceled() { + ProgressManager.checkCanceled(); + } + + @Override + protected void requestRebuild(@NotNull Exception ex) { + Runnable action = () -> FileBasedIndex.getInstance().requestRebuild(myIndexId, ex); + Application application = ApplicationManager.getApplication(); + if (application.isUnitTestMode() || application.isHeadlessEnvironment()) { + // avoid deadlock due to synchronous update in DumbServiceImpl#queueTask + application.invokeLater(action, ModalityState.any()); + } else { + action.run(); + } + } + + @Override + public void clear() throws StorageException { + super.clear(); + if (mySnapshotInputMappings != null) try { + mySnapshotInputMappings.clear(); + } + catch (IOException e) { + LOG.error(e); + } + } + + @Override + public void flush() throws StorageException { + super.flush(); + if (mySnapshotInputMappings != null) mySnapshotInputMappings.flush(); + } + + @Override + public void dispose() { + super.dispose(); + if (mySnapshotInputMappings != null) try { + mySnapshotInputMappings.close(); + } + catch (IOException e) { + LOG.error(e); + } + } + + @Nullable + private static ForwardIndex getForwardIndex(@NotNull IndexExtension indexExtension) + throws IOException { + final boolean hasSnapshotMapping = indexExtension instanceof FileBasedIndexExtension && + ((FileBasedIndexExtension)indexExtension).hasSnapshotMapping() && + IdIndex.ourSnapshotMappingsEnabled; + + return hasSnapshotMapping ? null : new SharedMapBasedForwardIndex<>(new MyForwardIndex<>(indexExtension)); + } + + private static class MyForwardIndex extends MapBasedForwardIndex { + protected MyForwardIndex(IndexExtension indexExtension) throws IOException { + super(indexExtension); + } + + @NotNull + @Override + public PersistentHashMap> createMap() throws IOException { + return createIdToDataKeysIndex(myIndexExtension); + } + + @NotNull + private static PersistentHashMap> createIdToDataKeysIndex(@NotNull IndexExtension extension) throws IOException { + final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(extension.getName()); + return new PersistentHashMap<>(indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, createInputsIndexExternalizer(extension)); + } + } + + private boolean isContentPhysical(Input content) { + return content == null || + (content instanceof UserDataHolder && + FileBasedIndexImpl.ourPhysicalContentKey.get((UserDataHolder)content, Boolean.FALSE)); + } + + private void installMemoryModeListener() { + IndexStorage storage = getStorage(); + if (storage instanceof MemoryIndexStorage) { + ((MemoryIndexStorage)storage).addBufferingStateListener(new MemoryIndexStorage.BufferingStateListener() { + @Override + public void bufferingStateChanged(boolean newState) { + myInMemoryMode.set(newState); + } + + @Override + public void memoryStorageCleared() { + synchronized (myInMemoryKeys) { + myInMemoryKeys.clear(); + } + } + }); + } + } + + protected static DataExternalizer> createInputsIndexExternalizer(IndexExtension extension) { + return extension instanceof CustomInputsIndexFileBasedIndexExtension + ? ((CustomInputsIndexFileBasedIndexExtension)extension).createExternalizer() + : new InputIndexDataExternalizer<>(extension.getKeyDescriptor(), extension.getName()); + } +} diff --git a/platform/util/src/com/intellij/util/indexing/InvertedIndexUtil.java b/platform/util/src/com/intellij/util/indexing/InvertedIndexUtil.java new file mode 100644 index 000000000000..8e3da3a11de8 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/InvertedIndexUtil.java @@ -0,0 +1,83 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.util.Condition; +import com.intellij.util.containers.EmptyIntHashSet; +import gnu.trove.TIntHashSet; +import gnu.trove.TIntProcedure; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Collection; + +@ApiStatus.Experimental +public class InvertedIndexUtil { + @NotNull + public static TIntHashSet collectInputIdsContainingAllKeys(@NotNull InvertedIndex index, + @NotNull Collection dataKeys, + @Nullable Condition keyChecker, + @Nullable Condition valueChecker, + @Nullable ValueContainer.IntPredicate idChecker) + throws StorageException { + TIntHashSet mainIntersection = null; + + for (K dataKey : dataKeys) { + if (keyChecker != null && !keyChecker.value(dataKey)) continue; + + final TIntHashSet copy = new TIntHashSet(); + final ValueContainer container = index.getData(dataKey); + + for (ValueContainer.ValueIterator valueIt = container.getValueIterator(); valueIt.hasNext(); ) { + final V value = valueIt.next(); + if (valueChecker != null && !valueChecker.value(value)) { + continue; + } + + ValueContainer.IntIterator iterator = valueIt.getInputIdsIterator(); + + final ValueContainer.IntPredicate predicate; + if (mainIntersection == null || iterator.size() < mainIntersection.size() || (predicate = valueIt.getValueAssociationPredicate()) == null) { + while (iterator.hasNext()) { + final int id = iterator.next(); + if (mainIntersection == null && (idChecker == null || idChecker.contains(id)) || + mainIntersection != null && mainIntersection.contains(id) + ) { + copy.add(id); + } + } + } + else { + mainIntersection.forEach(new TIntProcedure() { + @Override + public boolean execute(int id) { + if (predicate.contains(id)) copy.add(id); + return true; + } + }); + } + } + + mainIntersection = copy; + if (mainIntersection.isEmpty()) { + return EmptyIntHashSet.INSTANCE; + } + } + + return mainIntersection == null ? EmptyIntHashSet.INSTANCE : mainIntersection; + } +} diff --git a/platform/util/src/com/intellij/util/indexing/ValueContainer.java b/platform/util/src/com/intellij/util/indexing/ValueContainer.java index ad71e8d2c195..64b4bbfeba24 100644 --- a/platform/util/src/com/intellij/util/indexing/ValueContainer.java +++ b/platform/util/src/com/intellij/util/indexing/ValueContainer.java @@ -17,6 +17,7 @@ package com.intellij.util.indexing; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.Iterator; @@ -33,12 +34,19 @@ public abstract class ValueContainer { int size(); } + public interface IntPredicate { + boolean contains(int id); + } + @NotNull public abstract ValueIterator getValueIterator(); public interface ValueIterator extends Iterator { @NotNull IntIterator getInputIdsIterator(); + + @Nullable + IntPredicate getValueAssociationPredicate(); } public abstract int size(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java b/platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java similarity index 96% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java rename to platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java index 8758edbcd0f8..7929d1cdc57b 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java +++ b/platform/util/src/com/intellij/util/indexing/containers/ChangeBufferingList.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,13 @@ */ package com.intellij.util.indexing.containers; -import com.intellij.util.indexing.DebugAssertions; -import com.intellij.util.indexing.IntPredicate; +import com.intellij.util.indexing.impl.DebugAssertions; import com.intellij.util.indexing.ValueContainer; import gnu.trove.TIntProcedure; import java.util.Arrays; -import static com.intellij.util.indexing.DebugAssertions.EXTRA_SANITY_CHECKS; +import static com.intellij.util.indexing.impl.DebugAssertions.EXTRA_SANITY_CHECKS; /** * Class buffers changes in 2 modes: @@ -267,10 +266,10 @@ public class ChangeBufferingList implements Cloneable { return intContainer.size() == 0; } - public IntPredicate intPredicate() { - final IntPredicate predicate = getRandomAccessContainer().intPredicate(); + public ValueContainer.IntPredicate intPredicate() { + final ValueContainer.IntPredicate predicate = getRandomAccessContainer().intPredicate(); if (checkSet != null) { - return new IntPredicate() { + return new ValueContainer.IntPredicate() { @Override public boolean contains(int id) { boolean answer = predicate.contains(id); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java b/platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java similarity index 97% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java rename to platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java index ba685a1ef414..093ea6f255d2 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdBitSet.java +++ b/platform/util/src/com/intellij/util/indexing/containers/IdBitSet.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package com.intellij.util.indexing.containers; import com.intellij.openapi.diagnostic.Logger; -import com.intellij.util.indexing.IntPredicate; import com.intellij.util.indexing.ValueContainer; /** @@ -127,8 +126,8 @@ class IdBitSet implements Cloneable, RandomAccessIntContainer { } @Override - public IntPredicate intPredicate() { - return new IntPredicate() { + public ValueContainer.IntPredicate intPredicate() { + return new ValueContainer.IntPredicate() { @Override public boolean contains(int id) { return IdBitSet.this.contains(id); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java b/platform/util/src/com/intellij/util/indexing/containers/IdSet.java similarity index 88% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java rename to platform/util/src/com/intellij/util/indexing/containers/IdSet.java index 7c326679aec3..2e3beca84205 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/IdSet.java +++ b/platform/util/src/com/intellij/util/indexing/containers/IdSet.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ package com.intellij.util.indexing.containers; -import com.intellij.util.indexing.IntPredicate; +import com.intellij.util.indexing.ValueContainer; import gnu.trove.TIntHashSet; public class IdSet extends TIntHashSet implements RandomAccessIntContainer { @@ -36,8 +36,8 @@ public class IdSet extends TIntHashSet implements RandomAccessIntContainer { } @Override - public IntPredicate intPredicate() { - return new IntPredicate() { + public ValueContainer.IntPredicate intPredicate() { + return new ValueContainer.IntPredicate() { @Override public boolean contains(int id) { return IdSet.this.contains(id); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/IntIdsIterator.java b/platform/util/src/com/intellij/util/indexing/containers/IntIdsIterator.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/IntIdsIterator.java rename to platform/util/src/com/intellij/util/indexing/containers/IntIdsIterator.java diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java b/platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java similarity index 87% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java rename to platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java index 1646fcc45d54..cd8bfc3c0010 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java +++ b/platform/util/src/com/intellij/util/indexing/containers/RandomAccessIntContainer.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ package com.intellij.util.indexing.containers; -import com.intellij.util.indexing.IntPredicate; +import com.intellij.util.indexing.ValueContainer; /** * Created by Maxim.Mossienko on 5/27/2014. @@ -25,7 +25,7 @@ interface RandomAccessIntContainer { boolean add(int value); boolean remove(int value); IntIdsIterator intIterator(); - IntPredicate intPredicate(); + ValueContainer.IntPredicate intPredicate(); void compact(); int size(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java b/platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java similarity index 98% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java rename to platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java index 7a877e063fd5..08d947a03636 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java +++ b/platform/util/src/com/intellij/util/indexing/containers/SortedFileIdSetIterator.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/SortedIdSet.java b/platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java similarity index 96% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/SortedIdSet.java rename to platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java index 889ec57570d0..0637308815cc 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/SortedIdSet.java +++ b/platform/util/src/com/intellij/util/indexing/containers/SortedIdSet.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ package com.intellij.util.indexing.containers; -import com.intellij.util.indexing.IntPredicate; +import com.intellij.util.indexing.ValueContainer; import gnu.trove.TIntProcedure; /** @@ -94,8 +94,8 @@ public class SortedIdSet implements Cloneable, RandomAccessIntContainer { } @Override - public IntPredicate intPredicate() { - return new IntPredicate() { + public ValueContainer.IntPredicate intPredicate() { + return new ValueContainer.IntPredicate() { @Override public boolean contains(int id) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java b/platform/util/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java similarity index 100% rename from platform/lang-impl/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java rename to platform/util/src/com/intellij/util/indexing/containers/TroveSetIntIterator.java diff --git a/platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java b/platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java new file mode 100644 index 000000000000..738a95bf9d75 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java @@ -0,0 +1,42 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.IndexExtension; +import com.intellij.util.io.KeyDescriptor; +import org.jetbrains.annotations.NotNull; + +public abstract class AbstractForwardIndex implements ForwardIndex { + protected final ID myIndexId; + protected final KeyDescriptor myKeyDescriptor; + protected final IndexExtension myIndexExtension; + + protected AbstractForwardIndex(@NotNull IndexExtension extension) { + myIndexId = extension.getName(); + myKeyDescriptor = extension.getKeyDescriptor(); + myIndexExtension = extension; + } + + @NotNull + public IndexExtension getIndexExtension() { + return myIndexExtension; + } + + public boolean hasOnlyKeysData() { + return true; + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java b/platform/util/src/com/intellij/util/indexing/impl/ChangeTrackingValueContainer.java similarity index 92% rename from platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java rename to platform/util/src/com/intellij/util/indexing/impl/ChangeTrackingValueContainer.java index 048a4adde898..c0feffadd901 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java +++ b/platform/util/src/com/intellij/util/indexing/impl/ChangeTrackingValueContainer.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; import com.intellij.openapi.util.Computable; +import com.intellij.util.indexing.ValueContainer; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.DataInputOutputUtil; import gnu.trove.TIntHashSet; @@ -31,7 +32,7 @@ import java.io.IOException; * @author Eugene Zhuravlev * Date: Dec 20, 2007 */ -class ChangeTrackingValueContainer extends UpdatableValueContainer{ +public class ChangeTrackingValueContainer extends UpdatableValueContainer{ // there is no volatile as we modify under write lock and read under read lock private ValueContainerImpl myAdded; private TIntHashSet myInvalidated; @@ -53,7 +54,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer merged.addValue(inputId, value); } - if (myAdded == null) myAdded = new ValueContainerImpl<>(); + if (myAdded == null) myAdded = new ValueContainerImpl(); myAdded.addValue(inputId, value); } @@ -77,7 +78,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer @NotNull @Override - public ValueIterator getValueIterator() { + public ValueContainer.ValueIterator getValueIterator() { return getMergedData().getValueIterator(); } @@ -112,7 +113,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer (newMerged.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD || (myAdded != null && myAdded.size() > ValueContainerImpl.NUMBER_OF_VALUES_THRESHOLD))) { // Calculate file ids that have Value mapped to avoid O(NumberOfValuesInMerged) during removal - fileId2ValueMapping = new FileId2ValueMapping<>(newMerged); + fileId2ValueMapping = new FileId2ValueMapping(newMerged); } final FileId2ValueMapping finalFileId2ValueMapping = fileId2ValueMapping; if (myInvalidated != null) { @@ -132,7 +133,7 @@ class ChangeTrackingValueContainer extends UpdatableValueContainer fileId2ValueMapping.disableOneValuePerFileValidation(); } - myAdded.forEach(new ContainerAction() { + myAdded.forEach(new ValueContainer.ContainerAction() { @Override public boolean perform(final int inputId, final Value value) { // enforcing "one-value-per-file for particular key" invariant diff --git a/platform/util/src/com/intellij/util/indexing/impl/CollectionInputKeyIterator.java b/platform/util/src/com/intellij/util/indexing/impl/CollectionInputKeyIterator.java new file mode 100644 index 000000000000..f789cd4d8850 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/CollectionInputKeyIterator.java @@ -0,0 +1,62 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import org.jetbrains.annotations.Nullable; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; + +public class CollectionInputKeyIterator implements ForwardIndex.InputKeyIterator { + private final Collection mySeq; + private Iterator myIt; + + public CollectionInputKeyIterator(Collection seq) { + mySeq = seq; + } + + @Override + public boolean isAssociatedValueEqual(@Nullable Value value) { + return false; + } + + @Override + public boolean hasNext() { + init(); + return myIt.hasNext(); + } + + @Override + public Key next() { + return myIt.next(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + public Collection getCollection() { + return mySeq == null ? Collections.emptySet() : mySeq; + } + + private void init() { + if (myIt == null) { + myIt = getCollection().iterator(); + } + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java b/platform/util/src/com/intellij/util/indexing/impl/DebugAssertions.java similarity index 74% rename from platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java rename to platform/util/src/com/intellij/util/indexing/impl/DebugAssertions.java index e5b124880032..53531283a001 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/DebugAssertions.java +++ b/platform/util/src/com/intellij/util/indexing/impl/DebugAssertions.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; import com.intellij.util.SystemProperties; import com.intellij.util.containers.hash.LinkedHashMap; @@ -27,9 +26,10 @@ import java.util.Formatter; public class DebugAssertions { private static final Logger LOG = Logger.getInstance(DebugAssertions.class); - public static final boolean DEBUG = SystemProperties.getBooleanProperty( + @SuppressWarnings("StaticNonFinalField") + public static volatile boolean DEBUG = SystemProperties.getBooleanProperty( "intellij.idea.indices.debug", - ApplicationManager.getApplication().isInternal() || ApplicationManager.getApplication().isEAP() + false ); public static final boolean EXTRA_SANITY_CHECKS = SystemProperties.getBooleanProperty( @@ -53,12 +53,12 @@ public class DebugAssertions { LOG.error(new Formatter().format(message, args)); } - static boolean equals(Collection keys, Collection keys2, KeyDescriptor keyDescriptor) { + public static boolean equals(Collection keys, Collection keys2, KeyDescriptor keyDescriptor) { if (keys == null && keys2 == null) return true; if (keys == null || keys2 == null || keys.size() != keys2.size()) return false; - LinkedHashMap map = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor); + LinkedHashMap map = new LinkedHashMap(keys.size(), 0.8f, keyDescriptor); for(Key key:keys) map.put(key, Boolean.TRUE); - LinkedHashMap map2 = new LinkedHashMap<>(keys.size(), 0.8f, keyDescriptor); + LinkedHashMap map2 = new LinkedHashMap(keys.size(), 0.8f, keyDescriptor); for(Key key:keys2) map2.put(key, Boolean.TRUE); return map.equals(map2); } diff --git a/platform/util/src/com/intellij/util/indexing/impl/DiffUpdateData.java b/platform/util/src/com/intellij/util/indexing/impl/DiffUpdateData.java new file mode 100644 index 000000000000..b1895f0614b7 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/DiffUpdateData.java @@ -0,0 +1,109 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.ThrowableComputable; +import com.intellij.util.SystemProperties; +import com.intellij.util.ThrowableRunnable; +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.StorageException; +import gnu.trove.THashSet; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +public class DiffUpdateData extends UpdateData { + public static final boolean ourDiffUpdateEnabled = SystemProperties.getBooleanProperty("idea.disable.diff.index.update", true); + + public DiffUpdateData(@NotNull Map newData, + @NotNull ThrowableComputable, IOException> currentData, + @NotNull ID indexId, ThrowableRunnable forwardIndexUpdate) { + super(newData, currentData, indexId, forwardIndexUpdate); + } + + @Override + public void iterateKeys(int inputId, + KeyValueUpdateProcessor addProcessor, + KeyValueUpdateProcessor updateProcessor, + RemovedKeyProcessor removeProcessor) throws StorageException { + final Set processedKeys = new THashSet(); + int oldSize = 0; //kept for debug reasons + int addedKeys = 0; + int removedKeys = 0; + boolean newDataIsEmpty = myNewData.isEmpty(); + final ForwardIndex.InputKeyIterator currentData; + try { + currentData = myCurrentData.compute(); + } + catch (IOException e) { + throw new StorageException(e); + } + while (currentData.hasNext()) { + oldSize++; + Key key = currentData.next(); + if (!newDataIsEmpty) { + processedKeys.add(key); + } + if (newDataIsEmpty || !myNewData.containsKey(key)) { + removeProcessor.process(key, inputId); + removedKeys++; + } else { + Value newValue = myNewData.get(key); + if (!currentData.isAssociatedValueEqual(newValue)) { + updateProcessor.process(key, newValue, inputId); + removedKeys++; + addedKeys++; + } + } + } + + if (!newDataIsEmpty) { + for (Map.Entry entry : myNewData.entrySet()) { + if (!processedKeys.contains(entry.getKey())) { + addProcessor.process(entry.getKey(), entry.getValue(), inputId); + addedKeys++; + } + } + } + + int totalRequests = requests.incrementAndGet(); + totalRemovals.addAndGet(oldSize); + totalAdditions.addAndGet(myNewData.size()); + incrementalAdditions.addAndGet(removedKeys); + incrementalRemovals.addAndGet(addedKeys); + + if ((totalRequests & 0xFFF) == 0 && DebugAssertions.DEBUG) { + Logger.getInstance(getClass()).info("Incremental index diff update:"+requests + + ", removals:" + totalRemovals + "->" + incrementalRemovals + + ", additions:" +totalAdditions + "->" +incrementalAdditions); + } + } + + private static final AtomicInteger requests = new AtomicInteger(); + private static final AtomicInteger totalRemovals = new AtomicInteger(); + private static final AtomicInteger totalAdditions = new AtomicInteger(); + private static final AtomicInteger incrementalRemovals = new AtomicInteger(); + private static final AtomicInteger incrementalAdditions = new AtomicInteger(); + + @NotNull + protected Map getMap() { + return myNewData; + } +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/EmptyInputKeyIterator.java b/platform/util/src/com/intellij/util/indexing/impl/EmptyInputKeyIterator.java new file mode 100644 index 000000000000..a3bead9a9825 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/EmptyInputKeyIterator.java @@ -0,0 +1,49 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; + +@ApiStatus.Experimental +public class EmptyInputKeyIterator implements ForwardIndex.InputKeyIterator { + public static final EmptyInputKeyIterator EMPTY_INPUT_KEY_ITERATOR = new EmptyInputKeyIterator(); + + public static ForwardIndex.InputKeyIterator getInstance() { + //noinspection unchecked + return EMPTY_INPUT_KEY_ITERATOR; + } + + @Override + public boolean isAssociatedValueEqual(@Nullable Value value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNext() { + return false; + } + + @Override + public Key next() { + throw new UnsupportedOperationException(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java b/platform/util/src/com/intellij/util/indexing/impl/FileId2ValueMapping.java similarity index 92% rename from platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java rename to platform/util/src/com/intellij/util/indexing/impl/FileId2ValueMapping.java index eed732eac356..a13e23c2dd51 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileId2ValueMapping.java +++ b/platform/util/src/com/intellij/util/indexing/impl/FileId2ValueMapping.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; import com.intellij.util.SmartList; +import com.intellij.util.indexing.ValueContainer; import gnu.trove.TIntArrayList; import gnu.trove.TIntObjectHashMap; @@ -30,7 +31,7 @@ class FileId2ValueMapping { private boolean myOnePerFileValidationEnabled = true; FileId2ValueMapping(ValueContainerImpl _valueContainer) { - id2ValueMap = new TIntObjectHashMap<>(); + id2ValueMap = new TIntObjectHashMap(); valueContainer = _valueContainer; TIntArrayList removedFileIdList = null; @@ -45,7 +46,7 @@ class FileId2ValueMapping { if (previousValue != null) { // delay removal of duplicated id -> value mapping since it will affect valueIterator we are using if (removedFileIdList == null) { removedFileIdList = new TIntArrayList(); - removedValueList = new SmartList<>(); + removedValueList = new SmartList(); } removedFileIdList.add(id); removedValueList.add(previousValue); diff --git a/platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java b/platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java new file mode 100644 index 000000000000..819f8f3086a1 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java @@ -0,0 +1,43 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; + +@ApiStatus.Experimental +public interface ForwardIndex { + @NotNull + InputKeyIterator getInputKeys(int inputId) throws IOException; + + void putInputData(int inputId, @NotNull Map data) throws IOException; + + void flush(); + + void clear() throws IOException; + + void close() throws IOException; + + @ApiStatus.Experimental + interface InputKeyIterator extends Iterator { + boolean isAssociatedValueEqual(@Nullable Value value); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java b/platform/util/src/com/intellij/util/indexing/impl/IndexStorage.java similarity index 72% rename from platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java rename to platform/util/src/com/intellij/util/indexing/impl/IndexStorage.java index 1020876ca2e4..f7f290254d91 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java +++ b/platform/util/src/com/intellij/util/indexing/impl/IndexStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,21 +14,21 @@ * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; -import com.intellij.psi.search.GlobalSearchScope; -import com.intellij.util.Processor; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.indexing.ValueContainer; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.Flushable; import java.io.IOException; -import java.util.Collection; /** * @author Eugene Zhuravlev * Date: Dec 10, 2007 */ +@ApiStatus.Experimental public interface IndexStorage extends Flushable { void addValue(Key key, int inputId, Value value) throws StorageException; @@ -40,10 +40,7 @@ public interface IndexStorage extends Flushable { @NotNull ValueContainer read(Key key) throws StorageException; - boolean processKeys(@NotNull Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) throws StorageException; - - @NotNull - Collection getKeys() throws StorageException; + void clearCaches(); void close() throws StorageException; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java b/platform/util/src/com/intellij/util/indexing/impl/InputIndexDataExternalizer.java similarity index 92% rename from platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java rename to platform/util/src/com/intellij/util/indexing/impl/InputIndexDataExternalizer.java index b33074a2433a..749d960072c1 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/InputIndexDataExternalizer.java +++ b/platform/util/src/com/intellij/util/indexing/impl/InputIndexDataExternalizer.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; +import com.intellij.util.indexing.ID; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.DataInputOutputUtil; import com.intellij.util.io.KeyDescriptor; @@ -57,7 +58,7 @@ public class InputIndexDataExternalizer implements DataExternalizer read(@NotNull DataInput in) throws IOException { try { final int size = DataInputOutputUtil.readINT(in); - final List list = new ArrayList<>(size); + final List list = new ArrayList(size); for (int idx = 0; idx < size; idx++) { list.add(myKeyDescriptor.read(in)); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/InvertedIndexValueIterator.java b/platform/util/src/com/intellij/util/indexing/impl/InvertedIndexValueIterator.java similarity index 69% rename from platform/lang-impl/src/com/intellij/util/indexing/InvertedIndexValueIterator.java rename to platform/util/src/com/intellij/util/indexing/impl/InvertedIndexValueIterator.java index c7ed86d94f9b..881abe578061 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/InvertedIndexValueIterator.java +++ b/platform/util/src/com/intellij/util/indexing/impl/InvertedIndexValueIterator.java @@ -13,16 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; +import com.intellij.util.indexing.ValueContainer; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; /** * Created by Maxim.Mossienko on 11/22/2016. */ -interface InvertedIndexValueIterator extends ValueContainer.ValueIterator { +@ApiStatus.Experimental +public interface InvertedIndexValueIterator extends ValueContainer.ValueIterator { + @Override @NotNull - IntPredicate getValueAssociationPredicate(); + ValueContainer.IntPredicate getValueAssociationPredicate(); Object getFileSetObject(); } diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java b/platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java new file mode 100644 index 000000000000..db25c003b972 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java @@ -0,0 +1,90 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.IndexExtension; +import com.intellij.util.io.IOUtil; +import com.intellij.util.io.PersistentHashMap; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +public abstract class MapBasedForwardIndex extends AbstractForwardIndex { + @NotNull + private volatile PersistentHashMap> myInputsIndex; + + protected MapBasedForwardIndex(IndexExtension indexExtension) throws IOException { + super(indexExtension); + myInputsIndex = createMap(); + } + + @NotNull + public abstract PersistentHashMap> createMap() throws IOException; + + @NotNull + @Override + public InputKeyIterator getInputKeys(final int inputId) throws IOException { + return new CollectionInputKeyIterator(myInputsIndex.get(inputId)); + } + + @NotNull + public PersistentHashMap> getInputsIndex() { + return myInputsIndex; + } + + @Override + public void putInputData(int inputId, @NotNull Map data) throws IOException { + putData(inputId, data.keySet()); + } + + public void putData(int inputId, Collection keyCollection) throws IOException { + if (keyCollection.size() > 0) { + myInputsIndex.put(inputId, keyCollection); + } + else { + myInputsIndex.remove(inputId); + } + } + + @Override + public void flush() { + if (myInputsIndex.isDirty()) { + myInputsIndex.force(); + } + } + + @Override + public void close() throws IOException { + myInputsIndex.close(); + } + + @Override + public void clear() throws IOException { + final File baseFile = myInputsIndex.getBaseFile(); + try { + myInputsIndex.close(); + } + catch (Throwable ignored) { + } + if (baseFile != null) { + IOUtil.deleteAllFilesStartingWith(baseFile); + } + myInputsIndex = createMap(); + } +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapIndexStorage.java b/platform/util/src/com/intellij/util/indexing/impl/MapIndexStorage.java new file mode 100644 index 000000000000..79278aa1016a --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/MapIndexStorage.java @@ -0,0 +1,278 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.util.Processor; +import com.intellij.util.containers.SLRUCache; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.indexing.ValueContainer; +import com.intellij.util.io.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public abstract class MapIndexStorage implements IndexStorage { + private static final Logger LOG = Logger.getInstance(MapIndexStorage.class); + protected PersistentMap> myMap; + protected SLRUCache> myCache; + protected final File myBaseStorageFile; + protected final KeyDescriptor myKeyDescriptor; + private final int myCacheSize; + + protected final Lock l = new ReentrantLock(); + private final DataExternalizer myDataExternalizer; + private final boolean myKeyIsUniqueForIndexedFile; + + public MapIndexStorage(@NotNull File storageFile, + @NotNull KeyDescriptor keyDescriptor, + @NotNull DataExternalizer valueExternalizer, + final int cacheSize, + boolean keyIsUniqueForIndexedFile) throws IOException { + this(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, true); + } + + protected MapIndexStorage(@NotNull File storageFile, + @NotNull KeyDescriptor keyDescriptor, + @NotNull DataExternalizer valueExternalizer, + final int cacheSize, + boolean keyIsUniqueForIndexedFile, + boolean initialize) throws IOException { + myBaseStorageFile = storageFile; + myKeyDescriptor = keyDescriptor; + myCacheSize = cacheSize; + myDataExternalizer = valueExternalizer; + myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile; + if (initialize) initMapAndCache(); + } + + protected void initMapAndCache() throws IOException { + final ValueContainerMap map; + PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set( + new PersistentHashMapValueStorage.ExceptionalIOCancellationCallback() { + @Override + public void checkCancellation() { + checkCanceled(); + } + }); + PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE); + try { + map = new ValueContainerMap(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile); + } finally { + PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null); + PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null); + } + myCache = new SLRUCache>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) { + @Override + @NotNull + public ChangeTrackingValueContainer createValue(final Key key) { + return new ChangeTrackingValueContainer(new ChangeTrackingValueContainer.Initializer() { + @NotNull + @Override + public Object getLock() { + return map.getDataAccessLock(); + } + + @Nullable + @Override + public ValueContainer compute() { + ValueContainer value; + try { + value = map.get(key); + if (value == null) { + value = new ValueContainerImpl(); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + return value; + } + }); + } + + @Override + protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer valueContainer) { + if (valueContainer.isDirty()) { + try { + map.put(key, valueContainer); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + } + }; + + myMap = map; + } + + protected abstract void checkCanceled(); + + @NotNull + private File getStorageFile() { + return new File(myBaseStorageFile.getPath() + ".storage"); + } + + @Override + public void flush() { + l.lock(); + try { + if (!myMap.isClosed()) { + myCache.clear(); + if (myMap.isDirty()) myMap.force(); + } + } + finally { + l.unlock(); + } + } + + @Override + public void close() throws StorageException { + try { + flush(); + myMap.close(); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + unwrapCauseAndRethrow(e); + } + } + + @Override + public void clear() throws StorageException{ + try { + myMap.close(); + } + catch (IOException e) { + LOG.error(e); + } + catch (RuntimeException e) { + LOG.error(e); + } + try { + IOUtil.deleteAllFilesStartingWith(getStorageFile()); + initMapAndCache(); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + unwrapCauseAndRethrow(e); + } + } + + @Override + @NotNull + public ChangeTrackingValueContainer read(final Key key) throws StorageException { + l.lock(); + try { + return myCache.get(key); + } + catch (RuntimeException e) { + return unwrapCauseAndRethrow(e); + } + finally { + l.unlock(); + } + } + + @Override + public void addValue(final Key key, final int inputId, final Value value) throws StorageException { + try { + myMap.markDirty(); + if (!myKeyIsUniqueForIndexedFile) { + read(key).addValue(inputId, value); + return; + } + + ChangeTrackingValueContainer cached; + try { + l.lock(); + cached = myCache.getIfCached(key); + } finally { + l.unlock(); + } + + if (cached != null) { + cached.addValue(inputId, value); + return; + } + // do not pollute the cache with keys unique to indexed file + ChangeTrackingValueContainer valueContainer = new ChangeTrackingValueContainer(null); + valueContainer.addValue(inputId, value); + myMap.put(key, valueContainer); + } + catch (IOException e) { + throw new StorageException(e); + } + } + + @Override + public void removeAllValues(@NotNull Key key, int inputId) throws StorageException { + try { + myMap.markDirty(); + // important: assuming the key exists in the index + read(key).removeAssociatedValue(inputId); + } + catch (IOException e) { + throw new StorageException(e); + } + } + + @Override + public void clearCaches() { + + } + + protected static T unwrapCauseAndRethrow(RuntimeException e) throws StorageException { + final Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw new StorageException(cause); + } + if (cause instanceof StorageException) { + throw (StorageException)cause; + } + throw e; + } + + @TestOnly + public boolean processKeys(@NotNull Processor processor) throws StorageException { + l.lock(); + try { + myCache.clear(); // this will ensure that all new keys are made into the map + return myMap.processKeys(processor); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + unwrapCauseAndRethrow(e); + return false; + } + finally { + l.unlock(); + } + } +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapInputKeyIterator.java b/platform/util/src/com/intellij/util/indexing/impl/MapInputKeyIterator.java new file mode 100644 index 000000000000..837b10c220e1 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/MapInputKeyIterator.java @@ -0,0 +1,62 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.util.Comparing; +import org.jetbrains.annotations.Nullable; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; + +public class MapInputKeyIterator implements ForwardIndex.InputKeyIterator { + private final Map myMap; + private Iterator> myIterator; + private Value myCurrentValue; + + public MapInputKeyIterator(Map map) { + myMap = map; + } + + @Override + public boolean isAssociatedValueEqual(@Nullable Value value) { + return Comparing.equal(myCurrentValue, value); + } + + @Override + public boolean hasNext() { + init(); + return myIterator.hasNext(); + } + + @Override + public Key next() { + Map.Entry entry = myIterator.next(); + myCurrentValue = entry.getValue(); + return entry.getKey(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + private void init() { + if (myIterator == null) { + myIterator = (myMap == null ? Collections.emptyMap() : myMap).entrySet().iterator(); + } + } +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java new file mode 100644 index 000000000000..df8395f16d0c --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java @@ -0,0 +1,350 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.progress.ProcessCanceledException; +import com.intellij.openapi.util.Comparing; +import com.intellij.openapi.util.Computable; +import com.intellij.openapi.util.LowMemoryWatcher; +import com.intellij.openapi.util.ThrowableComputable; +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; +import com.intellij.util.ThrowableRunnable; +import com.intellij.util.indexing.*; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.DataOutputStream; +import com.intellij.util.io.UnsyncByteArrayInputStream; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.DataInputStream; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +@ApiStatus.Experimental +public abstract class MapReduceIndex implements InvertedIndex { + private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.MapReduceIndex"); + @NotNull protected final ID myIndexId; + @NotNull protected final IndexStorage myStorage; + + protected final DataExternalizer myValueExternalizer; + protected final IndexExtension myExtension; + private final AtomicLong myModificationStamp = new AtomicLong(); + private final DataIndexer myIndexer; + + protected volatile ForwardIndex myForwardIndex; + private final boolean myUseDiffUpdate; + + private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); + private volatile boolean myDisposed; + + private final LowMemoryWatcher myLowMemoryFlusher = LowMemoryWatcher.register(new Runnable() { + @Override + public void run() { + try { + Lock writeLock = getWriteLock(); + if (writeLock.tryLock()) { + try { + myStorage.clearCaches(); + } finally { + writeLock.unlock(); + } + } + flush(); + } catch (StorageException e) { + LOG.info(e); + requestRebuild(e); + } + } + }); + + protected MapReduceIndex(@NotNull IndexExtension extension, + @NotNull IndexStorage storage, + ForwardIndex forwardIndex) throws IOException { + myIndexId = extension.getName(); + myExtension = extension; + myIndexer = myExtension.getIndexer(); + myStorage = storage; + myValueExternalizer = extension.getValueExternalizer(); + myForwardIndex = forwardIndex; + myUseDiffUpdate = DiffUpdateData.ourDiffUpdateEnabled && (forwardIndex == null || + myForwardIndex instanceof AbstractForwardIndex && + !((AbstractForwardIndex)myForwardIndex).hasOnlyKeysData()); + } + + @NotNull + public IndexStorage getStorage() { + return myStorage; + } + + @Override + public void clear() throws StorageException { + try { + getWriteLock().lock(); + myStorage.clear(); + if (myForwardIndex != null) myForwardIndex.clear(); + } + catch (StorageException e) { + LOG.error(e); + } + catch (IOException e) { + LOG.error(e); + } + finally { + getWriteLock().unlock(); + } + } + + @Override + public void flush() throws StorageException{ + try { + getReadLock().lock(); + if (myForwardIndex != null) myForwardIndex.flush(); + myStorage.flush(); + } + catch (IOException e) { + throw new StorageException(e); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof StorageException || cause instanceof IOException) { + throw new StorageException(cause); + } + else { + throw e; + } + } + finally { + getReadLock().unlock(); + } + } + + @Override + public void dispose() { + myLowMemoryFlusher.stop(); + final Lock lock = getWriteLock(); + try { + lock.lock(); + try { + myStorage.close(); + } + finally { + try { + if (myForwardIndex != null) myForwardIndex.close(); + } + catch (IOException e) { + LOG.error(e); + } + } + } + catch (StorageException e) { + LOG.error(e); + } + finally { + myDisposed = true; + lock.unlock(); + } + } + + @NotNull + public final Lock getReadLock() { + return myLock.readLock(); + } + + @NotNull + public final Lock getWriteLock() { + return myLock.writeLock(); + } + + @Override + @NotNull + public ValueContainer getData(@NotNull final Key key) throws StorageException { + final Lock lock = getReadLock(); + try { + lock.lock(); + if (myDisposed) { + return new ValueContainerImpl(); + } + ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); + return myStorage.read(key); + } + finally { + ValueContainerImpl.ourDebugIndexInfo.set(null); + lock.unlock(); + } + } + + @NotNull + @Override + public final Computable update(final int inputId, @Nullable final Input content) { + final UpdateData updateData = calculateUpdateData(inputId, content); + + return new Computable() { + @Override + public Boolean compute() { + try { + updateWithMap(inputId, updateData); + } + catch (StorageException ex) { + LOG.info("Exception during updateWithMap:" + ex); + requestRebuild(ex); + return Boolean.FALSE; + } + catch (ProcessCanceledException ex) { + LOG.info("Exception during updateWithMap:" + ex); + requestRebuild(ex); + return Boolean.FALSE; + } + + return Boolean.TRUE; + } + }; + } + + @NotNull + protected UpdateData calculateUpdateData(final int inputId, @Nullable Input content) { + final Map data = mapInput(content); + return createUpdateData(data, new ThrowableComputable, IOException>() { + @Override + public ForwardIndex.InputKeyIterator compute() throws IOException { + return readInputKeys(inputId); + } + }, new ThrowableRunnable() { + @Override + public void run() throws IOException { + myForwardIndex.putInputData(inputId, data); + } + }); + } + + @NotNull + protected ForwardIndex.InputKeyIterator readInputKeys(int inputId) throws IOException { + return myForwardIndex.getInputKeys(inputId); + } + + @NotNull + protected UpdateData createUpdateData(Map data, + ThrowableComputable, IOException> keys, + ThrowableRunnable forwardIndexUpdate) { + return myUseDiffUpdate ? new DiffUpdateData(data, keys, myIndexId, forwardIndexUpdate) + : new SimpleUpdateData(data, keys, myIndexId, forwardIndexUpdate); + } + + protected Map mapInput(Input content) { + if (content == null) { + return Collections.emptyMap(); + } + else { + Map data = myIndexer.map(content); + checkValuesHaveProperEqualsAndHashCode(data, myIndexId, myValueExternalizer); + checkCanceled(); + return data; + } + } + + public abstract void checkCanceled(); + + protected abstract void requestRebuild(Exception e); + + public long getModificationStamp() { + return myModificationStamp.get(); + } + + private final UpdateData.RemovedKeyProcessor + myRemovedKeyProcessor = new UpdateData.RemovedKeyProcessor() { + @Override + public void process(Key key, int inputId) throws StorageException { + myModificationStamp.incrementAndGet(); + myStorage.removeAllValues(key, inputId); + } + }; + + private final UpdateData.KeyValueUpdateProcessor myAddedKeyProcessor = new UpdateData.KeyValueUpdateProcessor() { + @Override + public void process(Key key, Value value, int inputId) throws StorageException { + myModificationStamp.incrementAndGet(); + myStorage.addValue(key, inputId, value); + } + }; + + private final UpdateData.KeyValueUpdateProcessor myUpdatedKeyProcessor = new UpdateData.KeyValueUpdateProcessor() { + @Override + public void process(Key key, Value value, int inputId) throws StorageException { + myModificationStamp.incrementAndGet(); + myStorage.removeAllValues(key, inputId); + myStorage.addValue(key, inputId, value); + } + }; + + protected void updateWithMap(final int inputId, + @NotNull UpdateData updateData) throws StorageException { + getWriteLock().lock(); + try { + try { + ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); + updateData.iterateKeys(inputId, myAddedKeyProcessor, myUpdatedKeyProcessor, myRemovedKeyProcessor); + updateData.updateForwardIndex(); + } + catch (ProcessCanceledException e) { + throw e; + } + catch (Throwable e) { // e.g. IOException, AssertionError + throw new StorageException(e); + } + finally { + ValueContainerImpl.ourDebugIndexInfo.set(null); + } + } + finally { + getWriteLock().unlock(); + } + } + + public static void checkValuesHaveProperEqualsAndHashCode(@NotNull Map data, + @NotNull ID indexId, + @NotNull DataExternalizer valueExternalizer) { + if (DebugAssertions.DEBUG) { + for (Map.Entry e : data.entrySet()) { + final Value value = e.getValue(); + if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) { + LOG.error("Index " + indexId.toString() + " violates equals / hashCode contract for Value parameter"); + } + + try { + final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(); + DataOutputStream outputStream = new DataOutputStream(out); + valueExternalizer.save(outputStream, value); + outputStream.close(); + final Value deserializedValue = + valueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size()))); + + if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) { + LOG.error("Index " + indexId.toString() + " deserialization violates equals / hashCode contract for Value parameter"); + } + } + catch (IOException ex) { + LOG.error(ex); + } + } + } + } +} + diff --git a/platform/util/src/com/intellij/util/indexing/impl/SimpleUpdateData.java b/platform/util/src/com/intellij/util/indexing/impl/SimpleUpdateData.java new file mode 100644 index 000000000000..0585e6f63e8c --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/SimpleUpdateData.java @@ -0,0 +1,61 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.util.ThrowableComputable; +import com.intellij.util.ThrowableRunnable; +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.StorageException; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.util.Map; + +public class SimpleUpdateData extends UpdateData { + public SimpleUpdateData(@NotNull Map newData, + @NotNull ThrowableComputable, IOException> currentData, + @NotNull ID indexId, + ThrowableRunnable forwardIndexUpdate) { + super(newData, currentData, indexId, forwardIndexUpdate); + } + + @Override + public void iterateKeys(int inputId, + KeyValueUpdateProcessor addProcessor, + KeyValueUpdateProcessor updateProcessor, + RemovedKeyProcessor removeProcessor) throws StorageException { + final ForwardIndex.InputKeyIterator currentData; + try { + currentData = myCurrentData.compute(); + } + catch (IOException e) { + throw new StorageException(e); + } + iterateKeys(inputId, addProcessor, removeProcessor, currentData); + } + + protected void iterateKeys(int inputId, + KeyValueUpdateProcessor addProcessor, + RemovedKeyProcessor removeProcessor, ForwardIndex.InputKeyIterator currentData) + throws StorageException { + while (currentData.hasNext()) { + removeProcessor.process(currentData.next(), inputId); + } + for (Map.Entry entry : myNewData.entrySet()) { + addProcessor.process(entry.getKey(), entry.getValue(), inputId); + } + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableValueContainer.java b/platform/util/src/com/intellij/util/indexing/impl/UpdatableValueContainer.java similarity index 89% rename from platform/lang-impl/src/com/intellij/util/indexing/UpdatableValueContainer.java rename to platform/util/src/com/intellij/util/indexing/impl/UpdatableValueContainer.java index 2b41f52b93eb..6a7d3a8ee301 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableValueContainer.java +++ b/platform/util/src/com/intellij/util/indexing/impl/UpdatableValueContainer.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2009 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; +import com.intellij.util.indexing.ValueContainer; import com.intellij.util.io.DataExternalizer; import java.io.DataOutput; @@ -25,7 +26,7 @@ import java.io.IOException; * @author Eugene Zhuravlev * Date: Feb 27, 2008 */ -public abstract class UpdatableValueContainer extends ValueContainer{ +public abstract class UpdatableValueContainer extends ValueContainer { public abstract void addValue(int inputId, T value); diff --git a/platform/util/src/com/intellij/util/indexing/impl/UpdateData.java b/platform/util/src/com/intellij/util/indexing/impl/UpdateData.java new file mode 100644 index 000000000000..005da4a2e5d6 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/UpdateData.java @@ -0,0 +1,77 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.util.ThrowableComputable; +import com.intellij.util.ThrowableRunnable; +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.StorageException; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.Map; + +@ApiStatus.Experimental +public abstract class UpdateData { + protected final Map myNewData; + protected final ThrowableComputable, IOException> myCurrentData; + private final ID myIndexId; + private final ThrowableRunnable myForwardIndexUpdate; + + protected UpdateData(@NotNull Map newData, + @NotNull ThrowableComputable, IOException> currentData, + @NotNull ID indexId, + @Nullable ThrowableRunnable forwardIndexUpdate) { + myNewData = newData; + myCurrentData = currentData; + myIndexId = indexId; + myForwardIndexUpdate = forwardIndexUpdate; + } + + public abstract void iterateKeys(final int inputId, + final KeyValueUpdateProcessor addProcessor, + final KeyValueUpdateProcessor updateProcessor, + final RemovedKeyProcessor removeProcessor) throws StorageException; + + public Map getNewData() { + return myNewData; + } + + public interface KeyValueUpdateProcessor { + void process(Key key, Value value, int inputId) throws StorageException; + } + + public interface RemovedKeyProcessor { + void process(Key key, int inputId) throws StorageException; + } + + public ID getIndexId() { + return myIndexId; + } + + public void updateForwardIndex() throws IOException { + if (myForwardIndexUpdate != null) { + myForwardIndexUpdate.run(); + } + } + + @Override + public String toString() { + return myIndexId + "," + getClass().getName(); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java b/platform/util/src/com/intellij/util/indexing/impl/ValueContainerImpl.java similarity index 95% rename from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java rename to platform/util/src/com/intellij/util/indexing/impl/ValueContainerImpl.java index d1e5b76bbe72..64d666c1eadc 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java +++ b/platform/util/src/com/intellij/util/indexing/impl/ValueContainerImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,13 @@ * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; import com.intellij.openapi.diagnostic.Logger; import com.intellij.util.SmartList; import com.intellij.util.containers.EmptyIterator; +import com.intellij.util.indexing.ID; +import com.intellij.util.indexing.ValueContainer; import com.intellij.util.indexing.containers.ChangeBufferingList; import com.intellij.util.indexing.containers.IdSet; import com.intellij.util.indexing.containers.IntIdsIterator; @@ -40,7 +42,7 @@ import java.util.List; * Date: Dec 20, 2007 */ class ValueContainerImpl extends UpdatableValueContainer implements Cloneable{ - private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.ValueContainerImpl"); + private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.impl.ValueContainerImpl"); private final static Object myNullValue = new Object(); // there is no volatile as we modify under write lock and read under read lock @@ -78,7 +80,7 @@ class ValueContainerImpl extends UpdatableValueContainer implement return myInputIdMapping != null ? myInputIdMapping instanceof THashMap ? ((THashMap)myInputIdMapping).size(): 1 : 0; } - static final ThreadLocal ourDebugIndexInfo = new ThreadLocal<>(); + static final ThreadLocal ourDebugIndexInfo = new ThreadLocal(); @Override public void removeAssociatedValue(int inputId) { @@ -90,8 +92,8 @@ class ValueContainerImpl extends UpdatableValueContainer implement if (valueIterator.getValueAssociationPredicate().contains(inputId)) { if (fileSetObjects == null) { - fileSetObjects = new SmartList<>(); - valueObjects = new SmartList<>(); + fileSetObjects = new SmartList(); + valueObjects = new SmartList(); } else if (DebugAssertions.DEBUG) { LOG.error("Expected only one value per-inputId for " + ourDebugIndexInfo.get(), String.valueOf(fileSetObjects.get(0)), String.valueOf(value)); @@ -151,7 +153,7 @@ class ValueContainerImpl extends UpdatableValueContainer implement @NotNull @Override - public IntIterator getInputIdsIterator() { + public ValueContainer.IntIterator getInputIdsIterator() { return getIntIteratorOutOfFileSetObject(getFileSetObject()); } @@ -212,7 +214,7 @@ class ValueContainerImpl extends UpdatableValueContainer implement @NotNull @Override - public IntIterator getInputIdsIterator() { + public ValueContainer.IntIterator getInputIdsIterator() { return getIntIteratorOutOfFileSetObject(getFileSetObject()); } @@ -238,7 +240,7 @@ class ValueContainerImpl extends UpdatableValueContainer implement @NotNull @Override - public IntIterator getInputIdsIterator() { + public ValueContainer.IntIterator getInputIdsIterator() { throw new IllegalStateException(); } @@ -272,7 +274,8 @@ class ValueContainerImpl extends UpdatableValueContainer implement return ((ChangeBufferingList)input).intPredicate(); } - private static @NotNull IntIterator getIntIteratorOutOfFileSetObject(@Nullable Object input) { + private static @NotNull + ValueContainer.IntIterator getIntIteratorOutOfFileSetObject(@Nullable Object input) { if (input == null) return EMPTY_ITERATOR; if (input instanceof Integer){ return new SingleValueIterator(((Integer)input).intValue()); @@ -312,7 +315,7 @@ class ValueContainerImpl extends UpdatableValueContainer implement } } - private static final IntIterator EMPTY_ITERATOR = new IntIdsIterator() { + private static final ValueContainer.IntIterator EMPTY_ITERATOR = new IntIdsIterator() { @Override public boolean hasNext() { return false; @@ -341,11 +344,11 @@ class ValueContainerImpl extends UpdatableValueContainer implement @NotNull public ValueContainerImpl copy() { - ValueContainerImpl container = new ValueContainerImpl<>(); + ValueContainerImpl container = new ValueContainerImpl(); if (myInputIdMapping instanceof THashMap) { final THashMap mapping = (THashMap)myInputIdMapping; - final THashMap newMapping = new THashMap<>(mapping.size()); + final THashMap newMapping = new THashMap(mapping.size()); container.myInputIdMapping = newMapping; mapping.forEachEntry(new TObjectObjectProcedure() { @@ -460,7 +463,7 @@ class ValueContainerImpl extends UpdatableValueContainer implement final int inputId = -valueCount; if (mapping == null && size() > NUMBER_OF_VALUES_THRESHOLD) { // avoid O(NumberOfValues) - mapping = new FileId2ValueMapping<>(this); + mapping = new FileId2ValueMapping(this); } boolean doCompact; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java b/platform/util/src/com/intellij/util/indexing/impl/ValueContainerMap.java similarity index 93% rename from platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java rename to platform/util/src/com/intellij/util/indexing/impl/ValueContainerMap.java index 6ff8441d813c..1f3ac9dab9ee 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java +++ b/platform/util/src/com/intellij/util/indexing/impl/ValueContainerMap.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2016 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.intellij.util.indexing; +package com.intellij.util.indexing.impl; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.KeyDescriptor; @@ -35,7 +35,7 @@ class ValueContainerMap extends PersistentHashMap valueExternalizer, boolean keyIsUniqueForIndexedFile ) throws IOException { - super(file, keyKeyDescriptor, new ValueContainerExternalizer<>(valueExternalizer)); + super(file, keyKeyDescriptor, new ValueContainerExternalizer(valueExternalizer)); myValueExternalizer = valueExternalizer; myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile; } @@ -48,7 +48,7 @@ class ValueContainerMap extends PersistentHashMap container) throws IOException { synchronized (myEnumerator) { - ChangeTrackingValueContainer valueContainer = (ChangeTrackingValueContainer)container; + final ChangeTrackingValueContainer valueContainer = (ChangeTrackingValueContainer)container; // try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files // note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular @@ -83,7 +83,7 @@ class ValueContainerMap extends PersistentHashMap read(@NotNull final DataInput in) throws IOException { - final ValueContainerImpl valueContainer = new ValueContainerImpl<>(); + final ValueContainerImpl valueContainer = new ValueContainerImpl(); valueContainer.readFrom((DataInputStream)in, myValueExternalizer); return valueContainer; diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java index 389b2a068330..7f8fbef4be25 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java @@ -16,23 +16,27 @@ package com.intellij.vcs.log.data.index; import com.intellij.openapi.Disposable; +import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.util.Disposer; import com.intellij.util.Consumer; import com.intellij.util.indexing.*; +import com.intellij.util.indexing.impl.EmptyInputKeyIterator; +import com.intellij.util.indexing.impl.ForwardIndex; +import com.intellij.util.indexing.impl.MapIndexStorage; +import com.intellij.util.indexing.impl.MapReduceIndex; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.EnumeratorIntegerDescriptor; import com.intellij.util.io.KeyDescriptor; -import com.intellij.util.io.PersistentHashMap; import com.intellij.vcs.log.VcsFullCommitDetails; import com.intellij.vcs.log.impl.FatalErrorHandler; import com.intellij.vcs.log.util.PersistentUtil; import gnu.trove.TIntHashSet; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; import java.util.Collection; +import java.util.Map; import java.util.Set; import java.util.function.ObjIntConsumer; @@ -78,35 +82,29 @@ public class VcsLogFullDetailsIndex implements Disposable { } @NotNull - public ValueContainer.IntIterator getCommitsWithAllKeys(@NotNull Collection keys) throws StorageException { - return FileBasedIndexImpl.collectInputIdsContainingAllKeys(myMapReduceIndex, keys); + public TIntHashSet getCommitsWithAllKeys(@NotNull Collection keys) throws StorageException { + return InvertedIndexUtil.collectInputIdsContainingAllKeys(myMapReduceIndex, keys, (k) -> { + ProgressManager.checkCanceled(); + return true; + }, null, null); } private void iterateCommitIds(int key, @NotNull Consumer consumer) throws StorageException { ValueContainer data = myMapReduceIndex.getData(key); - - ValueContainer.ValueIterator valueIt = data.getValueIterator(); - while (valueIt.hasNext()) { - valueIt.next(); - ValueContainer.IntIterator inputIt = valueIt.getInputIdsIterator(); - while (inputIt.hasNext()) { - consumer.consume(inputIt.next()); + data.forEach(new ValueContainer.ContainerAction() { + @Override + public boolean perform(int id, T value) { + consumer.consume(id); + return true; } - } + }); } protected void iterateCommitIdsAndValues(int key, @NotNull ObjIntConsumer consumer) throws StorageException { - ValueContainer data = myMapReduceIndex.getData(key); - - ValueContainer.ValueIterator valueIt = data.getValueIterator(); - while (valueIt.hasNext()) { - T nextValue = valueIt.next(); - ValueContainer.IntIterator inputIt = valueIt.getInputIdsIterator(); - while (inputIt.hasNext()) { - int next = inputIt.next(); - consumer.accept(nextValue, next); - } - } + myMapReduceIndex.getData(key).forEach((id, value) -> { + consumer.accept(value, id); + return true; + }); } public void update(int commitId, @NotNull VcsFullCommitDetails details) throws IOException { @@ -128,24 +126,55 @@ public class VcsLogFullDetailsIndex implements Disposable { } private class MyMapReduceIndex extends MapReduceIndex { - public MyMapReduceIndex(@NotNull DataIndexer indexer, @NotNull DataExternalizer externalizer, int version) throws IOException { super(new MyIndexExtension(indexer, externalizer, version), - new MapIndexStorage<>(getStorageFile(myName, myLogId), - EnumeratorIntegerDescriptor.INSTANCE, - externalizer, 5000)); + new MapIndexStorage(getStorageFile(myName, myLogId), + EnumeratorIntegerDescriptor.INSTANCE, + externalizer, 5000, false) { + @Override + protected void checkCanceled() { + ProgressManager.checkCanceled(); + } + }, + new ForwardIndex() { + @NotNull + @Override + public InputKeyIterator getInputKeys(int inputId) { + return EmptyInputKeyIterator.getInstance(); + } + + @Override + public void putInputData(int inputId, @NotNull Map data) throws IOException { + + } + + @Override + public void flush() { + + } + + @Override + public void clear() throws IOException { + + } + + @Override + public void close() throws IOException { + + } + }); } @Override - protected PersistentHashMap> createInputsIndex() throws IOException { - return null; + public void checkCanceled() { + ProgressManager.checkCanceled(); } @Override - protected void requestRebuild(@Nullable Exception ex) { - myFatalErrorHandler.consume(this, ex != null ? ex : new Exception("Index rebuild requested")); + public void requestRebuild(@NotNull Exception ex) { + myFatalErrorHandler.consume(this, ex); } } diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java index f90961cf903b..d4997e0ede9a 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogMessagesTrigramIndex.java @@ -19,11 +19,11 @@ import com.intellij.openapi.Disposable; import com.intellij.openapi.util.text.TrigramBuilder; import com.intellij.util.indexing.DataIndexer; import com.intellij.util.indexing.StorageException; -import com.intellij.util.indexing.ValueContainer; import com.intellij.util.io.VoidDataExternalizer; import com.intellij.vcs.log.VcsFullCommitDetails; import com.intellij.vcs.log.impl.FatalErrorHandler; import gnu.trove.THashMap; +import gnu.trove.TIntHashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -43,7 +43,7 @@ public class VcsLogMessagesTrigramIndex extends VcsLogFullDetailsIndex { } @Nullable - public ValueContainer.IntIterator getCommitsForSubstring(@NotNull String string) throws StorageException { + public TIntHashSet getCommitsForSubstring(@NotNull String string) throws StorageException { MyTrigramProcessor trigramProcessor = new MyTrigramProcessor(); TrigramBuilder.processTrigrams(string, trigramProcessor); diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java index 4ed9ce77855d..6cb776fd59fa 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogPersistentIndex.java @@ -33,7 +33,6 @@ import com.intellij.util.ThrowableRunnable; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.containers.EmptyIntHashSet; import com.intellij.util.indexing.StorageException; -import com.intellij.util.indexing.ValueContainer; import com.intellij.util.io.*; import com.intellij.vcs.log.*; import com.intellij.vcs.log.data.*; @@ -270,11 +269,10 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { return filter(myIndexStorage.messages, message -> pattern.matcher(message).find()); } else { - ValueContainer.IntIterator commitsForSearch = myIndexStorage.trigrams.getCommitsForSubstring(text); + TIntHashSet commitsForSearch = myIndexStorage.trigrams.getCommitsForSubstring(text); if (commitsForSearch != null) { TIntHashSet result = new TIntHashSet(); - while (commitsForSearch.hasNext()) { - int commit = commitsForSearch.next(); + commitsForSearch.forEach(commit -> { try { String value = myIndexStorage.messages.get(commit); if (value != null) { @@ -285,9 +283,10 @@ public class VcsLogPersistentIndex implements VcsLogIndex, Disposable { } catch (IOException e) { myFatalErrorsConsumer.consume(this, e); - break; + return false; } - } + return true; + }); return result; } }