diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index db0ffc31143b..7fe4fa4955b4 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -573,12 +573,9 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponentAdap @NotNull final Map newValues) { try { final MyIndex index = (MyIndex)getAsyncState().myIndices.get(key); - final ThrowableComputable, IOException> - oldMapGetter = () -> new MapInputKeyIterator<>(oldValues); - index.updateWithMap(fileId, - DiffUpdateData.ourDiffUpdateEnabled - ? new DiffUpdateData<>(newValues, oldMapGetter, key, null) - : new SimpleUpdateData<>(newValues, oldMapGetter, key, null)); + final ThrowableComputable, IOException> + oldMapGetter = () -> new MapInputDataDiffBuilder<>(fileId, oldValues); + index.updateWithMap(fileId, new UpdateData<>(newValues, oldMapGetter, key, null)); } catch (StorageException e) { LOG.info(e); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 63dcb2555ffc..7064e3de5ad1 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -39,6 +39,7 @@ import com.intellij.util.indexing.impl.*; import com.intellij.util.io.*; import gnu.trove.THashMap; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.*; import java.io.DataOutputStream; @@ -406,29 +407,29 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi @NotNull @Override protected UpdateData createUpdateData(Map data, - ThrowableComputable, IOException> oldKeys, + ThrowableComputable, IOException> oldKeys, ThrowableRunnable forwardIndexUpdate) { return new StubUpdatingData(data, oldKeys, forwardIndexUpdate); } - static class StubUpdatingData extends SimpleUpdateData { + static class StubUpdatingData extends UpdateData { private Collection oldStubIndexKeys; public StubUpdatingData(@NotNull Map newData, - @NotNull ThrowableComputable, IOException> iterator, - ThrowableRunnable forwardIndexUpdate) { + @NotNull ThrowableComputable, IOException> iterator, + @Nullable ThrowableRunnable forwardIndexUpdate) { super(newData, iterator, INDEX_ID, forwardIndexUpdate); } @Override - protected void iterateKeys(int inputId, - KeyValueUpdateProcessor addProcessor, - RemovedKeyProcessor removeProcessor, - ForwardIndex.InputKeyIterator currentData) throws StorageException { - if (currentData instanceof CollectionInputKeyIterator) { - oldStubIndexKeys = ((CollectionInputKeyIterator)currentData).getCollection(); - } - super.iterateKeys(inputId, addProcessor, removeProcessor, currentData); + protected ThrowableComputable, IOException> getCurrentDataEvaluator() { + return () -> { + final InputDataDiffBuilder diffBuilder = super.getCurrentDataEvaluator().compute(); + if (diffBuilder instanceof CollectionInputDataDiffBuilder) { + oldStubIndexKeys = ((CollectionInputDataDiffBuilder) diffBuilder).getSeq(); + } + return diffBuilder; + }; } public Map> getOldStubIndicesValueMap() { @@ -458,7 +459,6 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } } - @Override protected void updateWithMap(int inputId, @NotNull UpdateData updateData) throws StorageException { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java index 4207206e8b19..def73a8fa4f8 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/SharedMapBasedForwardIndex.java @@ -16,10 +16,7 @@ package com.intellij.util.indexing; import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; -import com.intellij.util.indexing.impl.AbstractForwardIndex; -import com.intellij.util.indexing.impl.CollectionInputKeyIterator; -import com.intellij.util.indexing.impl.DebugAssertions; -import com.intellij.util.indexing.impl.MapBasedForwardIndex; +import com.intellij.util.indexing.impl.*; import com.intellij.util.io.DataExternalizer; import org.jetbrains.annotations.NotNull; @@ -39,7 +36,7 @@ class SharedMapBasedForwardIndex extends AbstractForwardIndex getInputKeys(int inputId) throws IOException { + public InputDataDiffBuilder getDiffBuilder(int inputId) throws IOException { Collection keys; if (SharedIndicesData.ourFileSharedIndicesEnabled) { keys = SharedIndicesData.recallFileData(inputId, myIndexId, mySnapshotIndexExternalizer); @@ -56,9 +53,9 @@ class SharedMapBasedForwardIndex extends AbstractForwardIndex(keys); + return new CollectionInputDataDiffBuilder<>(inputId, keys); } - return new CollectionInputKeyIterator<>(myUnderlying.getInputsIndex().get(inputId)); + return new CollectionInputDataDiffBuilder<>(inputId, myUnderlying.getInputsIndex().get(inputId)); } @Override diff --git a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java index ac32f4b11e1f..b62e0b5db016 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java @@ -98,20 +98,20 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex { if (mySnapshotInputMappings != null && isContentPhysical) { - return new MapInputKeyIterator<>(mySnapshotInputMappings.readInputKeys(inputId)); + return new MapInputDataDiffBuilder<>(inputId, mySnapshotInputMappings.readInputKeys(inputId)); } if (myInMemoryMode.get()) { synchronized (myInMemoryKeys) { Collection keys = myInMemoryKeys.get(inputId); if (keys != null) { - return new CollectionInputKeyIterator<>(keys); + return new CollectionInputDataDiffBuilder<>(inputId, keys); } } } if (myForwardIndex != null) { - return readInputKeys(inputId); + return getKeysDiffBuilder(inputId); } - return EmptyInputKeyIterator.getInstance(); + return new EmptyInputDataDiffBuilder(inputId); }, () -> { if (myInMemoryMode.get()) { synchronized (myInMemoryKeys) { diff --git a/platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java b/platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java index 738a95bf9d75..58b969b020d0 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java +++ b/platform/util/src/com/intellij/util/indexing/impl/AbstractForwardIndex.java @@ -35,8 +35,4 @@ public abstract class AbstractForwardIndex implements ForwardIndex getIndexExtension() { return myIndexExtension; } - - public boolean hasOnlyKeysData() { - return true; - } } diff --git a/platform/util/src/com/intellij/util/indexing/impl/CollectionInputDataDiffBuilder.java b/platform/util/src/com/intellij/util/indexing/impl/CollectionInputDataDiffBuilder.java new file mode 100644 index 000000000000..e635c66cb1ca --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/CollectionInputDataDiffBuilder.java @@ -0,0 +1,58 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.StorageException; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +@ApiStatus.Experimental +public class CollectionInputDataDiffBuilder extends InputDataDiffBuilder { + private final Collection mySeq; + + public CollectionInputDataDiffBuilder(int inputId, @Nullable Collection seq) { + super(inputId); + mySeq = seq == null ? Collections.emptySet() : seq; + } + + @Override + public void differentiate(@NotNull Map newData, + @NotNull KeyValueUpdateProcessor addProcessor, + @NotNull KeyValueUpdateProcessor updateProcessor, + @NotNull RemovedKeyProcessor removeProcessor) throws StorageException { + differentiateWithKeySeq(mySeq, newData, myInputId, addProcessor, removeProcessor); + } + + public Collection getSeq() { + return mySeq; + } + + static void differentiateWithKeySeq(@NotNull Collection currentData, + @NotNull Map newData, + int inputId, + @NotNull KeyValueUpdateProcessor addProcessor, + @NotNull RemovedKeyProcessor removeProcessor) throws StorageException { + for (Key key : currentData) { + removeProcessor.process(key, inputId); + } + EmptyInputDataDiffBuilder.processKeys(newData, addProcessor, inputId); + } +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/CollectionInputKeyIterator.java b/platform/util/src/com/intellij/util/indexing/impl/CollectionInputKeyIterator.java deleted file mode 100644 index f789cd4d8850..000000000000 --- a/platform/util/src/com/intellij/util/indexing/impl/CollectionInputKeyIterator.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.indexing.impl; - -import org.jetbrains.annotations.Nullable; - -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; - -public class CollectionInputKeyIterator implements ForwardIndex.InputKeyIterator { - private final Collection mySeq; - private Iterator myIt; - - public CollectionInputKeyIterator(Collection seq) { - mySeq = seq; - } - - @Override - public boolean isAssociatedValueEqual(@Nullable Value value) { - return false; - } - - @Override - public boolean hasNext() { - init(); - return myIt.hasNext(); - } - - @Override - public Key next() { - return myIt.next(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - public Collection getCollection() { - return mySeq == null ? Collections.emptySet() : mySeq; - } - - private void init() { - if (myIt == null) { - myIt = getCollection().iterator(); - } - } -} diff --git a/platform/util/src/com/intellij/util/indexing/impl/DiffUpdateData.java b/platform/util/src/com/intellij/util/indexing/impl/DiffUpdateData.java deleted file mode 100644 index b1895f0614b7..000000000000 --- a/platform/util/src/com/intellij/util/indexing/impl/DiffUpdateData.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.indexing.impl; - -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.ThrowableComputable; -import com.intellij.util.SystemProperties; -import com.intellij.util.ThrowableRunnable; -import com.intellij.util.indexing.ID; -import com.intellij.util.indexing.StorageException; -import gnu.trove.THashSet; -import org.jetbrains.annotations.NotNull; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -public class DiffUpdateData extends UpdateData { - public static final boolean ourDiffUpdateEnabled = SystemProperties.getBooleanProperty("idea.disable.diff.index.update", true); - - public DiffUpdateData(@NotNull Map newData, - @NotNull ThrowableComputable, IOException> currentData, - @NotNull ID indexId, ThrowableRunnable forwardIndexUpdate) { - super(newData, currentData, indexId, forwardIndexUpdate); - } - - @Override - public void iterateKeys(int inputId, - KeyValueUpdateProcessor addProcessor, - KeyValueUpdateProcessor updateProcessor, - RemovedKeyProcessor removeProcessor) throws StorageException { - final Set processedKeys = new THashSet(); - int oldSize = 0; //kept for debug reasons - int addedKeys = 0; - int removedKeys = 0; - boolean newDataIsEmpty = myNewData.isEmpty(); - final ForwardIndex.InputKeyIterator currentData; - try { - currentData = myCurrentData.compute(); - } - catch (IOException e) { - throw new StorageException(e); - } - while (currentData.hasNext()) { - oldSize++; - Key key = currentData.next(); - if (!newDataIsEmpty) { - processedKeys.add(key); - } - if (newDataIsEmpty || !myNewData.containsKey(key)) { - removeProcessor.process(key, inputId); - removedKeys++; - } else { - Value newValue = myNewData.get(key); - if (!currentData.isAssociatedValueEqual(newValue)) { - updateProcessor.process(key, newValue, inputId); - removedKeys++; - addedKeys++; - } - } - } - - if (!newDataIsEmpty) { - for (Map.Entry entry : myNewData.entrySet()) { - if (!processedKeys.contains(entry.getKey())) { - addProcessor.process(entry.getKey(), entry.getValue(), inputId); - addedKeys++; - } - } - } - - int totalRequests = requests.incrementAndGet(); - totalRemovals.addAndGet(oldSize); - totalAdditions.addAndGet(myNewData.size()); - incrementalAdditions.addAndGet(removedKeys); - incrementalRemovals.addAndGet(addedKeys); - - if ((totalRequests & 0xFFF) == 0 && DebugAssertions.DEBUG) { - Logger.getInstance(getClass()).info("Incremental index diff update:"+requests + - ", removals:" + totalRemovals + "->" + incrementalRemovals + - ", additions:" +totalAdditions + "->" +incrementalAdditions); - } - } - - private static final AtomicInteger requests = new AtomicInteger(); - private static final AtomicInteger totalRemovals = new AtomicInteger(); - private static final AtomicInteger totalAdditions = new AtomicInteger(); - private static final AtomicInteger incrementalRemovals = new AtomicInteger(); - private static final AtomicInteger incrementalAdditions = new AtomicInteger(); - - @NotNull - protected Map getMap() { - return myNewData; - } -} diff --git a/platform/util/src/com/intellij/util/indexing/impl/EmptyInputDataDiffBuilder.java b/platform/util/src/com/intellij/util/indexing/impl/EmptyInputDataDiffBuilder.java new file mode 100644 index 000000000000..9ca79f937891 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/EmptyInputDataDiffBuilder.java @@ -0,0 +1,69 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.StorageException; +import gnu.trove.THashMap; +import gnu.trove.TObjectObjectProcedure; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +import java.util.Map; + +@ApiStatus.Experimental +public class EmptyInputDataDiffBuilder extends InputDataDiffBuilder { + public EmptyInputDataDiffBuilder(int inputId) { + super(inputId); + } + + @Override + public void differentiate(@NotNull Map newData, + @NotNull final KeyValueUpdateProcessor addProcessor, + @NotNull KeyValueUpdateProcessor updateProcessor, + @NotNull RemovedKeyProcessor removeProcessor) throws StorageException { + processKeys(newData, addProcessor, myInputId); + } + + static void processKeys(@NotNull Map currentData, + @NotNull final KeyValueUpdateProcessor processor, + final int inputId) + throws StorageException { + if (currentData instanceof THashMap) { + final StorageException[] exception = new StorageException[]{null}; + ((THashMap)currentData).forEachEntry(new TObjectObjectProcedure() { + @Override + public boolean execute(Key k, Value v) { + try { + processor.process(k, v, inputId); + } + catch (StorageException e) { + exception[0] = e; + return false; + } + return true; + } + }); + if (exception[0] != null) { + throw exception[0]; + } + } + else { + for (Map.Entry entry : currentData.entrySet()) { + processor.process(entry.getKey(), entry.getValue(), inputId); + } + } + } +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/EmptyInputKeyIterator.java b/platform/util/src/com/intellij/util/indexing/impl/EmptyInputKeyIterator.java deleted file mode 100644 index a3bead9a9825..000000000000 --- a/platform/util/src/com/intellij/util/indexing/impl/EmptyInputKeyIterator.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.indexing.impl; - -import org.jetbrains.annotations.ApiStatus; -import org.jetbrains.annotations.Nullable; - -@ApiStatus.Experimental -public class EmptyInputKeyIterator implements ForwardIndex.InputKeyIterator { - public static final EmptyInputKeyIterator EMPTY_INPUT_KEY_ITERATOR = new EmptyInputKeyIterator(); - - public static ForwardIndex.InputKeyIterator getInstance() { - //noinspection unchecked - return EMPTY_INPUT_KEY_ITERATOR; - } - - @Override - public boolean isAssociatedValueEqual(@Nullable Value value) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasNext() { - return false; - } - - @Override - public Key next() { - throw new UnsupportedOperationException(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } -} diff --git a/platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java b/platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java index 819f8f3086a1..8cd591977b17 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java +++ b/platform/util/src/com/intellij/util/indexing/impl/ForwardIndex.java @@ -17,17 +17,24 @@ package com.intellij.util.indexing.impl; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - import java.io.IOException; -import java.util.Iterator; import java.util.Map; +/** + * Represents a forward index data structure: + * an index indented to hold a mappings of inputId-s to contained keys. + */ @ApiStatus.Experimental public interface ForwardIndex { + /** + * Creates a diff builder for given inputId. + */ @NotNull - InputKeyIterator getInputKeys(int inputId) throws IOException; + InputDataDiffBuilder getDiffBuilder(int inputId) throws IOException; + /** + * Update data for inputId. + */ void putInputData(int inputId, @NotNull Map data) throws IOException; void flush(); @@ -35,9 +42,4 @@ public interface ForwardIndex { void clear() throws IOException; void close() throws IOException; - - @ApiStatus.Experimental - interface InputKeyIterator extends Iterator { - boolean isAssociatedValueEqual(@Nullable Value value); - } } diff --git a/platform/util/src/com/intellij/util/indexing/impl/InputDataDiffBuilder.java b/platform/util/src/com/intellij/util/indexing/impl/InputDataDiffBuilder.java new file mode 100644 index 000000000000..caa9f8844703 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/InputDataDiffBuilder.java @@ -0,0 +1,39 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.StorageException; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +import java.util.Map; + +/** + * A class intended to make a diff between existing forward index data and new one. + */ +@ApiStatus.Experimental +public abstract class InputDataDiffBuilder { + protected final int myInputId; + + protected InputDataDiffBuilder(int id) {myInputId = id;} + /** + * produce a diff between existing data and newData and consume result to addProcessor, updateProcessor and removeProcessor. + */ + public abstract void differentiate(@NotNull Map newData, + @NotNull KeyValueUpdateProcessor addProcessor, + @NotNull KeyValueUpdateProcessor updateProcessor, + @NotNull RemovedKeyProcessor removeProcessor) throws StorageException; +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/KeyValueUpdateProcessor.java b/platform/util/src/com/intellij/util/indexing/impl/KeyValueUpdateProcessor.java new file mode 100644 index 000000000000..0592120f4d3e --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/KeyValueUpdateProcessor.java @@ -0,0 +1,24 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.StorageException; +import org.jetbrains.annotations.ApiStatus; + +@ApiStatus.Experimental +public interface KeyValueUpdateProcessor { + void process(Key key, Value value, int inputId) throws StorageException; +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java b/platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java index db25c003b972..0a1b5362042c 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java +++ b/platform/util/src/com/intellij/util/indexing/impl/MapBasedForwardIndex.java @@ -39,8 +39,8 @@ public abstract class MapBasedForwardIndex extends AbstractForwardIn @NotNull @Override - public InputKeyIterator getInputKeys(final int inputId) throws IOException { - return new CollectionInputKeyIterator(myInputsIndex.get(inputId)); + public InputDataDiffBuilder getDiffBuilder(final int inputId) throws IOException { + return new CollectionInputDataDiffBuilder(inputId, myInputsIndex.get(inputId)); } @NotNull diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapInputDataDiffBuilder.java b/platform/util/src/com/intellij/util/indexing/impl/MapInputDataDiffBuilder.java new file mode 100644 index 000000000000..9b74dfcac6cd --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/MapInputDataDiffBuilder.java @@ -0,0 +1,133 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Comparing; +import com.intellij.util.SystemProperties; +import com.intellij.util.indexing.StorageException; +import gnu.trove.THashMap; +import gnu.trove.TObjectObjectProcedure; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +@ApiStatus.Experimental +public class MapInputDataDiffBuilder extends InputDataDiffBuilder { + private static final boolean ourDiffUpdateEnabled = SystemProperties.getBooleanProperty("idea.disable.diff.index.update", true); + + private final Map myMap; + + public MapInputDataDiffBuilder(int inputId, @Nullable Map map) { + super(inputId); + myMap = map == null ? Collections.emptyMap() : map; + } + + @Override + public void differentiate(@NotNull Map newData, + @NotNull KeyValueUpdateProcessor addProcessor, + @NotNull KeyValueUpdateProcessor updateProcessor, + @NotNull RemovedKeyProcessor removeProcessor) throws StorageException { + if (ourDiffUpdateEnabled) { + if (myMap.isEmpty()) { + EmptyInputDataDiffBuilder.processKeys(newData, addProcessor, myInputId); + incrementalAdditions.addAndGet(newData.size()); + } + else if (newData.isEmpty()) { + processAllKeysAsDeleted(removeProcessor); + incrementalRemovals.addAndGet(myMap.size()); + } + else { + int added = 0; + int removed = 0; + + for (Map.Entry e: myMap.entrySet()) { + final Key key = e.getKey(); + final Value newValue = newData.get(key); + if (!Comparing.equal(e.getValue(), newValue) || (newValue == null && !newData.containsKey(key))) { + if (!newData.containsKey(key)) { + removeProcessor.process(key, myInputId); + removed++; + } else { + updateProcessor.process(key, newValue, myInputId); + added++; + removed++; + } + } + } + + for (Map.Entry e : newData.entrySet()) { + final Key key = e.getKey(); + if (!myMap.containsKey(key)) { + addProcessor.process(key, e.getValue(), myInputId); + added++; + } + } + + incrementalAdditions.addAndGet(added); + incrementalRemovals.addAndGet(removed); + } + + int totalRequests = requests.incrementAndGet(); + totalRemovals.addAndGet(myMap.size()); + totalAdditions.addAndGet(newData.size()); + + if ((totalRequests & 0xFFF) == 0 && DebugAssertions.DEBUG) { + Logger.getInstance(getClass()).info("Incremental index diff update:" + requests + + ", removals:" + totalRemovals + "->" + incrementalRemovals + + ", additions:" + totalAdditions + "->" + incrementalAdditions); + } + } + else { + CollectionInputDataDiffBuilder.differentiateWithKeySeq(myMap.keySet(), newData, myInputId, addProcessor, removeProcessor); + } + } + + private void processAllKeysAsDeleted(final RemovedKeyProcessor removeProcessor) throws StorageException { + if (myMap instanceof THashMap) { + final StorageException[] exception = new StorageException[]{null}; + ((THashMap)myMap).forEachEntry(new TObjectObjectProcedure() { + @Override + public boolean execute(Key k, Value v) { + try { + removeProcessor.process(k, myInputId); + } + catch (StorageException e) { + exception[0] = e; + return false; + } + return true; + } + }); + if (exception[0] != null) throw exception[0]; + } + else { + for (Key key : myMap.keySet()) { + removeProcessor.process(key, myInputId); + } + } + } + + private static final AtomicInteger requests = new AtomicInteger(); + private static final AtomicInteger totalRemovals = new AtomicInteger(); + private static final AtomicInteger totalAdditions = new AtomicInteger(); + private static final AtomicInteger incrementalRemovals = new AtomicInteger(); + private static final AtomicInteger incrementalAdditions = new AtomicInteger(); +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapInputKeyIterator.java b/platform/util/src/com/intellij/util/indexing/impl/MapInputKeyIterator.java deleted file mode 100644 index 837b10c220e1..000000000000 --- a/platform/util/src/com/intellij/util/indexing/impl/MapInputKeyIterator.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.indexing.impl; - -import com.intellij.openapi.util.Comparing; -import org.jetbrains.annotations.Nullable; - -import java.util.Collections; -import java.util.Iterator; -import java.util.Map; - -public class MapInputKeyIterator implements ForwardIndex.InputKeyIterator { - private final Map myMap; - private Iterator> myIterator; - private Value myCurrentValue; - - public MapInputKeyIterator(Map map) { - myMap = map; - } - - @Override - public boolean isAssociatedValueEqual(@Nullable Value value) { - return Comparing.equal(myCurrentValue, value); - } - - @Override - public boolean hasNext() { - init(); - return myIterator.hasNext(); - } - - @Override - public Key next() { - Map.Entry entry = myIterator.next(); - myCurrentValue = entry.getValue(); - return entry.getKey(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - private void init() { - if (myIterator == null) { - myIterator = (myMap == null ? Collections.emptyMap() : myMap).entrySet().iterator(); - } - } -} diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java index df8395f16d0c..a210623abab9 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java +++ b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java @@ -51,7 +51,6 @@ public abstract class MapReduceIndex implements InvertedIndex< private final DataIndexer myIndexer; protected volatile ForwardIndex myForwardIndex; - private final boolean myUseDiffUpdate; private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); private volatile boolean myDisposed; @@ -85,9 +84,6 @@ public abstract class MapReduceIndex implements InvertedIndex< myStorage = storage; myValueExternalizer = extension.getValueExternalizer(); myForwardIndex = forwardIndex; - myUseDiffUpdate = DiffUpdateData.ourDiffUpdateEnabled && (forwardIndex == null || - myForwardIndex instanceof AbstractForwardIndex && - !((AbstractForwardIndex)myForwardIndex).hasOnlyKeysData()); } @NotNull @@ -222,10 +218,10 @@ public abstract class MapReduceIndex implements InvertedIndex< @NotNull protected UpdateData calculateUpdateData(final int inputId, @Nullable Input content) { final Map data = mapInput(content); - return createUpdateData(data, new ThrowableComputable, IOException>() { + return createUpdateData(data, new ThrowableComputable, IOException>() { @Override - public ForwardIndex.InputKeyIterator compute() throws IOException { - return readInputKeys(inputId); + public InputDataDiffBuilder compute() throws IOException { + return getKeysDiffBuilder(inputId); } }, new ThrowableRunnable() { @Override @@ -236,16 +232,15 @@ public abstract class MapReduceIndex implements InvertedIndex< } @NotNull - protected ForwardIndex.InputKeyIterator readInputKeys(int inputId) throws IOException { - return myForwardIndex.getInputKeys(inputId); + protected InputDataDiffBuilder getKeysDiffBuilder(int inputId) throws IOException { + return myForwardIndex.getDiffBuilder(inputId); } @NotNull protected UpdateData createUpdateData(Map data, - ThrowableComputable, IOException> keys, + ThrowableComputable, IOException> keys, ThrowableRunnable forwardIndexUpdate) { - return myUseDiffUpdate ? new DiffUpdateData(data, keys, myIndexId, forwardIndexUpdate) - : new SimpleUpdateData(data, keys, myIndexId, forwardIndexUpdate); + return new UpdateData(data, keys, myIndexId, forwardIndexUpdate); } protected Map mapInput(Input content) { @@ -268,8 +263,8 @@ public abstract class MapReduceIndex implements InvertedIndex< return myModificationStamp.get(); } - private final UpdateData.RemovedKeyProcessor - myRemovedKeyProcessor = new UpdateData.RemovedKeyProcessor() { + private final RemovedKeyProcessor + myRemovedKeyProcessor = new RemovedKeyProcessor() { @Override public void process(Key key, int inputId) throws StorageException { myModificationStamp.incrementAndGet(); @@ -277,7 +272,7 @@ public abstract class MapReduceIndex implements InvertedIndex< } }; - private final UpdateData.KeyValueUpdateProcessor myAddedKeyProcessor = new UpdateData.KeyValueUpdateProcessor() { + private final KeyValueUpdateProcessor myAddedKeyProcessor = new KeyValueUpdateProcessor() { @Override public void process(Key key, Value value, int inputId) throws StorageException { myModificationStamp.incrementAndGet(); @@ -285,7 +280,7 @@ public abstract class MapReduceIndex implements InvertedIndex< } }; - private final UpdateData.KeyValueUpdateProcessor myUpdatedKeyProcessor = new UpdateData.KeyValueUpdateProcessor() { + private final KeyValueUpdateProcessor myUpdatedKeyProcessor = new KeyValueUpdateProcessor() { @Override public void process(Key key, Value value, int inputId) throws StorageException { myModificationStamp.incrementAndGet(); @@ -300,7 +295,7 @@ public abstract class MapReduceIndex implements InvertedIndex< try { try { ValueContainerImpl.ourDebugIndexInfo.set(myIndexId); - updateData.iterateKeys(inputId, myAddedKeyProcessor, myUpdatedKeyProcessor, myRemovedKeyProcessor); + updateData.iterateKeys(myAddedKeyProcessor, myUpdatedKeyProcessor, myRemovedKeyProcessor); updateData.updateForwardIndex(); } catch (ProcessCanceledException e) { diff --git a/platform/util/src/com/intellij/util/indexing/impl/RemovedKeyProcessor.java b/platform/util/src/com/intellij/util/indexing/impl/RemovedKeyProcessor.java new file mode 100644 index 000000000000..812c1fb86a85 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/RemovedKeyProcessor.java @@ -0,0 +1,24 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.StorageException; +import org.jetbrains.annotations.ApiStatus; + +@ApiStatus.Experimental +public interface RemovedKeyProcessor { + void process(Key key, int inputId) throws StorageException; +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/SimpleUpdateData.java b/platform/util/src/com/intellij/util/indexing/impl/SimpleUpdateData.java deleted file mode 100644 index 0585e6f63e8c..000000000000 --- a/platform/util/src/com/intellij/util/indexing/impl/SimpleUpdateData.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2000-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.util.indexing.impl; - -import com.intellij.openapi.util.ThrowableComputable; -import com.intellij.util.ThrowableRunnable; -import com.intellij.util.indexing.ID; -import com.intellij.util.indexing.StorageException; -import org.jetbrains.annotations.NotNull; - -import java.io.IOException; -import java.util.Map; - -public class SimpleUpdateData extends UpdateData { - public SimpleUpdateData(@NotNull Map newData, - @NotNull ThrowableComputable, IOException> currentData, - @NotNull ID indexId, - ThrowableRunnable forwardIndexUpdate) { - super(newData, currentData, indexId, forwardIndexUpdate); - } - - @Override - public void iterateKeys(int inputId, - KeyValueUpdateProcessor addProcessor, - KeyValueUpdateProcessor updateProcessor, - RemovedKeyProcessor removeProcessor) throws StorageException { - final ForwardIndex.InputKeyIterator currentData; - try { - currentData = myCurrentData.compute(); - } - catch (IOException e) { - throw new StorageException(e); - } - iterateKeys(inputId, addProcessor, removeProcessor, currentData); - } - - protected void iterateKeys(int inputId, - KeyValueUpdateProcessor addProcessor, - RemovedKeyProcessor removeProcessor, ForwardIndex.InputKeyIterator currentData) - throws StorageException { - while (currentData.hasNext()) { - removeProcessor.process(currentData.next(), inputId); - } - for (Map.Entry entry : myNewData.entrySet()) { - addProcessor.process(entry.getKey(), entry.getValue(), inputId); - } - } -} diff --git a/platform/util/src/com/intellij/util/indexing/impl/UpdateData.java b/platform/util/src/com/intellij/util/indexing/impl/UpdateData.java index 005da4a2e5d6..6f0e3eb04a7a 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/UpdateData.java +++ b/platform/util/src/com/intellij/util/indexing/impl/UpdateData.java @@ -27,39 +27,43 @@ import java.io.IOException; import java.util.Map; @ApiStatus.Experimental -public abstract class UpdateData { +public class UpdateData { protected final Map myNewData; - protected final ThrowableComputable, IOException> myCurrentData; + protected final ThrowableComputable, IOException> myCurrentDataEvaluator; private final ID myIndexId; private final ThrowableRunnable myForwardIndexUpdate; - protected UpdateData(@NotNull Map newData, - @NotNull ThrowableComputable, IOException> currentData, - @NotNull ID indexId, - @Nullable ThrowableRunnable forwardIndexUpdate) { + public UpdateData(@NotNull Map newData, + @NotNull ThrowableComputable, IOException> currentDataEvaluator, + @NotNull ID indexId, + @Nullable ThrowableRunnable forwardIndexUpdate) { myNewData = newData; - myCurrentData = currentData; + myCurrentDataEvaluator = currentDataEvaluator; myIndexId = indexId; myForwardIndexUpdate = forwardIndexUpdate; } - public abstract void iterateKeys(final int inputId, - final KeyValueUpdateProcessor addProcessor, - final KeyValueUpdateProcessor updateProcessor, - final RemovedKeyProcessor removeProcessor) throws StorageException; + public void iterateKeys(KeyValueUpdateProcessor addProcessor, + KeyValueUpdateProcessor updateProcessor, + RemovedKeyProcessor removeProcessor) throws StorageException { + final InputDataDiffBuilder currentData; + try { + currentData = getCurrentDataEvaluator().compute(); + } + catch (IOException e) { + throw new StorageException(e); + } + currentData.differentiate(myNewData, addProcessor, updateProcessor, removeProcessor); + } - public Map getNewData() { + protected ThrowableComputable, IOException> getCurrentDataEvaluator() { + return myCurrentDataEvaluator; + } + + protected Map getNewData() { return myNewData; } - public interface KeyValueUpdateProcessor { - void process(Key key, Value value, int inputId) throws StorageException; - } - - public interface RemovedKeyProcessor { - void process(Key key, int inputId) throws StorageException; - } - public ID getIndexId() { return myIndexId; } diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java index 430ab646b9bc..a2054c55fe44 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/VcsLogFullDetailsIndex.java @@ -20,10 +20,7 @@ import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.util.Disposer; import com.intellij.util.Consumer; import com.intellij.util.indexing.*; -import com.intellij.util.indexing.impl.EmptyInputKeyIterator; -import com.intellij.util.indexing.impl.ForwardIndex; -import com.intellij.util.indexing.impl.MapIndexStorage; -import com.intellij.util.indexing.impl.MapReduceIndex; +import com.intellij.util.indexing.impl.*; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.EnumeratorIntegerDescriptor; import com.intellij.util.io.KeyDescriptor; @@ -194,8 +191,8 @@ public class VcsLogFullDetailsIndex implements Disposable { private static class EmptyForwardIndex implements ForwardIndex { @NotNull @Override - public InputKeyIterator getInputKeys(int inputId) { - return EmptyInputKeyIterator.getInstance(); + public InputDataDiffBuilder getDiffBuilder(int inputId) { + return new EmptyInputDataDiffBuilder<>(inputId); } @Override