From aa7feafa829d32856f33fb8b81f98d3f77231ab8 Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Fri, 24 Jun 2016 12:59:12 +0200 Subject: [PATCH] simplier / faster duplicates removal / sorting for small fileId array --- .../util/indexing/ValueContainerImpl.java | 8 +-- .../containers/ChangeBufferingList.java | 71 +++++++++++++------ 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java index beda46464af9..8e779cb70315 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java @@ -21,7 +21,6 @@ import com.intellij.util.SmartList; import com.intellij.util.containers.EmptyIterator; import com.intellij.util.indexing.containers.ChangeBufferingList; import com.intellij.util.indexing.containers.IdSet; -import com.intellij.util.indexing.containers.SortedFileIdSetIterator; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.DataInputOutputUtil; import gnu.trove.THashMap; @@ -447,11 +446,8 @@ class ValueContainerImpl extends UpdatableValueContainer implement } else { // serialize positive file ids with delta encoding ChangeBufferingList originalInput = (ChangeBufferingList)fileSetObject; - IntIterator intIterator = originalInput.rawIntIterator(); - if (!intIterator.hasAscendingOrder()) { - // remove possible dupes - intIterator = SortedFileIdSetIterator.getTransientIterator(intIterator); - } + IntIterator intIterator = originalInput.sortedIntIterator(); + if (DebugAssertions.DEBUG) DebugAssertions.assertTrue(intIterator.hasAscendingOrder()); if (intIterator.size() == 1) { DataInputOutputUtil.writeINT(out, intIterator.next()); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java b/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java index d1a7b6734fdd..57512fb4fd4d 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/containers/ChangeBufferingList.java @@ -19,6 +19,8 @@ import com.intellij.util.indexing.DebugAssertions; import com.intellij.util.indexing.ValueContainer; import gnu.trove.TIntProcedure; +import java.util.Arrays; + import static com.intellij.util.indexing.DebugAssertions.EXTRA_SANITY_CHECKS; /** @@ -122,7 +124,6 @@ public class ChangeBufferingList implements Cloneable { if (randomAccessContainer == null) { int someElementsNumberEstimation = length; - int[] minMax = calcMinMax(changes, length); // todo we can check these lengths instead of only relying upon reaching MAX_FILES //int lengthOfBitSet = IdBitSet.sizeInBytes(minMax[1], minMax[0]); @@ -131,7 +132,7 @@ public class ChangeBufferingList implements Cloneable { if (someElementsNumberEstimation < MAX_FILES) { if (!hasRemovals) { if (mayHaveDupes) { - mergeChangesRemovingDupes(); + removingDupesAndSort(); } idSet = new SortedIdSet(currentChanges, length); @@ -144,7 +145,7 @@ public class ChangeBufferingList implements Cloneable { idSet = new IdBitSet(changes, length, 0); copyChanges = false; } else { - idSet = new IdBitSet(minMax, 0); + idSet = new IdBitSet(calcMinMax(changes, length), 0); } } else if (checkSet != null) { idSet = (RandomAccessIntContainer)randomAccessContainer.clone(); @@ -186,15 +187,40 @@ public class ChangeBufferingList implements Cloneable { } } - private void mergeChangesRemovingDupes() { // duplicated ids can be present for some index due to cancellation of indexing for next index - int[] currentChanges = changes; - ValueContainer.IntIterator sorted = SortedFileIdSetIterator.getTransientIterator(new ChangesIterator(currentChanges, length)); - int lastIndex = 0; - while(sorted.hasNext()) { - currentChanges[lastIndex++] = sorted.next(); - } + private void removingDupesAndSort() { // duplicated ids can be present for some index due to cancellation of indexing for next index + final int[] currentChanges = changes; + final int intLength = length; - length = (short)lastIndex; + if (intLength < 250) { // Plain sorting in Arrays works without allocations for small number of elements (see DualPivotQuicksort.QUICKSORT_THRESHOLD) + Arrays.sort(currentChanges, 0, intLength); + boolean hasDupes = false; + + for(int i = 0, max = intLength - 1; i < max; ++i) { + if (currentChanges[i] == currentChanges[i + 1]) { + hasDupes = true; + break; + } + } + + if (hasDupes) { + int ptr = 0; + for(int i = 1; i < intLength; ++i) { + if (currentChanges[i] != currentChanges[ptr]) { + currentChanges[++ptr] = currentChanges[i]; + } + } + length = (short)(ptr + 1); + } + } else { + ValueContainer.IntIterator sorted = + SortedFileIdSetIterator.getTransientIterator(new ChangesIterator(currentChanges, length, false)); + int lastIndex = 0; + while (sorted.hasNext()) { + currentChanges[lastIndex++] = sorted.next(); + } + + length = (short)lastIndex; + } mayHaveDupes = false; } @@ -262,21 +288,22 @@ public class ChangeBufferingList implements Cloneable { if (currentChanges != null) { if (mayHaveDupes) { synchronized (currentChanges) { - if (mayHaveDupes) mergeChangesRemovingDupes(); + if (mayHaveDupes) removingDupesAndSort(); } } - return new ChangesIterator(currentChanges, length); + return new ChangesIterator(currentChanges, length, true); } } return getRandomAccessContainer().intIterator(); } - public ValueContainer.IntIterator rawIntIterator() { - RandomAccessIntContainer intContainer = randomAccessContainer; - if (intContainer == null && !hasRemovals) { - return new ChangesIterator(changes, length); // dupes are possible + public ValueContainer.IntIterator sortedIntIterator() { + ValueContainer.IntIterator intIterator = intIterator(); + + if (!intIterator.hasAscendingOrder()) { + intIterator = SortedFileIdSetIterator.getTransientIterator(intIterator); } - return getRandomAccessContainer().intIterator(); + return intIterator; } public IdSet getCheckSet() { @@ -287,10 +314,12 @@ public class ChangeBufferingList implements Cloneable { private int cursor; private final int length; private final int[] changes; + private final boolean sorted; - ChangesIterator(int[] _changes, int _length) { + ChangesIterator(int[] _changes, int _length, boolean _sorted) { changes = _changes; length = _length; + sorted = _sorted; } @Override @@ -312,12 +341,12 @@ public class ChangeBufferingList implements Cloneable { @Override public boolean hasAscendingOrder() { - return false; + return sorted; } @Override public ValueContainer.IntIterator createCopyInInitialState() { - return new ChangesIterator(changes, length); + return new ChangesIterator(changes, length, sorted); } } }