MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

less memory for storing indices (optimize for one pair of ket / value - 80% case) (cherry picked from commit 37a7f7b)

Browse Source
This commit is contained in:
Maxim.Mossienko
2012-05-15 09:36:04 +04:00
parent 6b6e2e19a7
commit be5278d42e
6 changed files with 203 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/util/indexing/ChangeTrackingValueContainer.java
+59 -62
View File
@@ -19,6 +19,7 @@ package com.intellij.util.indexing;
import com.intellij.openapi.util.Computable;
import gnu.trove.TIntHashSet;
import gnu.trove.TIntProcedure;
import org.jetbrains.annotations.Nullable;
import java.util.Iterator;
import java.util.List;
@@ -28,11 +29,11 @@ import java.util.List;
* Date: Dec 20, 2007
*/
class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>{
private final ValueContainerImpl<Value> myAdded;
private final ValueContainerImpl<Value> myRemoved;
private final TIntHashSet myInvalidated;
// there is no volatile as we modify under write lock and read under read lock
private ValueContainerImpl<Value> myAdded;
private TIntHashSet myInvalidated;
private volatile ValueContainerImpl<Value> myMerged;
private final Initializer<Value> myInitializer;
private volatile ValueContainerImpl<Value> myMerged = null;
public interface Initializer<T> extends Computable<ValueContainer<T>> {
Object getLock();
@@ -40,51 +41,48 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
public ChangeTrackingValueContainer(Initializer<Value> initializer) {
myInitializer = initializer;
myAdded = new ValueContainerImpl<Value>();
myRemoved = new ValueContainerImpl<Value>();
myInvalidated = new TIntHashSet(1);
}
//public void log(String op, int id, final Value value) {
// System.out.print("@" + mcount + ": ");
// System.out.print(op);
// System.out.print("(" + id + ")");
// System.out.print(" value=" + value + " ");
// System.out.print("+[" + myAdded.dumpInputIdMapping() + "], ");
// System.out.print("-[" + myRemoved.dumpInputIdMapping() + "], ");
// System.out.println("*[" + (myMerged != null ? myMerged.dumpInputIdMapping() : "null") + "] ");
//}
@Override
public void addValue(int inputId, Value value) {
if (myMerged != null) {
myMerged.addValue(inputId, value);
ValueContainerImpl<Value> merged = myMerged;
if (merged != null) {
merged.addValue(inputId, value);
}
if (!myRemoved.removeValue(inputId, value)) {
myAdded.addValue(inputId, value);
ValueContainerImpl<Value> added = myAdded;
if (added == null) {
myAdded = added = new ValueContainerImpl<Value>();
}
added.addValue(inputId, value); // will flush the changes & caller should ensure exclusiveness to avoid intermediate visibility issues
}
@Override
public void removeAssociatedValue(int inputId) {
if (myMerged != null) {
myMerged.removeAssociatedValue(inputId);
ValueContainerImpl<Value> merged = myMerged;
if (merged != null) {
merged.removeAssociatedValue(inputId);
}
myAdded.removeAssociatedValue(inputId);
myRemoved.removeAssociatedValue(inputId);
myInvalidated.add(inputId);
ValueContainerImpl<Value> added = myAdded;
if (added != null) added.removeAssociatedValue(inputId);
TIntHashSet invalidated = myInvalidated;
if (invalidated == null) {
invalidated = new TIntHashSet(1);
}
invalidated.add(inputId);
myInvalidated = invalidated; // volatile write
}
@Override
public boolean removeValue(int inputId, Value value) {
if (myMerged != null) {
myMerged.removeValue(inputId, value);
}
if (!myAdded.removeValue(inputId, value)) {
if (!myInvalidated.contains(inputId)) {
myRemoved.addValue(inputId, value);
}
ValueContainerImpl<Value> merged = myMerged;
if (merged != null) {
merged.removeValue(inputId, value);
}
ValueContainerImpl<Value> added = myAdded;
if (added != null) added.removeValue(inputId, value);
return true;
}
@@ -143,28 +141,29 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
} else {
newMerged = ((ChangeTrackingValueContainer<Value>)fromDisk).getMergedData().copy();
}
myInvalidated.forEach(new TIntProcedure() {
@Override
public boolean execute(int inputId) {
newMerged.removeAssociatedValue(inputId);
return true;
}
});
myRemoved.forEach(new ContainerAction<Value>() {
@Override
public boolean perform(final int id, final Value value) {
newMerged.removeValue(id, value);
return true;
}
});
myAdded.forEach(new ContainerAction<Value>() {
@Override
public boolean perform(final int id, final Value value) {
newMerged.removeAssociatedValue(id); // enforcing "one-value-per-file for particular key" invariant
newMerged.addValue(id, value);
return true;
}
});
TIntHashSet invalidated = myInvalidated;
if (invalidated != null) {
invalidated.forEach(new TIntProcedure() {
@Override
public boolean execute(int inputId) {
newMerged.removeAssociatedValue(inputId);
return true;
}
});
}
ValueContainerImpl<Value> added = myAdded;
if (added != null) {
added.forEach(new ContainerAction<Value>() {
@Override
public boolean perform(final int id, final Value value) {
newMerged.removeAssociatedValue(id); // enforcing "one-value-per-file for particular key" invariant
newMerged.addValue(id, value);
return true;
}
});
}
setNeedsCompacting(fromDisk.needsCompacting());
myMerged = newMerged;
@@ -173,18 +172,16 @@ class ChangeTrackingValueContainer<Value> extends UpdatableValueContainer<Value>
}
public boolean isDirty() {
return myAdded.size() > 0 || myRemoved.size() > 0 || !myInvalidated.isEmpty() || needsCompacting();
return (myAdded != null && myAdded.size() > 0) ||
(myInvalidated != null && !myInvalidated.isEmpty()) ||
needsCompacting();
}
public ValueContainer<Value> getAddedDelta() {
public @Nullable ValueContainer<Value> getAddedDelta() {
return myAdded;
}
public ValueContainer<Value> getRemovedDelta() {
return myRemoved;
}
public TIntHashSet getInvalidated() {
public @Nullable TIntHashSet getInvalidated() {
return myInvalidated;
}
}
platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java
-2
View File
@@ -31,8 +31,6 @@ public interface IndexStorage<Key, Value> extends Flushable {
void addValue(Key key, int inputId, Value value) throws StorageException;
void removeValue(Key key, int inputId, Value value) throws StorageException;
void removeAllValues(Key key, int inputId) throws StorageException;
void clear() throws StorageException;
platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java
-11
View File
@@ -237,17 +237,6 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
}
}
@Override
public void removeValue(final Key key, final int inputId, final Value value) throws StorageException {
try {
myMap.markDirty();
read(key).removeValue(inputId, value);
}
catch (IOException e) {
throw new StorageException(e);
}
}
@Override
public void removeAllValues(Key key, int inputId) throws StorageException {
try {
platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java
-13
View File
@@ -144,19 +144,6 @@ public class MemoryIndexStorage<Key, Value> implements IndexStorage<Key, Value>
myBackendStorage.addValue(key, inputId, value);
}
@Override
public void removeValue(final Key key, final int inputId, final Value value) throws StorageException {
if (myBufferingEnabled.get()) {
getMemValueContainer(key).removeValue(inputId, value);
return;
}
final ChangeTrackingValueContainer<Value> valueContainer = myMap.get(key);
if (valueContainer != null) {
valueContainer.dropMergedData();
}
myBackendStorage.removeValue(key, inputId, value);
}
@Override
public void removeAllValues(Key key, int inputId) throws StorageException {
if (myBufferingEnabled.get()) {
platform/lang-impl/src/com/intellij/util/indexing/ValueContainerImpl.java
+139 -75
View File
@@ -33,27 +33,37 @@ import java.util.*;
class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implements Cloneable{
private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.ValueContainerImpl");
private final static Object myNullValue = new Object();
private THashMap<Value, Object> myInputIdMapping;
public ValueContainerImpl() {
// per statistic most maps (80%) has one value
myInputIdMapping = new THashMap<Value, Object>(1);
}
// there is no volatile as we modify under write lock and read under read lock
// Most often (80%) we store 0 or one mapping, then we store them in two fields: myInputIdMapping, myInputIdMappingValue
// when there are several value mapped, myInputIdMapping is THashMap<Value, Data>, myInputIdMappingValue = null
private Object myInputIdMapping;
private Object myInputIdMappingValue;
@Override
public void addValue(int inputId, Value value) {
value = maskNull(value);
final Object input = myInputIdMapping.get(value);
final Object input = getInput(value);
if (input == null) {
//idSet = new TIntHashSet(3, 0.98f);
myInputIdMapping.put(value, inputId);
if (myInputIdMapping != null) {
if (!(myInputIdMapping instanceof THashMap)) {
Object oldMapping = myInputIdMapping;
myInputIdMapping = new THashMap<Value, Object>(2);
((THashMap<Value, Object>)myInputIdMapping).put((Value)oldMapping, myInputIdMappingValue);
myInputIdMappingValue = null;
}
((THashMap<Value, Object>)myInputIdMapping).put(value, inputId);
} else {
myInputIdMapping = value != null ? value:(Value)myNullValue;
myInputIdMappingValue = inputId;
}
}
else {
final TIntHashSet idSet;
if (input instanceof Integer) {
idSet = new IdSet(3, 0.98f);
idSet.add(((Integer)input).intValue());
myInputIdMapping.put(value, idSet);
if (!(myInputIdMapping instanceof THashMap)) myInputIdMappingValue = idSet;
else ((THashMap<Value, Object>)myInputIdMapping).put(value, idSet);
}
else {
idSet = (TIntHashSet)input;
@@ -64,21 +74,19 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
@Override
public int size() {
return myInputIdMapping.size();
return myInputIdMapping != null ? myInputIdMapping instanceof THashMap ? ((THashMap)myInputIdMapping).size(): 1 : 0;
}
@Override
public void removeAssociatedValue(int inputId) {
if (myInputIdMapping.isEmpty()) return;
if (myInputIdMapping == null) return;
List<Value> toRemove = null;
for (final Iterator<Value> valueIterator = getValueIterator(); valueIterator.hasNext();) {
final Value value = valueIterator.next();
if (isAssociated(value, inputId)) {
if (toRemove == null) toRemove = new SmartList<Value>(value);
else {
LOG.error("Expected only one value per-inputId");
toRemove.add(value);
}
if (toRemove == null) toRemove = new SmartList<Value>();
else LOG.error("Expected only one value per-inputId");
toRemove.add(value);
}
}
@@ -91,12 +99,11 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
@Override
public boolean removeValue(int inputId, Value value) {
if (myInputIdMapping.isEmpty()) return false; // skipping hash code for value
value = maskNull(value);
final Object input = myInputIdMapping.get(value);
final Object input = getInput(value);
if (input == null) {
return false;
}
if (input instanceof TIntHashSet) {
final TIntHashSet idSet = (TIntHashSet)input;
final boolean reallyRemoved = idSet.remove(inputId);
@@ -112,57 +119,87 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
return false;
}
}
myInputIdMapping.remove(value);
return true;
}
private Value maskNull(Value value) {
if (value == null) {
return (Value)myNullValue;
if (!(myInputIdMapping instanceof THashMap)) {
myInputIdMapping = null;
myInputIdMappingValue = null;
} else {
THashMap<Value, Object> mapping = (THashMap<Value, Object>)myInputIdMapping;
mapping.remove(value);
if (mapping.size() == 1) {
myInputIdMapping = mapping.keySet().iterator().next();
myInputIdMappingValue = mapping.get((Value)myInputIdMapping);
}
}
return value;
return true;
}
@Override
public Iterator<Value> getValueIterator() {
if (myInputIdMapping.isEmpty()) {
if (myInputIdMapping != null) {
if (!(myInputIdMapping instanceof THashMap)) {
return new Iterator<Value>() {
private Value value = (Value)myInputIdMapping;
@Override
public boolean hasNext() {
return value != null;
}
@Override
public Value next() {
Value next = value;
if (next == myNullValue) next = null;
value = null;
return next;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
} else {
return new Iterator<Value>() {
final Iterator<Value> iterator = ((THashMap<Value, Object>)myInputIdMapping).keySet().iterator();
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public Value next() {
Value next = iterator.next();
if (next == myNullValue) next = null;
return next;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
} else {
return EmptyIterator.getInstance();
}
return new Iterator<Value>() {
final Iterator<Value> iterator = myInputIdMapping.keySet().iterator();
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public Value next() {
Value next = iterator.next();
if (next == myNullValue) next = null;
return next;
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
@Override
public List<Value> toValueList() {
if (myInputIdMapping.isEmpty()) {
if (myInputIdMapping == null) {
return Collections.emptyList();
} else if (myInputIdMapping instanceof THashMap) {
return new ArrayList<Value>(((THashMap<Value, Object>)myInputIdMapping).keySet());
} else {
return new SmartList<Value>((Value)myInputIdMapping);
}
return new ArrayList<Value>(myInputIdMapping.keySet());
}
@Override
public boolean isAssociated(Value value, final int inputId) {
value = maskNull(value);
final Object input = myInputIdMapping.get(value);
final Object input = getInput(value);
if (input instanceof TIntHashSet) {
return ((TIntHashSet)input).contains(inputId);
}
@@ -174,7 +211,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
@Override
public IntPredicate getValueAssociationPredicate(Value value) {
final Object input = myInputIdMapping.get(value);
final Object input = getInput(value);
if (input == null) return EMPTY_PREDICATE;
if (input instanceof Integer) {
return new IntPredicate() {
@@ -196,8 +233,7 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
@Override
public IntIterator getInputIdsIterator(Value value) {
value = maskNull(value);
final Object input = myInputIdMapping.get(value);
final Object input = getInput(value);
final IntIterator it;
if (input instanceof TIntHashSet) {
it = new IntSetIterator((TIntHashSet)input);
@@ -211,11 +247,30 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
return it;
}
private Object getInput(Value value) {
if (myInputIdMapping == null) return null;
value = value != null ? value:(Value)myNullValue;
if (myInputIdMapping == value || // myNullValue is Object
myInputIdMapping.equals(value)
) {
return myInputIdMappingValue;
}
if (!(myInputIdMapping instanceof THashMap)) return null;
return ((THashMap<Value, Object>)myInputIdMapping).get(value);
}
@Override
public ValueContainerImpl<Value> clone() {
try {
final ValueContainerImpl clone = (ValueContainerImpl)super.clone();
clone.myInputIdMapping = mapCopy(myInputIdMapping);
if (myInputIdMapping instanceof THashMap) {
clone.myInputIdMapping = mapCopy((THashMap<Value, Object>)myInputIdMapping);
} else if (myInputIdMappingValue instanceof TIntHashSet) {
clone.myInputIdMappingValue = ((TIntHashSet)myInputIdMappingValue).clone();
}
return clone;
}
catch (CloneNotSupportedException e) {
@@ -241,18 +296,30 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
};
public ValueContainerImpl<Value> copy() {
final ValueContainerImpl<Value> container = new ValueContainerImpl<Value>();
myInputIdMapping.forEachEntry(new TObjectObjectProcedure<Value, Object>() {
@Override
public boolean execute(Value key, Object val) {
if (val instanceof TIntHashSet) {
container.myInputIdMapping.put(key, ((TIntHashSet)val).clone());
} else {
container.myInputIdMapping.put(key, val);
ValueContainerImpl<Value> container = new ValueContainerImpl<Value>();
if (myInputIdMapping instanceof THashMap) {
final THashMap<Value, Object> mapping = (THashMap<Value, Object>)myInputIdMapping;
final THashMap<Value, Object> newMapping = new THashMap<Value, Object>(mapping.size());
container.myInputIdMapping = newMapping;
mapping.forEachEntry(new TObjectObjectProcedure<Value, Object>() {
@Override
public boolean execute(Value key, Object val) {
if (val instanceof TIntHashSet) {
newMapping.put(key, ((TIntHashSet)val).clone());
}
else {
newMapping.put(key, val);
}
return true;
}
return true;
}
});
});
} else {
container.myInputIdMapping = myInputIdMapping;
container.myInputIdMappingValue = myInputIdMappingValue instanceof TIntHashSet ?
((TIntHashSet)myInputIdMappingValue).clone():myInputIdMappingValue;
}
return container;
}
@@ -271,12 +338,9 @@ class ValueContainerImpl<Value> extends UpdatableValueContainer<Value> implement
@Override
public int next() {
try {
return myValue;
}
finally {
myValueRead = true;
}
int next = myValue;
myValueRead = true;
return next;
}
@Override
platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java
+5 -13
View File
@@ -38,18 +38,14 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
//noinspection IOResourceOpenedButNotSafelyClosed
final DataOutputStream _out = new DataOutputStream(bytes);
final TIntHashSet set = valueContainer.getInvalidated();
if (set.size() > 0) {
if (set != null && set.size() > 0) {
for (int inputId : set.toArray()) {
ValueContainerExternalizer.saveInvalidateCommand(_out, inputId);
}
}
final ValueContainer<Value> toRemove = valueContainer.getRemovedDelta();
if (toRemove.size() > 0) {
myValueContainerExternalizer.saveAsRemoved(_out, toRemove);
}
final ValueContainer<Value> toAppend = valueContainer.getAddedDelta();
if (toAppend.size() > 0) {
if (toAppend != null && toAppend.size() > 0) {
myValueContainerExternalizer.save(_out, toAppend);
}
@@ -76,18 +72,14 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
@Override
public void save(final DataOutput out, @NotNull final ValueContainer<T> container) throws IOException {
saveImpl(out, container, false);
}
public void saveAsRemoved(final DataOutput out, @NotNull final ValueContainer<T> container) throws IOException {
saveImpl(out, container, true);
saveImpl(out, container);
}
public static void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException {
DataInputOutputUtil.writeSINT(out, -inputId);
}
private void saveImpl(final DataOutput out, @NotNull final ValueContainer<T> container, final boolean asRemovedData) throws IOException {
private void saveImpl(final DataOutput out, @NotNull final ValueContainer<T> container) throws IOException {
DataInputOutputUtil.writeSINT(out, container.size());
for (final Iterator<T> valueIterator = container.getValueIterator(); valueIterator.hasNext();) {
final T value = valueIterator.next();
@@ -98,7 +90,7 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContaine
DataInputOutputUtil.writeSINT(out, ids.size());
while (ids.hasNext()) {
final int id = ids.next();
DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id);
DataInputOutputUtil.writeSINT(out, id);
}
}
else {