diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java index d19589c47fba..63cf323519f8 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java @@ -17,22 +17,16 @@ package com.intellij.util.indexing; import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.CommonProcessors; import com.intellij.util.Processor; import com.intellij.util.containers.SLRUCache; -import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.DataInputOutputUtil; -import com.intellij.util.io.KeyDescriptor; -import com.intellij.util.io.PersistentHashMap; -import gnu.trove.TIntHashSet; +import com.intellij.util.io.*; import org.jetbrains.annotations.NotNull; import java.io.*; import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -43,28 +37,29 @@ import java.util.concurrent.locks.ReentrantLock; */ public final class MapIndexStorage implements IndexStorage{ private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage"); - private PersistentHashMap> myMap; + private PersistentMap> myMap; private SLRUCache> myCache; private final File myStorageFile; private final KeyDescriptor myKeyDescriptor; - private final ValueContainerExternalizer myValueContainerExternalizer; private final int myCacheSize; private final Lock l = new ReentrantLock(); + private final DataExternalizer myDataExternalizer; - public MapIndexStorage(File storageFile, final KeyDescriptor keyDescriptor, final DataExternalizer valueExternalizer, + public MapIndexStorage(File storageFile, final KeyDescriptor keyDescriptor, + final DataExternalizer valueExternalizer, final int cacheSize) throws IOException { myStorageFile = storageFile; myKeyDescriptor = keyDescriptor; - myValueContainerExternalizer = new ValueContainerExternalizer(valueExternalizer); myCacheSize = cacheSize; + myDataExternalizer = valueExternalizer; initMapAndCache(); } private void initMapAndCache() throws IOException { - final PersistentHashMap> map = - new PersistentHashMap>(myStorageFile, myKeyDescriptor, myValueContainerExternalizer); + final PersistentMap> map = + new ValueContainerMap(myStorageFile, myKeyDescriptor, myDataExternalizer); myCache = new SLRUCache>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) { @NotNull public ChangeTrackingValueContainer createValue(final Key key) { @@ -74,7 +69,7 @@ public final class MapIndexStorage implements IndexStorage compute() { - ValueContainer value = null; + ValueContainer value; try { value = map.get(key); if (value == null) { @@ -90,43 +85,13 @@ public final class MapIndexStorage implements IndexStorage valueContainer) { - if (!valueContainer.isDirty()) { - return; - } - try { - if (!valueContainer.needsCompacting()) { - final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); - //noinspection IOResourceOpenedButNotSafelyClosed - final DataOutputStream _out = new DataOutputStream(bytes); - final TIntHashSet set = valueContainer.getInvalidated(); - if (set.size() > 0) { - for (int inputId : set.toArray()) { - myValueContainerExternalizer.saveInvalidateCommand(_out, inputId); - } - } - final ValueContainer toRemove = valueContainer.getRemovedDelta(); - if (toRemove.size() > 0) { - myValueContainerExternalizer.saveAsRemoved(_out, toRemove); - } - - final ValueContainer toAppend = valueContainer.getAddedDelta(); - if (toAppend.size() > 0) { - myValueContainerExternalizer.save(_out, toAppend); - } - - map.appendData(key, new PersistentHashMap.ValueDataAppender() { - public void append(final DataOutput out) throws IOException { - out.write(bytes.getInternalBuffer(), 0, bytes.size()); - } - }); - } - else { - // rewrite the value container for defragmentation + if (valueContainer.isDirty()) { + try { map.put(key, valueContainer); } - } - catch (IOException e) { - throw new RuntimeException(e); + catch (IOException e) { + throw new RuntimeException(e); + } } } }; @@ -275,73 +240,4 @@ public final class MapIndexStorage implements IndexStorage implements DataExternalizer> { - private final DataExternalizer myExternalizer; - - private ValueContainerExternalizer(DataExternalizer externalizer) { - myExternalizer = externalizer; - } - - public void save(final DataOutput out, final ValueContainer container) throws IOException { - saveImpl(out, container, false); - } - - public void saveAsRemoved(final DataOutput out, final ValueContainer container) throws IOException { - saveImpl(out, container, true); - } - - public void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException { - DataInputOutputUtil.writeSINT(out, -inputId); - } - - private void saveImpl(final DataOutput out, final ValueContainer container, final boolean asRemovedData) throws IOException { - DataInputOutputUtil.writeSINT(out, container.size()); - for (final Iterator valueIterator = container.getValueIterator(); valueIterator.hasNext();) { - final T value = valueIterator.next(); - myExternalizer.save(out, value); - - final ValueContainer.IntIterator ids = container.getInputIdsIterator(value); - if (ids != null) { - DataInputOutputUtil.writeSINT(out, ids.size()); - while (ids.hasNext()) { - final int id = ids.next(); - DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id); - } - } - else { - DataInputOutputUtil.writeSINT(out, 0); - } - } - } - - public ValueContainerImpl read(final DataInput in) throws IOException { - DataInputStream stream = (DataInputStream)in; - final ValueContainerImpl valueContainer = new ValueContainerImpl(); - - while (stream.available() > 0) { - final int valueCount = DataInputOutputUtil.readSINT(in); - if (valueCount < 0) { - valueContainer.removeAllValues(-valueCount); - valueContainer.setNeedsCompacting(true); - } - else { - for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) { - final T value = myExternalizer.read(in); - final int idCount = DataInputOutputUtil.readSINT(in); - for (int i = 0; i < idCount; i++) { - final int id = DataInputOutputUtil.readSINT(in); - if (id < 0) { - valueContainer.removeValue(-id, value); - valueContainer.setNeedsCompacting(true); - } - else { - valueContainer.addValue(id, value); - } - } - } - } - } - return valueContainer; - } - } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java b/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java new file mode 100644 index 000000000000..da356061ab0d --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java @@ -0,0 +1,134 @@ +package com.intellij.util.indexing; + +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.DataInputOutputUtil; +import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.PersistentHashMap; +import gnu.trove.TIntHashSet; + +import java.io.*; +import java.util.Iterator; + +/** + * @author Dmitry Avdeev + * Date: 8/10/11 + */ +public class ValueContainerMap extends PersistentHashMap> { + + private final ValueContainerExternalizer myValueContainerExternalizer; + + public ValueContainerMap(final File file, + KeyDescriptor keyKeyDescriptor, + DataExternalizer valueExternalizer) throws IOException { + + super(file, keyKeyDescriptor, new ValueContainerExternalizer(valueExternalizer)); + myValueContainerExternalizer = (ValueContainerExternalizer)myValueExternalizer; + } + + @Override + public synchronized void put(Key key, ValueContainer container) throws IOException { + ChangeTrackingValueContainer valueContainer = (ChangeTrackingValueContainer)container; + if (!valueContainer.needsCompacting()) { + final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); + //noinspection IOResourceOpenedButNotSafelyClosed + final DataOutputStream _out = new DataOutputStream(bytes); + final TIntHashSet set = valueContainer.getInvalidated(); + if (set.size() > 0) { + for (int inputId : set.toArray()) { + ValueContainerExternalizer.saveInvalidateCommand(_out, inputId); + } + } + final ValueContainer toRemove = valueContainer.getRemovedDelta(); + if (toRemove.size() > 0) { + myValueContainerExternalizer.saveAsRemoved(_out, toRemove); + } + + final ValueContainer toAppend = valueContainer.getAddedDelta(); + if (toAppend.size() > 0) { + myValueContainerExternalizer.save(_out, toAppend); + } + + appendData(key, new PersistentHashMap.ValueDataAppender() { + public void append(final DataOutput out) throws IOException { + out.write(bytes.getInternalBuffer(), 0, bytes.size()); + } + }); + } + else { + // rewrite the value container for defragmentation + super.put(key, valueContainer); + } + } + + private static final class ValueContainerExternalizer implements DataExternalizer> { + private final DataExternalizer myExternalizer; + + private ValueContainerExternalizer(DataExternalizer externalizer) { + myExternalizer = externalizer; + } + + public void save(final DataOutput out, final ValueContainer container) throws IOException { + saveImpl(out, container, false); + } + + public void saveAsRemoved(final DataOutput out, final ValueContainer container) throws IOException { + saveImpl(out, container, true); + } + + public static void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException { + DataInputOutputUtil.writeSINT(out, -inputId); + } + + private void saveImpl(final DataOutput out, final ValueContainer container, final boolean asRemovedData) throws IOException { + DataInputOutputUtil.writeSINT(out, container.size()); + for (final Iterator valueIterator = container.getValueIterator(); valueIterator.hasNext();) { + final T value = valueIterator.next(); + myExternalizer.save(out, value); + + final ValueContainer.IntIterator ids = container.getInputIdsIterator(value); + if (ids != null) { + DataInputOutputUtil.writeSINT(out, ids.size()); + while (ids.hasNext()) { + final int id = ids.next(); + DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id); + } + } + else { + DataInputOutputUtil.writeSINT(out, 0); + } + } + } + + public ValueContainerImpl read(final DataInput in) throws IOException { + DataInputStream stream = (DataInputStream)in; + final ValueContainerImpl valueContainer = new ValueContainerImpl(); + + while (stream.available() > 0) { + final int valueCount = DataInputOutputUtil.readSINT(in); + if (valueCount < 0) { + valueContainer.removeAllValues(-valueCount); + valueContainer.setNeedsCompacting(true); + } + else { + for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) { + final T value = myExternalizer.read(in); + final int idCount = DataInputOutputUtil.readSINT(in); + for (int i = 0; i < idCount; i++) { + final int id = DataInputOutputUtil.readSINT(in); + if (id < 0) { + valueContainer.removeValue(-id, value); + valueContainer.setNeedsCompacting(true); + } + else { + valueContainer.addValue(id, value); + } + } + } + } + } + return valueContainer; + } + } + +} diff --git a/platform/util/src/com/intellij/util/io/PersistentHashMap.java b/platform/util/src/com/intellij/util/io/PersistentHashMap.java index 00e1fed872ed..50285ae9bace 100644 --- a/platform/util/src/com/intellij/util/io/PersistentHashMap.java +++ b/platform/util/src/com/intellij/util/io/PersistentHashMap.java @@ -38,11 +38,11 @@ import java.util.List; * @author Eugene Zhuravlev * Date: Dec 18, 2007 */ -public class PersistentHashMap extends PersistentEnumeratorDelegate{ +public class PersistentHashMap extends PersistentEnumeratorDelegate implements PersistentMap { private static final Logger LOG = Logger.getInstance("#com.intellij.util.io.PersistentHashMap"); private PersistentHashMapValueStorage myValueStorage; - private final DataExternalizer myValueExternalizer; + protected final DataExternalizer myValueExternalizer; private static final long NULL_ADDR = 0; private static final int INITIAL_INDEX_SIZE; static { diff --git a/platform/util/src/com/intellij/util/io/PersistentMap.java b/platform/util/src/com/intellij/util/io/PersistentMap.java new file mode 100644 index 000000000000..f39ab275667b --- /dev/null +++ b/platform/util/src/com/intellij/util/io/PersistentMap.java @@ -0,0 +1,29 @@ +package com.intellij.util.io; + +import com.intellij.util.Processor; + +import java.io.IOException; + +/** + * @author Dmitry Avdeev + * Date: 8/10/11 + */ +public interface PersistentMap { + + V get(K key) throws IOException; + + void put(K key, V value) throws IOException; + + boolean processKeys(Processor processor) throws IOException; + + + boolean isClosed(); + + boolean isDirty(); + + void force(); + + void close() throws IOException; + + void markDirty() throws IOException; +}