MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

PersistentMap interface extracted

Browse Source
This commit is contained in:
Dmitry Avdeev
2011-08-22 13:27:48 +04:00
parent 97579406cb
commit 5b8641347c
4 changed files with 179 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java
+14 -118
View File
@@ -17,22 +17,16 @@
package com.intellij.util.indexing;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.CommonProcessors;
import com.intellij.util.Processor;
import com.intellij.util.containers.SLRUCache;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.DataInputOutputUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import gnu.trove.TIntHashSet;
import com.intellij.util.io.*;
import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -43,28 +37,29 @@ import java.util.concurrent.locks.ReentrantLock;
*/
public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Value>{
private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage");
private PersistentHashMap<Key, ValueContainer<Value>> myMap;
private PersistentMap<Key, ValueContainer<Value>> myMap;
private SLRUCache<Key, ChangeTrackingValueContainer<Value>> myCache;
private final File myStorageFile;
private final KeyDescriptor<Key> myKeyDescriptor;
private final ValueContainerExternalizer<Value> myValueContainerExternalizer;
private final int myCacheSize;
private final Lock l = new ReentrantLock();
private final DataExternalizer<Value> myDataExternalizer;
public MapIndexStorage(File storageFile, final KeyDescriptor<Key> keyDescriptor, final DataExternalizer<Value> valueExternalizer,
public MapIndexStorage(File storageFile, final KeyDescriptor<Key> keyDescriptor,
final DataExternalizer<Value> valueExternalizer,
final int cacheSize) throws IOException {
myStorageFile = storageFile;
myKeyDescriptor = keyDescriptor;
myValueContainerExternalizer = new ValueContainerExternalizer<Value>(valueExternalizer);
myCacheSize = cacheSize;
myDataExternalizer = valueExternalizer;
initMapAndCache();
}
private void initMapAndCache() throws IOException {
final PersistentHashMap<Key, ValueContainer<Value>> map =
new PersistentHashMap<Key, ValueContainer<Value>>(myStorageFile, myKeyDescriptor, myValueContainerExternalizer);
final PersistentMap<Key, ValueContainer<Value>> map =
new ValueContainerMap<Key, Value>(myStorageFile, myKeyDescriptor, myDataExternalizer);
myCache = new SLRUCache<Key, ChangeTrackingValueContainer<Value>>(myCacheSize, (int)(Math.ceil(myCacheSize * 0.25)) /* 25% from the main cache size*/) {
@NotNull
public ChangeTrackingValueContainer<Value> createValue(final Key key) {
@@ -74,7 +69,7 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
}
public ValueContainer<Value> compute() {
ValueContainer<Value> value = null;
ValueContainer<Value> value;
try {
value = map.get(key);
if (value == null) {
@@ -90,43 +85,13 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
}
protected void onDropFromCache(final Key key, final ChangeTrackingValueContainer<Value> valueContainer) {
if (!valueContainer.isDirty()) {
return;
}
try {
if (!valueContainer.needsCompacting()) {
final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream();
//noinspection IOResourceOpenedButNotSafelyClosed
final DataOutputStream _out = new DataOutputStream(bytes);
final TIntHashSet set = valueContainer.getInvalidated();
if (set.size() > 0) {
for (int inputId : set.toArray()) {
myValueContainerExternalizer.saveInvalidateCommand(_out, inputId);
}
}
final ValueContainer<Value> toRemove = valueContainer.getRemovedDelta();
if (toRemove.size() > 0) {
myValueContainerExternalizer.saveAsRemoved(_out, toRemove);
}
final ValueContainer<Value> toAppend = valueContainer.getAddedDelta();
if (toAppend.size() > 0) {
myValueContainerExternalizer.save(_out, toAppend);
}
map.appendData(key, new PersistentHashMap.ValueDataAppender() {
public void append(final DataOutput out) throws IOException {
out.write(bytes.getInternalBuffer(), 0, bytes.size());
}
});
}
else {
// rewrite the value container for defragmentation
if (valueContainer.isDirty()) {
try {
map.put(key, valueContainer);
}
}
catch (IOException e) {
throw new RuntimeException(e);
catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
@@ -275,73 +240,4 @@ public final class MapIndexStorage<Key, Value> implements IndexStorage<Key, Valu
}
}
private static final class ValueContainerExternalizer<T> implements DataExternalizer<ValueContainer<T>> {
private final DataExternalizer<T> myExternalizer;
private ValueContainerExternalizer(DataExternalizer<T> externalizer) {
myExternalizer = externalizer;
}
public void save(final DataOutput out, final ValueContainer<T> container) throws IOException {
saveImpl(out, container, false);
}
public void saveAsRemoved(final DataOutput out, final ValueContainer<T> container) throws IOException {
saveImpl(out, container, true);
}
public void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException {
DataInputOutputUtil.writeSINT(out, -inputId);
}
private void saveImpl(final DataOutput out, final ValueContainer<T> container, final boolean asRemovedData) throws IOException {
DataInputOutputUtil.writeSINT(out, container.size());
for (final Iterator<T> valueIterator = container.getValueIterator(); valueIterator.hasNext();) {
final T value = valueIterator.next();
myExternalizer.save(out, value);
final ValueContainer.IntIterator ids = container.getInputIdsIterator(value);
if (ids != null) {
DataInputOutputUtil.writeSINT(out, ids.size());
while (ids.hasNext()) {
final int id = ids.next();
DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id);
}
}
else {
DataInputOutputUtil.writeSINT(out, 0);
}
}
}
public ValueContainerImpl<T> read(final DataInput in) throws IOException {
DataInputStream stream = (DataInputStream)in;
final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
while (stream.available() > 0) {
final int valueCount = DataInputOutputUtil.readSINT(in);
if (valueCount < 0) {
valueContainer.removeAllValues(-valueCount);
valueContainer.setNeedsCompacting(true);
}
else {
for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) {
final T value = myExternalizer.read(in);
final int idCount = DataInputOutputUtil.readSINT(in);
for (int i = 0; i < idCount; i++) {
final int id = DataInputOutputUtil.readSINT(in);
if (id < 0) {
valueContainer.removeValue(-id, value);
valueContainer.setNeedsCompacting(true);
}
else {
valueContainer.addValue(id, value);
}
}
}
}
}
return valueContainer;
}
}
}
platform/lang-impl/src/com/intellij/util/indexing/ValueContainerMap.java
+134
View File
@@ -0,0 +1,134 @@
package com.intellij.util.indexing;
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.DataInputOutputUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import gnu.trove.TIntHashSet;
import java.io.*;
import java.util.Iterator;
/**
* @author Dmitry Avdeev
* Date: 8/10/11
*/
public class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, ValueContainer<Value>> {
private final ValueContainerExternalizer<Value> myValueContainerExternalizer;
public ValueContainerMap(final File file,
KeyDescriptor<Key> keyKeyDescriptor,
DataExternalizer<Value> valueExternalizer) throws IOException {
super(file, keyKeyDescriptor, new ValueContainerExternalizer<Value>(valueExternalizer));
myValueContainerExternalizer = (ValueContainerExternalizer<Value>)myValueExternalizer;
}
@Override
public synchronized void put(Key key, ValueContainer<Value> container) throws IOException {
ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container;
if (!valueContainer.needsCompacting()) {
final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream();
//noinspection IOResourceOpenedButNotSafelyClosed
final DataOutputStream _out = new DataOutputStream(bytes);
final TIntHashSet set = valueContainer.getInvalidated();
if (set.size() > 0) {
for (int inputId : set.toArray()) {
ValueContainerExternalizer.saveInvalidateCommand(_out, inputId);
}
}
final ValueContainer<Value> toRemove = valueContainer.getRemovedDelta();
if (toRemove.size() > 0) {
myValueContainerExternalizer.saveAsRemoved(_out, toRemove);
}
final ValueContainer<Value> toAppend = valueContainer.getAddedDelta();
if (toAppend.size() > 0) {
myValueContainerExternalizer.save(_out, toAppend);
}
appendData(key, new PersistentHashMap.ValueDataAppender() {
public void append(final DataOutput out) throws IOException {
out.write(bytes.getInternalBuffer(), 0, bytes.size());
}
});
}
else {
// rewrite the value container for defragmentation
super.put(key, valueContainer);
}
}
private static final class ValueContainerExternalizer<T> implements DataExternalizer<ValueContainer<T>> {
private final DataExternalizer<T> myExternalizer;
private ValueContainerExternalizer(DataExternalizer<T> externalizer) {
myExternalizer = externalizer;
}
public void save(final DataOutput out, final ValueContainer<T> container) throws IOException {
saveImpl(out, container, false);
}
public void saveAsRemoved(final DataOutput out, final ValueContainer<T> container) throws IOException {
saveImpl(out, container, true);
}
public static void saveInvalidateCommand(final DataOutput out, int inputId) throws IOException {
DataInputOutputUtil.writeSINT(out, -inputId);
}
private void saveImpl(final DataOutput out, final ValueContainer<T> container, final boolean asRemovedData) throws IOException {
DataInputOutputUtil.writeSINT(out, container.size());
for (final Iterator<T> valueIterator = container.getValueIterator(); valueIterator.hasNext();) {
final T value = valueIterator.next();
myExternalizer.save(out, value);
final ValueContainer.IntIterator ids = container.getInputIdsIterator(value);
if (ids != null) {
DataInputOutputUtil.writeSINT(out, ids.size());
while (ids.hasNext()) {
final int id = ids.next();
DataInputOutputUtil.writeSINT(out, asRemovedData ? -id : id);
}
}
else {
DataInputOutputUtil.writeSINT(out, 0);
}
}
}
public ValueContainerImpl<T> read(final DataInput in) throws IOException {
DataInputStream stream = (DataInputStream)in;
final ValueContainerImpl<T> valueContainer = new ValueContainerImpl<T>();
while (stream.available() > 0) {
final int valueCount = DataInputOutputUtil.readSINT(in);
if (valueCount < 0) {
valueContainer.removeAllValues(-valueCount);
valueContainer.setNeedsCompacting(true);
}
else {
for (int valueIdx = 0; valueIdx < valueCount; valueIdx++) {
final T value = myExternalizer.read(in);
final int idCount = DataInputOutputUtil.readSINT(in);
for (int i = 0; i < idCount; i++) {
final int id = DataInputOutputUtil.readSINT(in);
if (id < 0) {
valueContainer.removeValue(-id, value);
valueContainer.setNeedsCompacting(true);
}
else {
valueContainer.addValue(id, value);
}
}
}
}
}
return valueContainer;
}
}
}
platform/util/src/com/intellij/util/io/PersistentHashMap.java
+2 -2
View File
@@ -38,11 +38,11 @@ import java.util.List;
* @author Eugene Zhuravlev
* Date: Dec 18, 2007
*/
public class PersistentHashMap<Key, Value> extends PersistentEnumeratorDelegate<Key>{
public class PersistentHashMap<Key, Value> extends PersistentEnumeratorDelegate<Key> implements PersistentMap<Key, Value> {
private static final Logger LOG = Logger.getInstance("#com.intellij.util.io.PersistentHashMap");
private PersistentHashMapValueStorage myValueStorage;
private final DataExternalizer<Value> myValueExternalizer;
protected final DataExternalizer<Value> myValueExternalizer;
private static final long NULL_ADDR = 0;
private static final int INITIAL_INDEX_SIZE;
static {
platform/util/src/com/intellij/util/io/PersistentMap.java
+29
View File
@@ -0,0 +1,29 @@
package com.intellij.util.io;
import com.intellij.util.Processor;
import java.io.IOException;
/**
* @author Dmitry Avdeev
* Date: 8/10/11
*/
public interface PersistentMap<K, V> {
V get(K key) throws IOException;
void put(K key, V value) throws IOException;
boolean processKeys(Processor<K> processor) throws IOException;
boolean isClosed();
boolean isDirty();
void force();
void close() throws IOException;
void markDirty() throws IOException;
}