From c8e27bf3d361fcd49e2e1819ca37fd304d980b57 Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Thu, 10 Apr 2014 23:15:04 +0200 Subject: [PATCH] ability to store PHM in more compact format --- .../intellij/util/io/PersistentMapTest.java | 22 ++++++++++++++--- .../intellij/util/io/PersistentHashMap.java | 24 ++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java b/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java index 957e9abd4453..4dcd4da5aa87 100644 --- a/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java +++ b/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java @@ -440,14 +440,25 @@ public class PersistentMapTest extends TestCase { TIntIntHashMap checkMap = new TIntIntHashMap(size); Random r = new Random(1); while(size != checkMap.size()) { - checkMap.put(r.nextInt(), size == 0 ? 0 : r.nextInt()); + if (checkMap.size() == 0) { + checkMap.put(r.nextInt(), 0); + checkMap.put(r.nextInt(), 0); + checkMap.put(0, r.nextInt()); + } else { + checkMap.put(r.nextInt(), r.nextInt()); + } } long started = System.currentTimeMillis(); PersistentHashMap map = null; try { - map = new PersistentHashMap(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE); + map = new PersistentHashMap(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE) { + @Override + protected boolean wantCompactIntegralValues() { + return true; + } + }; final PersistentHashMap mapFinal = map; boolean result = checkMap.forEachEntry(new TIntIntProcedure() { @@ -467,7 +478,12 @@ public class PersistentMapTest extends TestCase { map.close(); System.out.println("Done:"+(System.currentTimeMillis() - started)); started = System.currentTimeMillis(); - map = new PersistentHashMap(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE); + map = new PersistentHashMap(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE) { + @Override + protected boolean wantCompactIntegralValues() { + return true; + } + }; final PersistentHashMap mapFinal2 = map; result = checkMap.forEachEntry(new TIntIntProcedure() { @Override diff --git a/platform/util/src/com/intellij/util/io/PersistentHashMap.java b/platform/util/src/com/intellij/util/io/PersistentHashMap.java index 7defd2974edf..00a10d958bbf 100644 --- a/platform/util/src/com/intellij/util/io/PersistentHashMap.java +++ b/platform/util/src/com/intellij/util/io/PersistentHashMap.java @@ -62,6 +62,7 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< private final int myParentValueRefOffset; @NotNull private final byte[] myRecordBuffer; @NotNull private final byte[] mySmallRecordBuffer; + private final boolean myIntMapping; private final boolean myCanReEnumerate; private int myLargeIndexWatermarkId; // starting with this id we store offset in adjacent file in long format private boolean myIntAddressForNewRecord; @@ -112,6 +113,8 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< myAppendCache = createAppendCache(keyDescriptor); final PersistentEnumeratorBase.RecordBufferHandler recordHandler = myEnumerator.getRecordHandler(); myParentValueRefOffset = recordHandler.getRecordBuffer(myEnumerator).length; + myIntMapping = valueExternalizer instanceof IntInlineKeyDescriptor && wantCompactIntegralValues(); + myRecordBuffer = new byte[myParentValueRefOffset + 8]; mySmallRecordBuffer = new byte[myParentValueRefOffset + 4]; @@ -180,6 +183,10 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< } } + protected boolean wantCompactIntegralValues() { + return false; + } + private SLRUCache createAppendCache(final KeyDescriptor keyDescriptor) { final EqualityPolicy hashingStrategy = new EqualityPolicy() { @Override @@ -306,14 +313,18 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< myEnumerator.markDirty(true); myAppendCache.remove(key); + final int id = enumerate(key); + if (myIntMapping) { + myEnumerator.myStorage.putInt(id + myParentValueRefOffset, (Integer)value); + return; + } + final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); AppendStream appenderStream = ourFlyweightAppenderStream.getValue(); appenderStream.setOut(bytes); myValueExternalizer.save(appenderStream, value); appenderStream.setOut(null); - final int id = enumerate(key); - long oldheader = readValueId(id); if (oldheader != NULL_ADDR) { myLiveAndGarbageKeysCounter++; @@ -357,6 +368,7 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< }; protected void doAppendData(Key key, @NotNull ValueDataAppender appender) throws IOException { + assert !myIntMapping; myEnumerator.markDirty(true); AppendStream appenderStream = ourFlyweightAppenderStream.getValue(); @@ -413,11 +425,15 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< if (id == PersistentEnumerator.NULL_ID) { return null; } + + if (myIntMapping) { + return (Value)(Integer)myEnumerator.myStorage.getInt(id + myParentValueRefOffset); + } + final long oldHeader = readValueId(id); if (oldHeader == PersistentEnumerator.NULL_ID) { return null; } - PersistentHashMapValueStorage.ReadResult readResult = myValueStorage.readBytes(oldHeader); if (readResult.offset != oldHeader) { myEnumerator.markDirty(true); @@ -454,6 +470,7 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< if (id == PersistentEnumerator.NULL_ID) { return false; } + if(myIntMapping) return true; return readValueId(id) != NULL_ADDR; } finally { @@ -475,6 +492,7 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< if (id == PersistentEnumerator.NULL_ID) { return; } + assert !myIntMapping; // removal isn't supported myEnumerator.markDirty(true); final long record = readValueId(id);