diff --git a/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java b/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java index 6876c0201a80..37c9244722f4 100644 --- a/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java +++ b/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java @@ -6,12 +6,10 @@ import com.intellij.testFramework.PlatformTestUtil; import com.intellij.util.ThrowableRunnable; import com.intellij.util.containers.IntObjectCache; import com.intellij.util.io.storage.Storage; +import gnu.trove.THashSet; import junit.framework.TestCase; -import java.io.DataOutput; -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; +import java.io.*; import java.util.*; import static com.intellij.util.io.StringEnumeratorTest.createRandomString; @@ -340,7 +338,7 @@ public class PersistentMapTest extends TestCase { .printf("Data file size = %d bytes\n", new File(myDataFile.getParentFile(), myDataFile.getName() + Storage.DATA_EXTENSION).length()); } - private static final boolean DO_SLOW_TEST = false; + private static final boolean DO_SLOW_TEST = true; public void test2GLimit() throws IOException { if (!DO_SLOW_TEST) return; @@ -421,4 +419,56 @@ public class PersistentMapTest extends TestCase { clearMap(file, map); } } + + public void test2GLimitWithAppend() throws IOException { + if (!DO_SLOW_TEST) return; + File file = FileUtil.createTempFile("persistent", "map"); + FileUtil.createParentDirs(file); + EnumeratorStringDescriptor stringDescriptor = new EnumeratorStringDescriptor(); + class PathCollectionExternalizer implements DataExternalizer> { + public void save(DataOutput out, Collection value) throws IOException { + for (String str : value) { + IOUtil.writeString(str, out); + } + } + + public Collection read(DataInput in) throws IOException { + final Set result = new THashSet(FileUtil.PATH_HASHING_STRATEGY); + final DataInputStream stream = (DataInputStream)in; + while (stream.available() > 0) { + final String str = IOUtil.readString(stream); + result.add(str); + } + return result; + } + } + PathCollectionExternalizer externalizer = new PathCollectionExternalizer(); + PersistentHashMap> map = new PersistentHashMap>(file, stringDescriptor, + externalizer); + for (int j = 0; j < 7; ++j) { + for (int i = 0; i < 2000; i++) { + final int finalJ = j; + map.appendData("abc" + i, new PersistentHashMap.ValueDataAppender() { + @Override + public void append(DataOutput out) throws IOException { + IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out); + } + }); + } + } + + map.close(); + + map = new PersistentHashMap>(file, stringDescriptor, externalizer); + + long len = 0; + + for (String key : map.getAllKeysWithExistingMapping()) { + for (String k : map.get(key)) { + len += k.length(); + } + } + map.close(); + assertEquals(1400000000L, len); + } } diff --git a/platform/util/src/com/intellij/util/io/PagedFileStorage.java b/platform/util/src/com/intellij/util/io/PagedFileStorage.java index be94fa7957d6..5e11d99885de 100644 --- a/platform/util/src/com/intellij/util/io/PagedFileStorage.java +++ b/platform/util/src/com/intellij/util/io/PagedFileStorage.java @@ -373,7 +373,7 @@ public class PagedFileStorage implements Forceable { } try { - assert page <= MAX_PAGES_COUNT; + assert page >= 0 && page <= MAX_PAGES_COUNT:page; if (myStorageIndex == -1) { myStorageIndex = myStorageLockContext.myStorageLock.registerPagedFileStorage(this); diff --git a/platform/util/src/com/intellij/util/io/PersistentHashMap.java b/platform/util/src/com/intellij/util/io/PersistentHashMap.java index 8a596e63e5a0..39acbc04c9a9 100644 --- a/platform/util/src/com/intellij/util/io/PersistentHashMap.java +++ b/platform/util/src/com/intellij/util/io/PersistentHashMap.java @@ -673,14 +673,14 @@ public class PersistentHashMap extends PersistentEnumeratorDelegate< myEnumerator.myStorage.putInt(keyId + myParentValueRefOffset, -(int)(value + POSITIVE_VALUE_SHIFT)); if (newKey) ++smallKeys; } else { - if (newKey && myLargeIndexWatermarkId == 0) { - myLargeIndexWatermarkId = keyId; - } - if (keyId < myLargeIndexWatermarkId && (oldValue == NULL_ADDR || canUseIntAddressForNewRecord(oldValue))) { + if ((keyId < myLargeIndexWatermarkId || myLargeIndexWatermarkId == 0) && (newKey || canUseIntAddressForNewRecord(oldValue))) { // keyId is result of enumerate, if we do reenumerate then it is no longer accessible unless somebody cached it myIntAddressForNewRecord = false; keyId = myEnumerator.reenumerate(key == null ? myEnumerator.getValue(keyId, processingKey) : key); ++transformedKeys; + if (myLargeIndexWatermarkId == 0) { + myLargeIndexWatermarkId = keyId; + } } } }