Cannot build artifact for big project - IU 12 Preview (IDEA-95035)

platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java

@@ -6,12 +6,10 @@ import com.intellij.testFramework.PlatformTestUtil;
 import com.intellij.util.ThrowableRunnable;
 import com.intellij.util.containers.IntObjectCache;
 import com.intellij.util.io.storage.Storage;
+import gnu.trove.THashSet;
 import junit.framework.TestCase;
 
-import java.io.DataOutput;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
+import java.io.*;
 import java.util.*;
 
 import static com.intellij.util.io.StringEnumeratorTest.createRandomString;
@@ -340,7 +338,7 @@ public class PersistentMapTest extends TestCase {
       .printf("Data file size = %d bytes\n", new File(myDataFile.getParentFile(), myDataFile.getName() + Storage.DATA_EXTENSION).length());
   }
 
-  private static final boolean DO_SLOW_TEST = false;
+  private static final boolean DO_SLOW_TEST = true;
 
   public void test2GLimit() throws IOException {
     if (!DO_SLOW_TEST) return;
@@ -421,4 +419,56 @@ public class PersistentMapTest extends TestCase {
       clearMap(file, map);
     }
   }
+
+  public void test2GLimitWithAppend() throws IOException {
+    if (!DO_SLOW_TEST) return;
+    File file = FileUtil.createTempFile("persistent", "map");
+    FileUtil.createParentDirs(file);
+    EnumeratorStringDescriptor stringDescriptor = new EnumeratorStringDescriptor();
+    class PathCollectionExternalizer implements DataExternalizer<Collection<String>> {
+      public void save(DataOutput out, Collection<String> value) throws IOException {
+        for (String str : value) {
+          IOUtil.writeString(str, out);
+        }
+      }
+
+      public Collection<String> read(DataInput in) throws IOException {
+        final Set<String> result = new THashSet<String>(FileUtil.PATH_HASHING_STRATEGY);
+        final DataInputStream stream = (DataInputStream)in;
+        while (stream.available() > 0) {
+          final String str = IOUtil.readString(stream);
+          result.add(str);
+        }
+        return result;
+      }
+    }
+    PathCollectionExternalizer externalizer = new PathCollectionExternalizer();
+    PersistentHashMap<String, Collection<String>> map = new PersistentHashMap<String, Collection<String>>(file, stringDescriptor,
+                                                                                                          externalizer);
+    for (int j = 0; j < 7; ++j) {
+      for (int i = 0; i < 2000; i++) {
+        final int finalJ = j;
+        map.appendData("abc" + i, new PersistentHashMap.ValueDataAppender() {
+          @Override
+          public void append(DataOutput out) throws IOException {
+            IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out);
+          }
+        });
+      }
+    }
+
+    map.close();
+
+    map = new PersistentHashMap<String, Collection<String>>(file, stringDescriptor, externalizer);
+
+    long len = 0;
+
+    for (String key : map.getAllKeysWithExistingMapping()) {
+      for (String k : map.get(key)) {
+        len += k.length();
+      }
+    }
+    map.close();
+    assertEquals(1400000000L, len);
+  }
 }

platform/util/src/com/intellij/util/io/PagedFileStorage.java

@@ -373,7 +373,7 @@ public class PagedFileStorage implements Forceable {
     }
 
     try {
-      assert page <= MAX_PAGES_COUNT;
+      assert page >= 0 && page <= MAX_PAGES_COUNT:page;
 
       if (myStorageIndex == -1) {
         myStorageIndex = myStorageLockContext.myStorageLock.registerPagedFileStorage(this);

platform/util/src/com/intellij/util/io/PersistentHashMap.java

@@ -673,14 +673,14 @@ public class PersistentHashMap<Key, Value> extends PersistentEnumeratorDelegate<
         myEnumerator.myStorage.putInt(keyId + myParentValueRefOffset, -(int)(value + POSITIVE_VALUE_SHIFT));
         if (newKey) ++smallKeys;
       } else {
-        if (newKey && myLargeIndexWatermarkId == 0) {
-          myLargeIndexWatermarkId = keyId;
-        }
-        if (keyId < myLargeIndexWatermarkId && (oldValue == NULL_ADDR || canUseIntAddressForNewRecord(oldValue))) {
+        if ((keyId < myLargeIndexWatermarkId || myLargeIndexWatermarkId == 0) && (newKey || canUseIntAddressForNewRecord(oldValue))) {
           // keyId is result of enumerate, if we do reenumerate then it is no longer accessible unless somebody cached it
           myIntAddressForNewRecord = false;
           keyId = myEnumerator.reenumerate(key == null ? myEnumerator.getValue(keyId, processingKey) : key);
           ++transformedKeys;
+          if (myLargeIndexWatermarkId == 0) {
+            myLargeIndexWatermarkId = keyId;
+          }
         }
       }
     }