From 8bbfe17680a9f1eed528ca4c16fb2277e37586e0 Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Tue, 18 Apr 2017 19:11:24 +0200 Subject: [PATCH] - OutOfMemory on index compaction (IDEA-169621) - Index corruption during compaction --- .../util/io/PersistentMapPerformanceTest.java | 196 +++++++++++++----- .../util/io/PersistentBTreeEnumerator.java | 18 +- 2 files changed, 162 insertions(+), 52 deletions(-) diff --git a/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapPerformanceTest.java b/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapPerformanceTest.java index 72c6f45c4d37..d1c837281345 100644 --- a/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapPerformanceTest.java +++ b/platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapPerformanceTest.java @@ -33,22 +33,47 @@ import java.util.Set; */ @SkipSlowTestLocally public class PersistentMapPerformanceTest extends PersistentMapTestBase { - public void test2GLimit() throws IOException { + interface MapConstructor { + PersistentHashMap createMap(File file) throws IOException; + } + + interface MapIntSetter { + void putValue(PersistentHashMap map, int indexKey, T2 value) throws IOException; + } + + interface MapGetter { + T2 readValue(PersistentHashMap map, T key) throws IOException; + } + + interface MapIntGetter { + T2 readValue(PersistentHashMap map, int indexKey) throws IOException; + } + + interface MapIntRemover { + void remove(PersistentHashMap map, int indexKey) throws IOException; + } + + interface MapIntAppender { + void append(PersistentHashMap map, int indexKey, PersistentHashMap.ValueDataAppender appender) throws IOException; + } + + private static void run2GTest(MapConstructor constructor, MapIntSetter setter, MapGetter getter) + throws IOException { File file = FileUtil.createTempFile("persistent", "map"); FileUtil.createParentDirs(file); - PersistentHashMap map = null; + PersistentHashMap map = null; try { - map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE); + map = constructor.createMap(file); for (int i = 0; i < 12000; i++) { - map.put("abc" + i, StringUtil.repeat("0123456789", 10000)); + setter.putValue(map, i, StringUtil.repeat("0123456789", 10000)); } map.close(); - map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE); + map = constructor.createMap(file); long len = 0; - for (String key : map.getAllKeysWithExistingMapping()) { - len += map.get(key).length(); + for (T key : map.getAllKeysWithExistingMapping()) { + len += getter.readValue(map, key).length(); } assertEquals(1200000000L, len); } @@ -57,28 +82,44 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { } } - public void testOpeningWithCompact3() throws IOException { + public void test2GLimit() throws IOException { + run2GTest( + file -> new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE), + (map, indexKey, value) -> map.put("abc" + indexKey, value), + PersistentHashMap::get + ); + } + + public void test2GLimit2() throws IOException { + run2GTest( + file -> new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE), + PersistentHashMap::put, + PersistentHashMap::get + ); + } + + private static void runOpeningWithCompact3Test(MapConstructor constructor, MapIntGetter intGetter, + MapIntAppender appender, MapIntRemover remover) throws IOException { File file = FileUtil.createTempFile("persistent", "map"); - EnumeratorIntegerDescriptor integerDescriptor = new EnumeratorIntegerDescriptor(); - PersistentHashMap map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, integerDescriptor); + PersistentHashMap map = constructor.createMap(file); try { final int stringsCount = 10000002; //final int stringsCount = 102; - for(int t = 0; t < 4; ++t) { + for (int t = 0; t < 4; ++t) { for (int i = 0; i < stringsCount; ++i) { final int finalI = i; final int finalT = t; - PersistentHashMap.ValueDataAppender appender = out -> out.write((finalI + finalT) & 0xFF); - map.appendData(String.valueOf(i), appender); + + appender.append(map, i, out -> out.write((finalI + finalT) & 0xFF)); } } map.close(); - map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, integerDescriptor); + map = constructor.createMap(file); for (int i = 0; i < stringsCount; ++i) { if (i < 2 * stringsCount / 3) { - map.remove(String.valueOf(i)); + remover.remove(map, i); } } map.close(); @@ -86,14 +127,18 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { assertTrue(isSmall || map.makesSenseToCompact()); long started = System.currentTimeMillis(); - map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, integerDescriptor); - if (isSmall) map.compact(); - else assertTrue(map.isDirty()); // autocompact on open should leave the map dirty + map = constructor.createMap(file); + if (isSmall) { + map.compact(); + } + else { + assertTrue(map.isDirty()); // autocompact on open should leave the map dirty + } assertTrue(!map.makesSenseToCompact()); System.out.println(System.currentTimeMillis() - started); for (int i = 0; i < stringsCount; ++i) { if (i >= 2 * stringsCount / 3) { - Integer s = map.get(String.valueOf(i)); + Integer s = intGetter.readValue(map, i); assertEquals((s & 0xFF), ((i + 3) & 0xFF)); assertEquals(((s >>> 8) & 0xFF), ((i + 2) & 0xFF)); assertEquals((s >>> 16) & 0xFF, ((i + 1) & 0xFF)); @@ -106,6 +151,24 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { } } + public void testOpeningWithCompact3() throws IOException { + runOpeningWithCompact3Test( + (file) -> new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE), + (map, indexKey) -> map.get(String.valueOf(indexKey)), + (map, indexKey, appender) -> map.appendData(String.valueOf(indexKey), appender), + (map, indexKey) -> map.remove(String.valueOf(indexKey)) + ); + } + + public void testOpeningWithCompact3_2() throws IOException { + runOpeningWithCompact3Test( + (file) -> new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE), + PersistentHashMap::get, + PersistentHashMap::appendData, + PersistentHashMap::remove + ); + } + public void testIntToIntMapPerformance() throws IOException { File file = FileUtil.createTempFile("persistent", "map"); FileUtil.createParentDirs(file); @@ -113,12 +176,13 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { int size = 10000000; TIntIntHashMap checkMap = new TIntIntHashMap(size); Random r = new Random(1); - while(size != checkMap.size()) { + while (size != checkMap.size()) { if (checkMap.size() == 0) { checkMap.put(r.nextInt(), 0); checkMap.put(r.nextInt(), 0); checkMap.put(0, Math.abs(r.nextInt())); - } else { + } + else { checkMap.put(r.nextInt(), Math.abs(r.nextInt())); } } @@ -138,16 +202,17 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { boolean result = checkMap.forEachEntry((a, b) -> { try { mapFinal.put(a, b); - }catch(IOException e) { + } + catch (IOException e) { e.printStackTrace(); assertTrue(false); - return false; + return false; } return true; }); assertTrue(result); map.close(); - System.out.println("Done:"+(System.currentTimeMillis() - started)); + System.out.println("Done:" + (System.currentTimeMillis() - started)); started = System.currentTimeMillis(); map = new PersistentHashMap(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE) { @Override @@ -159,7 +224,8 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { result = checkMap.forEachEntry((a, b) -> { try { assertTrue(b == mapFinal2.get(a)); - }catch(IOException e) { + } + catch (IOException e) { e.printStackTrace(); assertTrue(false); return false; @@ -168,54 +234,66 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { }); assertTrue(result); - System.out.println("Done 2:"+(System.currentTimeMillis() - started)); + System.out.println("Done 2:" + (System.currentTimeMillis() - started)); } finally { clearMap(file, map); } } - public void test2GLimitWithAppend() throws IOException { - File file = FileUtil.createTempFile("persistent", "map"); - FileUtil.createParentDirs(file); - class PathCollectionExternalizer implements DataExternalizer> { - public void save(@NotNull DataOutput out, Collection value) throws IOException { - for (String str : value) { - IOUtil.writeString(str, out); - } - } + private static class PathCollectionExternalizer implements DataExternalizer> { + static final PathCollectionExternalizer INSTANCE = new PathCollectionExternalizer(); - public Collection read(@NotNull DataInput in) throws IOException { - final Set result = new THashSet<>(FileUtil.PATH_HASHING_STRATEGY); - final DataInputStream stream = (DataInputStream)in; - while (stream.available() > 0) { - final String str = IOUtil.readString(stream); - result.add(str); - } - return result; + public void save(@NotNull DataOutput out, Collection value) throws IOException { + for (String str : value) { + IOUtil.writeString(str, out); } } - PathCollectionExternalizer externalizer = new PathCollectionExternalizer(); - PersistentHashMap> map = null; + + public Collection read(@NotNull DataInput in) throws IOException { + final Set result = new THashSet<>(FileUtil.PATH_HASHING_STRATEGY); + final DataInputStream stream = (DataInputStream)in; + while (stream.available() > 0) { + final String str = IOUtil.readString(stream); + result.add(str); + } + return result; + } + } + + private static void run2GLimitWithAppendTest(MapConstructor> constructor, MapIntAppender intAppender, + MapIntRemover intRemover) + throws IOException { + File file = FileUtil.createTempFile("persistent", "map"); + FileUtil.createParentDirs(file); + + PersistentHashMap> map = null; try { - map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, - externalizer); + map = constructor.createMap(file); + final int max = 2000; for (int j = 0; j < 7; ++j) { - for (int i = 0; i < 2000; i++) { + for (int i = 0; i < max; i++) { final int finalJ = j; - map.appendData("abc" + i, out -> IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out)); + intAppender.append(map, i, out -> IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out)); } map.force(); } + final int lastKey = max - 1; + intRemover.remove(map, lastKey); + for (int j = 0; j < 7; ++j) { + final int finalJ = j; + intAppender.append(map, lastKey, out -> IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out)); + } + map.close(); - map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, externalizer); + map = constructor.createMap(file); long len = 0; - for (String key : map.getAllKeysWithExistingMapping()) { + for (T key : map.getAllKeysWithExistingMapping()) { for (String k : map.get(key)) { len += k.length(); } @@ -227,4 +305,20 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase { clearMap(file, map); } } + + public void test2GLimitWithAppend() throws IOException { + run2GLimitWithAppendTest(file -> new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, + PathCollectionExternalizer.INSTANCE), + (map, indexKey, appender) -> map.appendData("abc" + indexKey, appender), + (map, indexKey) -> map.remove("abc" + indexKey) + ); + } + + public void test2GLimitWithAppend2() throws IOException { + run2GLimitWithAppendTest(file -> new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE, + PathCollectionExternalizer.INSTANCE), + (map, indexKey, appender) -> map.appendData(indexKey, appender), + (map, indexKey) -> map.remove(indexKey) + ); + } } diff --git a/platform/util/src/com/intellij/util/io/PersistentBTreeEnumerator.java b/platform/util/src/com/intellij/util/io/PersistentBTreeEnumerator.java index 14f0ea9e21a7..bff3ead36385 100644 --- a/platform/util/src/com/intellij/util/io/PersistentBTreeEnumerator.java +++ b/platform/util/src/com/intellij/util/io/PersistentBTreeEnumerator.java @@ -236,6 +236,10 @@ public class PersistentBTreeEnumerator extends PersistentEnumeratorBase extends PersistentEnumeratorBase)myDataDescriptor).toInt(key); + + markDirty(true); + if (value < Integer.MAX_VALUE) { myBTree.put(intKey, (int) value); } else { + // reuse long record if it was allocated + boolean hasMapping = myBTree.get(intKey, myResultBuf); + if (hasMapping) { + if (myResultBuf[0] < 0) { + myStorage.putLong(-myResultBuf[0], value); + return; + } + } + int pos = nextLongValueRecord(); myStorage.putLong(pos, value); myBTree.put(intKey, -pos);