MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

- OutOfMemory on index compaction (IDEA-169621)

Browse Source 
- Index corruption during compaction
This commit is contained in:
Maxim.Mossienko
2017-04-18 19:14:45 +02:00
parent 14f72e5e05
commit 8bbfe17680
2 changed files with 162 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapPerformanceTest.java
+145 -51
View File
@@ -33,22 +33,47 @@ import java.util.Set;
*/
@SkipSlowTestLocally
public class PersistentMapPerformanceTest extends PersistentMapTestBase {
public void test2GLimit() throws IOException {
interface MapConstructor<T, T2> {
PersistentHashMap<T, T2> createMap(File file) throws IOException;
}
interface MapIntSetter<T, T2> {
void putValue(PersistentHashMap<T, T2> map, int indexKey, T2 value) throws IOException;
}
interface MapGetter<T, T2> {
T2 readValue(PersistentHashMap<T, T2> map, T key) throws IOException;
}
interface MapIntGetter<T, T2> {
T2 readValue(PersistentHashMap<T, T2> map, int indexKey) throws IOException;
}
interface MapIntRemover<T> {
void remove(PersistentHashMap<T, ?> map, int indexKey) throws IOException;
}
interface MapIntAppender<T> {
void append(PersistentHashMap<T, ?> map, int indexKey, PersistentHashMap.ValueDataAppender appender) throws IOException;
}
private static <T> void run2GTest(MapConstructor<T, String> constructor, MapIntSetter<T, String> setter, MapGetter<T, String> getter)
throws IOException {
File file = FileUtil.createTempFile("persistent", "map");
FileUtil.createParentDirs(file);
PersistentHashMap<String, String> map = null;
PersistentHashMap<T, String> map = null;
try {
map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE);
map = constructor.createMap(file);
for (int i = 0; i < 12000; i++) {
map.put("abc" + i, StringUtil.repeat("0123456789", 10000));
setter.putValue(map, i, StringUtil.repeat("0123456789", 10000));
}
map.close();
map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE);
map = constructor.createMap(file);
long len = 0;
for (String key : map.getAllKeysWithExistingMapping()) {
len += map.get(key).length();
for (T key : map.getAllKeysWithExistingMapping()) {
len += getter.readValue(map, key).length();
}
assertEquals(1200000000L, len);
}
@@ -57,28 +82,44 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
}
}
public void testOpeningWithCompact3() throws IOException {
public void test2GLimit() throws IOException {
run2GTest(
file -> new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE),
(map, indexKey, value) -> map.put("abc" + indexKey, value),
PersistentHashMap::get
);
}
public void test2GLimit2() throws IOException {
run2GTest(
file -> new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE),
PersistentHashMap::put,
PersistentHashMap::get
);
}
private static <T> void runOpeningWithCompact3Test(MapConstructor<T, Integer> constructor, MapIntGetter<T, Integer> intGetter,
MapIntAppender<T> appender, MapIntRemover<T> remover) throws IOException {
File file = FileUtil.createTempFile("persistent", "map");
EnumeratorIntegerDescriptor integerDescriptor = new EnumeratorIntegerDescriptor();
PersistentHashMap<String, Integer> map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, integerDescriptor);
PersistentHashMap<T, Integer> map = constructor.createMap(file);
try {
final int stringsCount = 10000002;
//final int stringsCount = 102;
for(int t = 0; t < 4; ++t) {
for (int t = 0; t < 4; ++t) {
for (int i = 0; i < stringsCount; ++i) {
final int finalI = i;
final int finalT = t;
PersistentHashMap.ValueDataAppender appender = out -> out.write((finalI + finalT) & 0xFF);
map.appendData(String.valueOf(i), appender);
appender.append(map, i, out -> out.write((finalI + finalT) & 0xFF));
}
}
map.close();
map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, integerDescriptor);
map = constructor.createMap(file);
for (int i = 0; i < stringsCount; ++i) {
if (i < 2 * stringsCount / 3) {
map.remove(String.valueOf(i));
remover.remove(map, i);
}
}
map.close();
@@ -86,14 +127,18 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
assertTrue(isSmall || map.makesSenseToCompact());
long started = System.currentTimeMillis();
map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, integerDescriptor);
if (isSmall) map.compact();
else assertTrue(map.isDirty()); // autocompact on open should leave the map dirty
map = constructor.createMap(file);
if (isSmall) {
map.compact();
}
else {
assertTrue(map.isDirty()); // autocompact on open should leave the map dirty
}
assertTrue(!map.makesSenseToCompact());
System.out.println(System.currentTimeMillis() - started);
for (int i = 0; i < stringsCount; ++i) {
if (i >= 2 * stringsCount / 3) {
Integer s = map.get(String.valueOf(i));
Integer s = intGetter.readValue(map, i);
assertEquals((s & 0xFF), ((i + 3) & 0xFF));
assertEquals(((s >>> 8) & 0xFF), ((i + 2) & 0xFF));
assertEquals((s >>> 16) & 0xFF, ((i + 1) & 0xFF));
@@ -106,6 +151,24 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
}
}
public void testOpeningWithCompact3() throws IOException {
runOpeningWithCompact3Test(
(file) -> new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE),
(map, indexKey) -> map.get(String.valueOf(indexKey)),
(map, indexKey, appender) -> map.appendData(String.valueOf(indexKey), appender),
(map, indexKey) -> map.remove(String.valueOf(indexKey))
);
}
public void testOpeningWithCompact3_2() throws IOException {
runOpeningWithCompact3Test(
(file) -> new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE),
PersistentHashMap::get,
PersistentHashMap::appendData,
PersistentHashMap::remove
);
}
public void testIntToIntMapPerformance() throws IOException {
File file = FileUtil.createTempFile("persistent", "map");
FileUtil.createParentDirs(file);
@@ -113,12 +176,13 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
int size = 10000000;
TIntIntHashMap checkMap = new TIntIntHashMap(size);
Random r = new Random(1);
while(size != checkMap.size()) {
while (size != checkMap.size()) {
if (checkMap.size() == 0) {
checkMap.put(r.nextInt(), 0);
checkMap.put(r.nextInt(), 0);
checkMap.put(0, Math.abs(r.nextInt()));
} else {
}
else {
checkMap.put(r.nextInt(), Math.abs(r.nextInt()));
}
}
@@ -138,16 +202,17 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
boolean result = checkMap.forEachEntry((a, b) -> {
try {
mapFinal.put(a, b);
}catch(IOException e) {
}
catch (IOException e) {
e.printStackTrace();
assertTrue(false);
return false;
return false;
}
return true;
});
assertTrue(result);
map.close();
System.out.println("Done:"+(System.currentTimeMillis() - started));
System.out.println("Done:" + (System.currentTimeMillis() - started));
started = System.currentTimeMillis();
map = new PersistentHashMap<Integer, Integer>(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorIntegerDescriptor.INSTANCE) {
@Override
@@ -159,7 +224,8 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
result = checkMap.forEachEntry((a, b) -> {
try {
assertTrue(b == mapFinal2.get(a));
}catch(IOException e) {
}
catch (IOException e) {
e.printStackTrace();
assertTrue(false);
return false;
@@ -168,54 +234,66 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
});
assertTrue(result);
System.out.println("Done 2:"+(System.currentTimeMillis() - started));
System.out.println("Done 2:" + (System.currentTimeMillis() - started));
}
finally {
clearMap(file, map);
}
}
public void test2GLimitWithAppend() throws IOException {
File file = FileUtil.createTempFile("persistent", "map");
FileUtil.createParentDirs(file);
class PathCollectionExternalizer implements DataExternalizer<Collection<String>> {
public void save(@NotNull DataOutput out, Collection<String> value) throws IOException {
for (String str : value) {
IOUtil.writeString(str, out);
}
}
private static class PathCollectionExternalizer implements DataExternalizer<Collection<String>> {
static final PathCollectionExternalizer INSTANCE = new PathCollectionExternalizer();
public Collection<String> read(@NotNull DataInput in) throws IOException {
final Set<String> result = new THashSet<>(FileUtil.PATH_HASHING_STRATEGY);
final DataInputStream stream = (DataInputStream)in;
while (stream.available() > 0) {
final String str = IOUtil.readString(stream);
result.add(str);
}
return result;
public void save(@NotNull DataOutput out, Collection<String> value) throws IOException {
for (String str : value) {
IOUtil.writeString(str, out);
}
}
PathCollectionExternalizer externalizer = new PathCollectionExternalizer();
PersistentHashMap<String, Collection<String>> map = null;
public Collection<String> read(@NotNull DataInput in) throws IOException {
final Set<String> result = new THashSet<>(FileUtil.PATH_HASHING_STRATEGY);
final DataInputStream stream = (DataInputStream)in;
while (stream.available() > 0) {
final String str = IOUtil.readString(stream);
result.add(str);
}
return result;
}
}
private static <T> void run2GLimitWithAppendTest(MapConstructor<T, Collection<String>> constructor, MapIntAppender<T> intAppender,
MapIntRemover<T> intRemover)
throws IOException {
File file = FileUtil.createTempFile("persistent", "map");
FileUtil.createParentDirs(file);
PersistentHashMap<T, Collection<String>> map = null;
try {
map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE,
externalizer);
map = constructor.createMap(file);
final int max = 2000;
for (int j = 0; j < 7; ++j) {
for (int i = 0; i < 2000; i++) {
for (int i = 0; i < max; i++) {
final int finalJ = j;
map.appendData("abc" + i, out -> IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out));
intAppender.append(map, i, out -> IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out));
}
map.force();
}
final int lastKey = max - 1;
intRemover.remove(map, lastKey);
for (int j = 0; j < 7; ++j) {
final int finalJ = j;
intAppender.append(map, lastKey, out -> IOUtil.writeString(StringUtil.repeat("0123456789", 10000 + finalJ - 3), out));
}
map.close();
map = new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, externalizer);
map = constructor.createMap(file);
long len = 0;
for (String key : map.getAllKeysWithExistingMapping()) {
for (T key : map.getAllKeysWithExistingMapping()) {
for (String k : map.get(key)) {
len += k.length();
}
@@ -227,4 +305,20 @@ public class PersistentMapPerformanceTest extends PersistentMapTestBase {
clearMap(file, map);
}
}
public void test2GLimitWithAppend() throws IOException {
run2GLimitWithAppendTest(file -> new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE,
PathCollectionExternalizer.INSTANCE),
(map, indexKey, appender) -> map.appendData("abc" + indexKey, appender),
(map, indexKey) -> map.remove("abc" + indexKey)
);
}
public void test2GLimitWithAppend2() throws IOException {
run2GLimitWithAppendTest(file -> new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE,
PathCollectionExternalizer.INSTANCE),
(map, indexKey, appender) -> map.appendData(indexKey, appender),
(map, indexKey) -> map.remove(indexKey)
);
}
}
platform/util/src/com/intellij/util/io/PersistentBTreeEnumerator.java
+17 -1
View File
@@ -236,6 +236,10 @@ public class PersistentBTreeEnumerator<Data> extends PersistentEnumeratorBase<Da
if (!p.process(value)) return false;
}
else {
if (myInlineKeysNoMapping) {
if (!p.process(value)) return false;
return true;
}
int rec = -value;
while (rec != 0) {
int id = myStorage.getInt(rec);
@@ -335,11 +339,23 @@ public class PersistentBTreeEnumerator<Data> extends PersistentEnumeratorBase<Da
assert myInlineKeysNoMapping;
try {
lockStorage();
markDirty(true);
int intKey = ((InlineKeyDescriptor<Data>)myDataDescriptor).toInt(key);
markDirty(true);
if (value < Integer.MAX_VALUE) {
myBTree.put(intKey, (int) value);
} else {
// reuse long record if it was allocated
boolean hasMapping = myBTree.get(intKey, myResultBuf);
if (hasMapping) {
if (myResultBuf[0] < 0) {
myStorage.putLong(-myResultBuf[0], value);
return;
}
}
int pos = nextLongValueRecord();
myStorage.putLong(pos, value);
myBTree.put(intKey, -pos);