MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Fix of the bug in PersistentHashMap's backward compact

Browse Source 
When compacting fragmented PersistentHashMap in some cases records got
content not in the write order. In particular, append calls can produce
fragments A,B,C and after compact record read loads C,B,A
This commit is contained in:
Maxim.Mossienko
2018-01-18 11:02:23 +01:00
parent 12dc53ff6c
commit beaf591dfb
2 changed files with 74 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/platform-tests/testSrc/com/intellij/util/io/PersistentMapTest.java
+62 -6
View File
@@ -25,16 +25,72 @@ import com.intellij.util.io.storage.AbstractStorage;
import junit.framework.AssertionFailedError;
import org.jetbrains.annotations.NotNull;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.io.*;
import java.util.*;
/**
* @author Eugene Zhuravlev
*/
public class PersistentMapTest extends PersistentMapTestBase {
public void testRetainWriteOrderWhenCompactingBackward() throws IOException {
clearMap(myFile, myMap);
myMap = null;
String longString = StringUtil.repeat("1234567890", 120);
assertTrue(longString.length() > PersistentHashMapValueStorage.BLOCK_SIZE_TO_WRITE_WHEN_SOFT_MAX_RETAINED_LIMIT_IS_HIT);
String removalMarker = "\uFFFF";
PersistentMapPerformanceTest.MapConstructor<Integer, Collection<String>> mapConstructor =
(file) -> new PersistentHashMap<>(
file,
EnumeratorIntegerDescriptor.INSTANCE,
new DataExternalizer<Collection<String>>() {
@Override
public void save(@NotNull DataOutput out, Collection<String> value) throws IOException {
for(String str:value) {
IOUtil.writeUTF(out, str);
}
}
@Override
public Collection<String> read(@NotNull DataInput in) throws IOException {
List<String> result = new ArrayList<>();
while(((InputStream)in).available() > 0) {
String string = IOUtil.readUTF(in);
if (string.equals(removalMarker)) {
result.remove(result.size() - 1);
} else {
result.add(string);
}
}
return result;
}
}
);
PersistentHashMap<Integer, Collection<String>> map = mapConstructor.createMap(myFile);
try {
int keys = 10_000;
for(int iteration = 0; iteration < 5; ++iteration) {
String toAppend = iteration % 2 == 0 ? longString : removalMarker;
for (int i = 0; i < keys; ++i) {
map.appendData(i, out -> IOUtil.writeUTF(out, toAppend));
}
}
map.close();
assertTrue(map.getValueStorage().getSize() > 2 * PersistentHashMapValueStorage.SOFT_MAX_RETAINED_LIMIT);
map = mapConstructor.createMap(myFile);
map.compact();
for (int i = 0; i < keys; ++i) {
Collection<String> strings = map.get(i);
assertTrue(strings != null && strings.size() == 1);
assertEquals(longString, strings.iterator().next());
}
} finally {
clearMap(myFile, map);
}
}
public void testMap() throws IOException {
myMap.put("AAA", "AAA_VALUE");
@@ -308,7 +364,7 @@ public class PersistentMapTest extends PersistentMapTestBase {
public void testOpeningWithCompact2() throws IOException {
File file = FileUtil.createTempFile("persistent", "map");
PersistentHashMap<Integer, String> map = new PersistentHashMap<>(file, new IntInlineKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE);
PersistentHashMap<Integer, String> map = new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE);
try {
final int stringsCount = 5/*1000000*/;
Map<Integer, String> testMapping = new LinkedHashMap<>(stringsCount);
@@ -319,7 +375,7 @@ public class PersistentMapTest extends PersistentMapTestBase {
map.put(i, value);
}
map.close();
map = new PersistentHashMap<>(file, new IntInlineKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE);
map = new PersistentHashMap<>(file, EnumeratorIntegerDescriptor.INSTANCE, EnumeratorStringDescriptor.INSTANCE);
{ // before compact
final Collection<Integer> allKeys = new HashSet<>(map.getAllKeysWithExistingMapping());
platform/util/src/com/intellij/util/io/PersistentHashMapValueStorage.java
+12 -4
View File
@@ -47,6 +47,9 @@ public class PersistentHashMapValueStorage {
private static final int CACHE_PROTECTED_QUEUE_SIZE = 10;
private static final int CACHE_PROBATIONAL_QUEUE_SIZE = 20;
private static final long MAX_RETAINED_LIMIT_WHEN_COMPACTING = 100 * 1024 * 1024;
static final long SOFT_MAX_RETAINED_LIMIT = 10 * 1024 * 1024;
static final int BLOCK_SIZE_TO_WRITE_WHEN_SOFT_MAX_RETAINED_LIMIT_IS_HIT = 1024;
public static class CreationTimeOptions {
public static final ThreadLocal<ExceptionalIOCancellationCallback> EXCEPTIONAL_IO_CANCELLATION = new ThreadLocal<ExceptionalIOCancellationCallback>();
@@ -361,12 +364,10 @@ public class PersistentHashMapValueStorage {
int fragments = 0;
int newFragments = 0;
int allRecordsLength = 0;
byte[] stuffFromPreviousRecord = null;
int bytesRead = (int)(mySize - (mySize / fileBufferLength) * fileBufferLength);
long retained = 0;
final long softMaxRetainedLimit = 10 * 1024* 1024;
final int blockSizeToWriteWhenSoftMaxRetainedLimitIsHit = 1024;
final long maxRetainedLimit = 100 * 1024* 1024;
while(lastReadOffset != 0) {
final long readStartOffset = lastReadOffset - bytesRead;
@@ -457,12 +458,19 @@ public class PersistentHashMapValueStorage {
info.newValueAddress = storage.appendBytes(b, 0, chunkSize, info.newValueAddress);
++newFragments;
} else {
if (retained > softMaxRetainedLimit && b.length > blockSizeToWriteWhenSoftMaxRetainedLimitIsHit ||
if (retained > SOFT_MAX_RETAINED_LIMIT && b.length > BLOCK_SIZE_TO_WRITE_WHEN_SOFT_MAX_RETAINED_LIMIT_IS_HIT ||
retained > MAX_RETAINED_LIMIT_WHEN_COMPACTING) {
// to avoid OOME we need to save 'b' accumulated from chunks
// to preserve write order prev data is loaded in usual backward chunk reads
ReadResult result = readBytes(prevChunkAddress);
info.newValueAddress = storage.appendBytes(result.buffer, 0, result.buffer.length, info.newValueAddress);
++newFragments;
info.newValueAddress = storage.appendBytes(b, 0, chunkSize, info.newValueAddress);
++newFragments;
info.value = null;
info.valueAddress = 0;
retained -= b.length;
continue;
} else {
info.value = b;
}