From 9da271bf6d42577eb3761e7ef9132f9132ef02b8 Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Wed, 24 Jan 2018 13:36:06 +0100 Subject: [PATCH] more understandable code after review (IDEA-CR-28595) --- .../io/PersistentHashMapValueStorage.java | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/platform/util/src/com/intellij/util/io/PersistentHashMapValueStorage.java b/platform/util/src/com/intellij/util/io/PersistentHashMapValueStorage.java index 7e0776a17fa0..5045739bb7cd 100644 --- a/platform/util/src/com/intellij/util/io/PersistentHashMapValueStorage.java +++ b/platform/util/src/com/intellij/util/io/PersistentHashMapValueStorage.java @@ -356,7 +356,7 @@ public class PersistentHashMapValueStorage { final int fileBufferLength = 256 * 1024; final int maxRecordHeader = 5 /* max length - variable int */ + 10 /* max long offset*/; final byte[] buffer = new byte[fileBufferLength + maxRecordHeader]; - byte[] recordBuffer = {}; + byte[] reusedAccumulatedChunksBuffer = {}; long lastReadOffset = mySize; long lastConsumedOffset = lastReadOffset; @@ -405,23 +405,23 @@ public class PersistentHashMapValueStorage { prevChunkAddress = readPrevChunkAddress(info.valueAddress); dataOffset = available - myBufferStreamWrapper.available(); - byte[] b; + byte[] accumulatedChunksBuffer; if (info.value != null) { int defragmentedChunkSize = info.value.length + chunkSize; if (prevChunkAddress == 0) { - if (defragmentedChunkSize >= recordBuffer.length) recordBuffer = new byte[defragmentedChunkSize]; - b = recordBuffer; + if (defragmentedChunkSize >= reusedAccumulatedChunksBuffer.length) reusedAccumulatedChunksBuffer = new byte[defragmentedChunkSize]; + accumulatedChunksBuffer = reusedAccumulatedChunksBuffer; } else { - b = new byte[defragmentedChunkSize]; + accumulatedChunksBuffer = new byte[defragmentedChunkSize]; retained += defragmentedChunkSize; } - System.arraycopy(info.value, 0, b, chunkSize, info.value.length); + System.arraycopy(info.value, 0, accumulatedChunksBuffer, chunkSize, info.value.length); } else { if (prevChunkAddress == 0) { - if (chunkSize >= recordBuffer.length) recordBuffer = new byte[chunkSize]; - b = recordBuffer; + if (chunkSize >= reusedAccumulatedChunksBuffer.length) reusedAccumulatedChunksBuffer = new byte[chunkSize]; + accumulatedChunksBuffer = reusedAccumulatedChunksBuffer; } else { - b = new byte[chunkSize]; + accumulatedChunksBuffer = new byte[chunkSize]; retained += chunkSize; } } @@ -430,11 +430,11 @@ public class PersistentHashMapValueStorage { Math.max((int)(info.valueAddress + dataOffset + chunkSize - lastReadOffset), 0)); if (chunkSizeOutOfBuffer > 0) { if (allRecordsStart != 0) { - myCompactionModeReader.get(allRecordsStart, b, chunkSize - chunkSizeOutOfBuffer, chunkSizeOutOfBuffer); + myCompactionModeReader.get(allRecordsStart, accumulatedChunksBuffer, chunkSize - chunkSizeOutOfBuffer, chunkSizeOutOfBuffer); } else { int offsetInStuffFromPreviousRecord = Math.max((int)(info.valueAddress + dataOffset - lastReadOffset), 0); // stuffFromPreviousRecord starts from lastReadOffset - System.arraycopy(stuffFromPreviousRecord, offsetInStuffFromPreviousRecord, b, chunkSize - chunkSizeOutOfBuffer, chunkSizeOutOfBuffer); + System.arraycopy(stuffFromPreviousRecord, offsetInStuffFromPreviousRecord, accumulatedChunksBuffer, chunkSize - chunkSizeOutOfBuffer, chunkSizeOutOfBuffer); } } @@ -442,9 +442,9 @@ public class PersistentHashMapValueStorage { allRecordsStart = allRecordsLength = 0; lastConsumedOffset = info.valueAddress; - checkPreconditions(b, chunkSize, 0); + checkPreconditions(accumulatedChunksBuffer, chunkSize, 0); - System.arraycopy(buffer, recordStartInBuffer + dataOffset, b, 0, chunkSize - chunkSizeOutOfBuffer); + System.arraycopy(buffer, recordStartInBuffer + dataOffset, accumulatedChunksBuffer, 0, chunkSize - chunkSizeOutOfBuffer); ++fragments; records.remove(info); @@ -455,24 +455,17 @@ public class PersistentHashMapValueStorage { } if (prevChunkAddress == 0) { - info.newValueAddress = storage.appendBytes(b, 0, chunkSize, info.newValueAddress); + info.newValueAddress = storage.appendBytes(accumulatedChunksBuffer, 0, chunkSize, info.newValueAddress); ++newFragments; } else { - if (retained > SOFT_MAX_RETAINED_LIMIT && b.length > BLOCK_SIZE_TO_WRITE_WHEN_SOFT_MAX_RETAINED_LIMIT_IS_HIT || + if (retained > SOFT_MAX_RETAINED_LIMIT && accumulatedChunksBuffer.length > BLOCK_SIZE_TO_WRITE_WHEN_SOFT_MAX_RETAINED_LIMIT_IS_HIT || retained > MAX_RETAINED_LIMIT_WHEN_COMPACTING) { - // to avoid OOME we need to save 'b' accumulated from chunks - // to preserve write order prev data is loaded in usual backward chunk reads - ReadResult result = readBytes(prevChunkAddress); - info.newValueAddress = storage.appendBytes(result.buffer, 0, result.buffer.length, info.newValueAddress); - ++newFragments; - info.newValueAddress = storage.appendBytes(b, 0, chunkSize, info.newValueAddress); - ++newFragments; - info.value = null; - info.valueAddress = 0; - retained -= b.length; + // to avoid OOME we need to save bytes in accumulatedChunksBuffer + newFragments += saveAccumulatedDataOnDiskPreservingWriteOrder(storage, info, prevChunkAddress, accumulatedChunksBuffer, chunkSize); + retained -= accumulatedChunksBuffer.length; continue; } else { - info.value = b; + info.value = accumulatedChunksBuffer; } info.valueAddress = prevChunkAddress; records.add(info); @@ -497,6 +490,22 @@ public class PersistentHashMapValueStorage { return fragments | ((long)newFragments << 32); } + private int saveAccumulatedDataOnDiskPreservingWriteOrder(PersistentHashMapValueStorage storage, + PersistentHashMap.CompactionRecordInfo info, + long prevChunkAddress, + byte[] accumulatedChunksData, + int accumulatedChunkDataLength) throws IOException { + ReadResult result = readBytes(prevChunkAddress); + // to avoid possible OOME result.bytes and accumulatedChunksData are not combined in one chunk, instead they are + // placed one after another, such near placement should be fine because of disk caching + info.newValueAddress = storage.appendBytes(result.buffer, 0, result.buffer.length, info.newValueAddress); + info.newValueAddress = storage.appendBytes(accumulatedChunksData, 0, accumulatedChunkDataLength, info.newValueAddress); + + info.value = null; + info.valueAddress = 0; + return 2; // number of chunks produced = number of appendBytes called + } + public static class ReadResult { public final byte[] buffer; public final int chunksCount;