mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-09-27 10:03:11 +07:00
indexes: refactor key storage into 2 implementations
GitOrigin-RevId: 9920bfedef81d3a44864beaa4dfc0d665aceff81
This commit is contained in:
committed by
intellij-monorepo-bot
parent
2b6c44bb90
commit
076de9ccba
@@ -32,6 +32,8 @@ import com.intellij.util.containers.ContainerUtil;
|
||||
import com.intellij.util.indexing.impl.MapIndexStorage;
|
||||
import com.intellij.util.io.DataOutputStream;
|
||||
import com.intellij.util.io.*;
|
||||
import com.intellij.util.io.keyStorage.AppendableObjectStorage;
|
||||
import com.intellij.util.io.keyStorage.AppendableStorageBackedByResizableMappedFile;
|
||||
import gnu.trove.TIntHashSet;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
@@ -48,7 +50,7 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
|
||||
private static final Logger LOG = Logger.getInstance(MapIndexStorage.class);
|
||||
private static final boolean ENABLE_CACHED_HASH_IDS = SystemProperties.getBooleanProperty("idea.index.no.cashed.hashids", true);
|
||||
private final boolean myBuildKeyHashToVirtualFileMapping;
|
||||
private AppendableStorageBackedByResizableMappedFile myKeyHashToVirtualFileMapping;
|
||||
private AppendableObjectStorage<int[]> myKeyHashToVirtualFileMapping;
|
||||
private volatile int myLastScannedId;
|
||||
|
||||
private static final ConcurrentIntObjectMap<Boolean> ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap();
|
||||
@@ -82,7 +84,7 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
|
||||
FileSystem projectFileFS = getProjectFile().getFileSystem();
|
||||
assert !projectFileFS.isReadOnly() : "File system " + projectFileFS + " is read only";
|
||||
myKeyHashToVirtualFileMapping =
|
||||
new AppendableStorageBackedByResizableMappedFile(getProjectFile(), 4096, null, PagedFileStorage.MB, true);
|
||||
new AppendableStorageBackedByResizableMappedFile<>(getProjectFile(), 4096, null, PagedFileStorage.MB, true, IntPairInArrayKeyDescriptor.INSTANCE);
|
||||
}
|
||||
else {
|
||||
myKeyHashToVirtualFileMapping = null;
|
||||
@@ -100,11 +102,11 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
|
||||
}
|
||||
|
||||
private <T extends Throwable> void withLock(ThrowableRunnable<T> r) throws T {
|
||||
myKeyHashToVirtualFileMapping.getPagedFileStorage().lock();
|
||||
myKeyHashToVirtualFileMapping.lock();
|
||||
try {
|
||||
r.run();
|
||||
} finally {
|
||||
myKeyHashToVirtualFileMapping.getPagedFileStorage().unlock();
|
||||
myKeyHashToVirtualFileMapping.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,21 +128,30 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
|
||||
public void close() throws StorageException {
|
||||
super.close();
|
||||
try {
|
||||
if (myKeyHashToVirtualFileMapping != null) {
|
||||
withLock(() -> myKeyHashToVirtualFileMapping.close());
|
||||
}
|
||||
closeKeyHashToFileMapping();
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
unwrapCauseAndRethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void closeKeyHashToFileMapping() throws StorageException {
|
||||
if (myKeyHashToVirtualFileMapping != null) {
|
||||
try {
|
||||
withLock(() -> {
|
||||
myKeyHashToVirtualFileMapping.close();
|
||||
});
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new StorageException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() throws StorageException{
|
||||
try {
|
||||
if (myKeyHashToVirtualFileMapping != null) {
|
||||
withLock(() -> myKeyHashToVirtualFileMapping.close());
|
||||
}
|
||||
closeKeyHashToFileMapping();
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
LOG.info(e);
|
||||
@@ -198,7 +209,7 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
|
||||
if (!idFilter.containsFileId(key[1])) return true;
|
||||
finalHashMaskSet.add(key[0]);
|
||||
return true;
|
||||
}, IntPairInArrayKeyDescriptor.INSTANCE);
|
||||
});
|
||||
});
|
||||
|
||||
if (useCachedHashIds) {
|
||||
@@ -296,7 +307,7 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
|
||||
public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
|
||||
try {
|
||||
if (myKeyHashToVirtualFileMapping != null) {
|
||||
withLock(() -> myKeyHashToVirtualFileMapping.append(new int[] { myKeyDescriptor.getHashCode(key), inputId }, IntPairInArrayKeyDescriptor.INSTANCE));
|
||||
withLock(() -> myKeyHashToVirtualFileMapping.append(new int[] { myKeyDescriptor.getHashCode(key), inputId }));
|
||||
int lastScannedId = myLastScannedId;
|
||||
if (lastScannedId != 0) { // we have write lock
|
||||
ourInvalidatedSessionIds.cacheOrGet(lastScannedId, Boolean.TRUE);
|
||||
|
||||
@@ -108,6 +108,10 @@ public class PagedFileStorage implements Forceable {
|
||||
myNativeBytesOrder = nativeBytesOrder;
|
||||
}
|
||||
|
||||
public int getPageSize() {
|
||||
return myPageSize;
|
||||
}
|
||||
|
||||
public void lock() {
|
||||
myStorageLockContext.lock();
|
||||
}
|
||||
@@ -157,11 +161,11 @@ public class PagedFileStorage implements Forceable {
|
||||
}
|
||||
}
|
||||
|
||||
int getOffsetInPage(long addr) {
|
||||
public int getOffsetInPage(long addr) {
|
||||
return (int)(addr % myPageSize);
|
||||
}
|
||||
|
||||
ByteBufferWrapper getByteBuffer(long address, boolean modify) {
|
||||
public ByteBufferWrapper getByteBuffer(long address, boolean modify) {
|
||||
long page = address / myPageSize;
|
||||
assert page >= 0 && page <= MAX_PAGES_COUNT:address + " in " + myFile;
|
||||
return getBufferWrapper(page, modify);
|
||||
|
||||
@@ -86,8 +86,8 @@ public class PersistentBTreeEnumerator<Data> extends PersistentEnumeratorBase<Da
|
||||
false
|
||||
);
|
||||
|
||||
myInlineKeysNoMapping = myDataDescriptor instanceof InlineKeyDescriptor && !wantKeyMapping();
|
||||
myExternalKeysNoMapping = !(myDataDescriptor instanceof InlineKeyDescriptor) && !wantKeyMapping();
|
||||
myInlineKeysNoMapping = dataDescriptor instanceof InlineKeyDescriptor;
|
||||
myExternalKeysNoMapping = !(dataDescriptor instanceof InlineKeyDescriptor);
|
||||
|
||||
if (myBTree == null) {
|
||||
try {
|
||||
@@ -124,10 +124,6 @@ public class PersistentBTreeEnumerator<Data> extends PersistentEnumeratorBase<Da
|
||||
return file.resolveSibling(file.getFileName() + "_i");
|
||||
}
|
||||
|
||||
private static boolean wantKeyMapping() {
|
||||
return false;
|
||||
}
|
||||
|
||||
private void initBtree(boolean initial) throws IOException {
|
||||
myBTree = new IntToIntBtree(BTREE_PAGE_SIZE, indexFile(myFile), myStorage.getPagedFileStorage().getStorageLockContext(), initial);
|
||||
}
|
||||
|
||||
@@ -23,6 +23,10 @@ import com.intellij.util.IncorrectOperationException;
|
||||
import com.intellij.util.Processor;
|
||||
import com.intellij.util.containers.SLRUMap;
|
||||
import com.intellij.util.containers.ShareableKey;
|
||||
import com.intellij.util.io.keyStorage.AppendableObjectStorage;
|
||||
import com.intellij.util.io.keyStorage.AppendableStorageBackedByResizableMappedFile;
|
||||
import com.intellij.util.io.keyStorage.InlinedKeyStorage;
|
||||
import com.intellij.util.io.keyStorage.NoDataException;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.jetbrains.annotations.TestOnly;
|
||||
@@ -30,7 +34,6 @@ import org.jetbrains.annotations.TestOnly;
|
||||
import java.io.Closeable;
|
||||
import java.io.Flushable;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
@@ -50,8 +53,8 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
private static final CacheKey ourFlyweight = new FlyweightKey();
|
||||
|
||||
protected final ResizeableMappedFile myStorage;
|
||||
private final boolean myAssumeDifferentSerializedBytesMeansObjectsInequality;
|
||||
private final AppendableStorageBackedByResizableMappedFile myKeyStorage;
|
||||
@NotNull
|
||||
private final AppendableObjectStorage<Data> myKeyStorage;
|
||||
final KeyDescriptor<Data> myDataDescriptor;
|
||||
protected final Path myFile;
|
||||
private final Version myVersion;
|
||||
@@ -90,10 +93,10 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
}
|
||||
|
||||
private static class CacheKey implements ShareableKey {
|
||||
public PersistentEnumeratorBase owner;
|
||||
public PersistentEnumeratorBase<?> owner;
|
||||
public Object key;
|
||||
|
||||
private CacheKey(Object key, PersistentEnumeratorBase owner) {
|
||||
private CacheKey(Object key, PersistentEnumeratorBase<?> owner) {
|
||||
this.key = key;
|
||||
this.owner = owner;
|
||||
}
|
||||
@@ -231,11 +234,16 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
}
|
||||
|
||||
if (dataDescriptor instanceof InlineKeyDescriptor) {
|
||||
myKeyStorage = null;
|
||||
myKeyStorage = new InlinedKeyStorage<>((InlineKeyDescriptor<Data>)dataDescriptor);
|
||||
}
|
||||
else {
|
||||
try {
|
||||
myKeyStorage = new AppendableStorageBackedByResizableMappedFile(keyStreamFile(), initialSize, myStorage.getPagedFileStorage().getStorageLockContext(), PagedFileStorage.MB, false);
|
||||
myKeyStorage = new AppendableStorageBackedByResizableMappedFile<>(keyStreamFile(),
|
||||
initialSize,
|
||||
myStorage.getPagedFileStorage().getStorageLockContext(),
|
||||
PagedFileStorage.MB,
|
||||
false,
|
||||
dataDescriptor);
|
||||
}
|
||||
catch (Throwable e) {
|
||||
LOG.info(e);
|
||||
@@ -243,7 +251,6 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
throw new CorruptedException(file);
|
||||
}
|
||||
}
|
||||
myAssumeDifferentSerializedBytesMeansObjectsInequality = myDataDescriptor instanceof DifferentSerializableBytesImplyNonEqualityPolicy;
|
||||
}
|
||||
|
||||
void lockStorage() {
|
||||
@@ -396,15 +403,16 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
protected abstract int enumerateImpl(final Data value, final boolean onlyCheckForExisting, boolean saveNewValue) throws IOException;
|
||||
|
||||
protected boolean isKeyAtIndex(final Data value, final int idx) throws IOException {
|
||||
if (myKeyStorage == null) return false;
|
||||
if (myKeyStorage instanceof InlinedKeyStorage) return false;
|
||||
|
||||
// check if previous serialized state is the same as for value
|
||||
// this is much faster than myDataDescriptor.isEqualTo(valueOf(idx), value) for identical objects
|
||||
// TODO: key storage lock
|
||||
final int addr = indexToAddr(idx);
|
||||
|
||||
if (myKeyStorage.checkBytesAreTheSame(addr, value, myDataDescriptor)) return true;
|
||||
if (myAssumeDifferentSerializedBytesMeansObjectsInequality) return false;
|
||||
if (myKeyStorage.checkBytesAreTheSame(addr, value)) return true;
|
||||
|
||||
if (myDataDescriptor instanceof DifferentSerializableBytesImplyNonEqualityPolicy) return false;
|
||||
return myDataDescriptor.isEqual(valueOf(idx), value);
|
||||
}
|
||||
|
||||
@@ -422,16 +430,11 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
}
|
||||
|
||||
public int getLargestId() {
|
||||
assert myKeyStorage != null;
|
||||
return myKeyStorage.getCurrentLength();
|
||||
}
|
||||
|
||||
protected int doWriteData(Data value) throws IOException {
|
||||
if (myKeyStorage != null) {
|
||||
return myKeyStorage.append(value, myDataDescriptor);
|
||||
|
||||
}
|
||||
return ((InlineKeyDescriptor<Data>)myDataDescriptor).toInt(value);
|
||||
return myKeyStorage.append(value);
|
||||
}
|
||||
|
||||
protected int setupValueId(int hashCode, int dataOff) {
|
||||
@@ -444,10 +447,6 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
}
|
||||
|
||||
public boolean iterateData(@NotNull Processor<? super Data> processor) throws IOException {
|
||||
if (myKeyStorage == null) {
|
||||
throw new UnsupportedOperationException("Iteration over InlineIntegerKeyDescriptors is not supported");
|
||||
}
|
||||
|
||||
lockStorage(); // todo locking in key storage
|
||||
try {
|
||||
myKeyStorage.force();
|
||||
@@ -456,7 +455,7 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
unlockStorage();
|
||||
}
|
||||
|
||||
return myKeyStorage.processAll(processor, myDataDescriptor);
|
||||
return myKeyStorage.processAll(processor);
|
||||
}
|
||||
|
||||
private Path keyStreamFile() {
|
||||
@@ -472,8 +471,7 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
try {
|
||||
int addr = indexToAddr(idx);
|
||||
|
||||
if (myKeyStorage == null) return ((InlineKeyDescriptor<Data>)myDataDescriptor).fromInt(addr);
|
||||
return myKeyStorage.read(addr, myDataDescriptor);
|
||||
return myKeyStorage.read(addr);
|
||||
}
|
||||
finally {
|
||||
unlockStorage();
|
||||
@@ -520,9 +518,7 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
|
||||
protected void doClose() throws IOException {
|
||||
try {
|
||||
if (myKeyStorage != null) {
|
||||
myKeyStorage.close();
|
||||
}
|
||||
myKeyStorage.close();
|
||||
flush();
|
||||
}
|
||||
finally {
|
||||
@@ -565,9 +561,7 @@ public abstract class PersistentEnumeratorBase<Data> implements DataEnumeratorEx
|
||||
lockStorage();
|
||||
|
||||
try {
|
||||
if (myKeyStorage != null) {
|
||||
myKeyStorage.force();
|
||||
}
|
||||
myKeyStorage.force();
|
||||
flush();
|
||||
}
|
||||
catch (IOException e) {
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.io.keyStorage;
|
||||
|
||||
import com.intellij.openapi.Forceable;
|
||||
import com.intellij.util.Processor;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
|
||||
public interface AppendableObjectStorage<Data> extends Forceable, Closeable {
|
||||
Data read(int addr) throws IOException;
|
||||
|
||||
boolean processAll(@NotNull Processor<? super Data> processor) throws IOException;
|
||||
|
||||
int append(Data value) throws IOException;
|
||||
|
||||
boolean checkBytesAreTheSame(int addr, Data value) throws IOException;
|
||||
|
||||
void lock();
|
||||
|
||||
void unlock();
|
||||
|
||||
int getCurrentLength();
|
||||
}
|
||||
+41
-125
@@ -1,22 +1,10 @@
|
||||
/*
|
||||
* Copyright 2000-2014 JetBrains s.r.o.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.intellij.util.io;
|
||||
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.io.keyStorage;
|
||||
|
||||
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
|
||||
import com.intellij.util.Processor;
|
||||
import com.intellij.util.io.DataOutputStream;
|
||||
import com.intellij.util.io.*;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
@@ -25,22 +13,25 @@ import java.nio.ByteBuffer;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class AppendableStorageBackedByResizableMappedFile extends ResizeableMappedFile {
|
||||
public class AppendableStorageBackedByResizableMappedFile<Data> extends ResizeableMappedFile implements AppendableObjectStorage<Data> {
|
||||
private final MyDataIS myReadStream;
|
||||
private byte[] myAppendBuffer;
|
||||
private volatile int myFileLength;
|
||||
private volatile int myBufferPosition;
|
||||
private static final int ourAppendBufferLength = 4096;
|
||||
@NotNull
|
||||
private final KeyDescriptor<Data> myDataDescriptor;
|
||||
|
||||
public AppendableStorageBackedByResizableMappedFile(final Path file,
|
||||
int initialSize,
|
||||
@Nullable PagedFileStorage.StorageLockContext lockContext,
|
||||
int pageSize,
|
||||
boolean valuesAreBufferAligned) {
|
||||
boolean valuesAreBufferAligned,
|
||||
@NotNull KeyDescriptor<Data> dataDescriptor) {
|
||||
super(file, initialSize, lockContext, pageSize, valuesAreBufferAligned);
|
||||
myDataDescriptor = dataDescriptor;
|
||||
myReadStream = new MyDataIS(this);
|
||||
myFileLength = (int)length();
|
||||
myCompressedAppendableFile = /*CompressedAppendableFile.ENABLED &&*/ null;
|
||||
}
|
||||
|
||||
private void flushKeyStoreBuffer() {
|
||||
@@ -54,28 +45,17 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
@Override
|
||||
public void force() {
|
||||
flushKeyStoreBuffer();
|
||||
if (myCompressedAppendableFile != null) myCompressedAppendableFile.force();
|
||||
super.force();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
flushKeyStoreBuffer();
|
||||
if (myCompressedAppendableFile != null) myCompressedAppendableFile.dispose();
|
||||
super.close();
|
||||
}
|
||||
|
||||
private final CompressedAppendableFile myCompressedAppendableFile;
|
||||
private static final boolean testMode = false;
|
||||
|
||||
public <Data> Data read(final int addr, KeyDescriptor<Data> descriptor) throws IOException {
|
||||
Data tempData = null;
|
||||
|
||||
if (myCompressedAppendableFile != null) {
|
||||
tempData = myCompressedAppendableFile.read(addr, descriptor);
|
||||
if (!testMode) return tempData;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Data read(final int addr) throws IOException {
|
||||
if (myFileLength <= addr) {
|
||||
// addr points to un-existed data
|
||||
if (myAppendBuffer == null) {
|
||||
@@ -88,40 +68,16 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
throw new NoDataException("requested address points to un-existed data");
|
||||
}
|
||||
|
||||
Data data =
|
||||
descriptor.read(new DataInputStream(new UnsyncByteArrayInputStream(myAppendBuffer, bufferOffset, myBufferPosition)));
|
||||
assert tempData == null || descriptor.isEqual(data, tempData);
|
||||
return data;
|
||||
return myDataDescriptor.read(new DataInputStream(new UnsyncByteArrayInputStream(myAppendBuffer, bufferOffset, myBufferPosition)));
|
||||
}
|
||||
// we do not need to flushKeyBuffer since we store complete records
|
||||
myReadStream.setup(addr, myFileLength);
|
||||
Data data = descriptor.read(myReadStream);
|
||||
assert tempData == null || descriptor.isEqual(data, tempData);
|
||||
return data;
|
||||
return myDataDescriptor.read(myReadStream);
|
||||
}
|
||||
|
||||
public <Data> boolean processAll(@NotNull Processor<? super Data> processor, @NotNull KeyDescriptor<Data> descriptor) throws IOException {
|
||||
@Override
|
||||
public boolean processAll(@NotNull Processor<? super Data> processor) throws IOException {
|
||||
assert !isDirty();
|
||||
DataInputStream keysStream2 = myCompressedAppendableFile != null ?myCompressedAppendableFile.getStream(0) : null;
|
||||
if (!testMode && keysStream2 != null) {
|
||||
getPagedFileStorage().lock(); // todo support it inside myCompressedAppendableFile to avoid filling the cache
|
||||
try {
|
||||
try {
|
||||
while (true) {
|
||||
Data key = descriptor.read(keysStream2);
|
||||
if (!processor.process(key)) return false;
|
||||
}
|
||||
}
|
||||
catch (EOFException e) {
|
||||
// Done
|
||||
}
|
||||
return true;
|
||||
}
|
||||
finally {
|
||||
getPagedFileStorage().unlock();
|
||||
keysStream2.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (myFileLength == 0) return true;
|
||||
|
||||
@@ -135,11 +91,7 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
}, 32768))) {
|
||||
try {
|
||||
while (true) {
|
||||
Data key = descriptor.read(keysStream);
|
||||
if (keysStream2 != null) {
|
||||
Data tempKey = descriptor.read(keysStream2);
|
||||
assert descriptor.isEqual(key, tempKey);
|
||||
}
|
||||
Data key = myDataDescriptor.read(keysStream);
|
||||
if (!processor.process(key)) return false;
|
||||
}
|
||||
}
|
||||
@@ -150,36 +102,21 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCurrentLength() {
|
||||
int currentLength;
|
||||
if (myCompressedAppendableFile != null) {
|
||||
currentLength = (int)myCompressedAppendableFile.length();
|
||||
if (testMode) {
|
||||
assert currentLength == myBufferPosition + myFileLength;
|
||||
}
|
||||
}
|
||||
else {
|
||||
currentLength = myBufferPosition + myFileLength;
|
||||
}
|
||||
|
||||
return currentLength;
|
||||
return myBufferPosition + myFileLength;
|
||||
}
|
||||
|
||||
public <Data> int append(Data value, KeyDescriptor<Data> descriptor) throws IOException {
|
||||
@Override
|
||||
public int append(Data value) throws IOException {
|
||||
final BufferExposingByteArrayOutputStream bos = new BufferExposingByteArrayOutputStream();
|
||||
DataOutput out = new DataOutputStream(bos);
|
||||
descriptor.save(out, value);
|
||||
DataOutput out = new com.intellij.util.io.DataOutputStream(bos);
|
||||
myDataDescriptor.save(out, value);
|
||||
final int size = bos.size();
|
||||
final byte[] buffer = bos.getInternalBuffer();
|
||||
|
||||
int currentLength = getCurrentLength();
|
||||
|
||||
if (myCompressedAppendableFile != null) {
|
||||
//myCompressedAppendableFile.append(value, descriptor);
|
||||
myCompressedAppendableFile.append(buffer, size);
|
||||
if (!testMode) return currentLength;
|
||||
}
|
||||
|
||||
if (size > ourAppendBufferLength) {
|
||||
flushKeyStoreBuffer();
|
||||
put(currentLength, buffer, 0, size);
|
||||
@@ -199,47 +136,26 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
return currentLength;
|
||||
}
|
||||
|
||||
<Data> boolean checkBytesAreTheSame(final int addr, Data value, KeyDescriptor<Data> descriptor) throws IOException {
|
||||
@Override
|
||||
public boolean checkBytesAreTheSame(final int addr, Data value) throws IOException {
|
||||
final boolean[] sameValue = new boolean[1];
|
||||
OutputStream comparer;
|
||||
|
||||
if (myCompressedAppendableFile != null) {
|
||||
final DataInputStream compressedStream = myCompressedAppendableFile.getStream(addr);
|
||||
|
||||
comparer = new OutputStream() {
|
||||
boolean same = true;
|
||||
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
if (same) {
|
||||
same = compressedStream.readByte() == (byte)b;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
sameValue[0] = same;
|
||||
}
|
||||
};
|
||||
} else {
|
||||
comparer = buildOldComparerStream(addr, sameValue);
|
||||
}
|
||||
|
||||
OutputStream comparer = buildOldComparerStream(addr, sameValue);
|
||||
DataOutput out = new DataOutputStream(comparer);
|
||||
descriptor.save(out, value);
|
||||
myDataDescriptor.save(out, value);
|
||||
comparer.close();
|
||||
|
||||
if (testMode) {
|
||||
final boolean[] sameValue2 = new boolean[1];
|
||||
OutputStream comparer2 = buildOldComparerStream(addr, sameValue2);
|
||||
out = new DataOutputStream(comparer2);
|
||||
descriptor.save(out, value);
|
||||
comparer2.close();
|
||||
assert sameValue[0] == sameValue2[0];
|
||||
}
|
||||
return sameValue[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lock() {
|
||||
getPagedFileStorage().lock();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unlock() {
|
||||
getPagedFileStorage().unlock();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private OutputStream buildOldComparerStream(final int addr, final boolean[] sameValue) {
|
||||
OutputStream comparer;
|
||||
@@ -269,7 +185,7 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
int address = storage.getOffsetInPage(addr);
|
||||
boolean same = true;
|
||||
ByteBuffer buffer = storage.getByteBuffer(addr, false).getCachedBuffer();
|
||||
final int myPageSize = storage.myPageSize;
|
||||
final int myPageSize = storage.getPageSize();
|
||||
|
||||
@Override
|
||||
public void write(int b) {
|
||||
@@ -298,7 +214,7 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
super(new MyBufferedIS(new MappedFileInputStream(raf, 0, 0)));
|
||||
}
|
||||
|
||||
public void setup(long pos, long limit) {
|
||||
void setup(long pos, long limit) {
|
||||
((MyBufferedIS)in).setup(pos, limit);
|
||||
}
|
||||
}
|
||||
@@ -308,9 +224,9 @@ public class AppendableStorageBackedByResizableMappedFile extends ResizeableMapp
|
||||
super(in, 512);
|
||||
}
|
||||
|
||||
public void setup(long pos, long limit) {
|
||||
void setup(long pos, long limit) {
|
||||
this.pos = 0;
|
||||
count = 0;
|
||||
this.count = 0;
|
||||
((MappedFileInputStream)in).setup(pos, limit);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.io.keyStorage;
|
||||
|
||||
import com.intellij.util.Processor;
|
||||
import com.intellij.util.io.InlineKeyDescriptor;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class InlinedKeyStorage<Data> implements AppendableObjectStorage<Data> {
|
||||
|
||||
private final InlineKeyDescriptor<Data> myDescriptor;
|
||||
|
||||
public InlinedKeyStorage(@NotNull InlineKeyDescriptor<Data> descriptor) {
|
||||
myDescriptor = descriptor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Data read(int addr) throws IOException {
|
||||
return myDescriptor.fromInt(addr);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean processAll(@NotNull Processor<? super Data> processor) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int append(Data value) throws IOException {
|
||||
return myDescriptor.toInt(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkBytesAreTheSame(int addr, Data value) throws IOException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lock() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unlock() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCurrentLength() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDirty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void force() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
|
||||
}
|
||||
}
|
||||
+2
-4
@@ -1,13 +1,11 @@
|
||||
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.io;
|
||||
package com.intellij.util.io.keyStorage;
|
||||
|
||||
import org.jetbrains.annotations.ApiStatus;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@ApiStatus.Internal
|
||||
class NoDataException extends IOException {
|
||||
public class NoDataException extends IOException {
|
||||
NoDataException(@NotNull String message) {
|
||||
super(message);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
@ApiStatus.Internal
|
||||
package com.intellij.util.io.keyStorage;
|
||||
|
||||
import org.jetbrains.annotations.ApiStatus;
|
||||
Reference in New Issue
Block a user