MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 06:05:01 +07:00
Code Issues Packages Projects Releases Wiki Activity

allow storage to have custom allocation policy =>

Browse Source 
this allows storing vfs content with smaller slack (5% for zipped size)
This commit is contained in:
Maxim.Mossienko
2012-06-21 19:42:45 +04:00
parent 7931bdba15
commit 03d19dd59a
5 changed files with 119 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/FSRecords.java
+40 -43
View File
@@ -34,14 +34,9 @@ import com.intellij.util.concurrency.JBLock;
import com.intellij.util.concurrency.JBReentrantReadWriteLock;
import com.intellij.util.concurrency.LockFactory;
import com.intellij.util.containers.IntArrayList;
import com.intellij.util.io.PagedFileStorage;
import com.intellij.util.io.PersistentStringEnumerator;
import com.intellij.util.io.ResizeableMappedFile;
import com.intellij.util.io.*;
import com.intellij.util.io.DataOutputStream;
import com.intellij.util.io.storage.AbstractStorage;
import com.intellij.util.io.storage.HeavyProcessLatch;
import com.intellij.util.io.storage.RefCountingStorage;
import com.intellij.util.io.storage.Storage;
import com.intellij.util.io.storage.*;
import gnu.trove.TIntArrayList;
import gnu.trove.TObjectIntHashMap;
import org.jetbrains.annotations.NotNull;
@@ -56,7 +51,7 @@ import java.util.concurrent.ScheduledFuture;
public class FSRecords implements Forceable {
private static final Logger LOG = Logger.getInstance("#com.intellij.vfs.persistent.FSRecords");
private static final int VERSION = 14;
private static final int VERSION = 15;
private static final int PARENT_OFFSET = 0;
private static final int PARENT_SIZE = 4;
@@ -213,7 +208,7 @@ public class FSRecords implements Forceable {
PagedFileStorage.StorageLockContext storageLockContext = new PagedFileStorage.StorageLock(false).myDefaultStorageLockContext;
myNames = new PersistentStringEnumerator(namesFile, storageLockContext);
myAttributes = new Storage(attributesFile.getCanonicalPath());
myContents = new RefCountingStorage(contentsFile.getCanonicalPath());
myContents = new RefCountingStorage(contentsFile.getCanonicalPath(), CapacityAllocationPolicy.FIVE_PERCENT_FOR_GROWTH); // sources usually zipped with 4x ratio
boolean aligned = PagedFileStorage.BUFFER_SIZE % RECORD_SIZE == 0;
assert aligned; // for performance
myRecords = new ResizeableMappedFile(recordsFile, 20 * 1024, storageLockContext,
@@ -599,8 +594,8 @@ public class FSRecords implements Forceable {
if (att_page != 0) {
final DataInputStream attStream = getAttributesStorage().readStream(att_page);
while (attStream.available() > 0) {
attStream.readInt(); // Attribute ID;
int attAddress = attStream.readInt();
DataInputOutputUtil.readINT(attStream); // Attribute ID;
int attAddress = DataInputOutputUtil.readINT(attStream);
getAttributesStorage().deleteRecord(attAddress);
}
attStream.close();
@@ -621,11 +616,11 @@ public class FSRecords implements Forceable {
int[] result;
try {
final int count = input.readInt();
final int count = DataInputOutputUtil.readINT(input);
result = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
input.readInt(); // Name
result[i] = input.readInt(); // Id
DataInputOutputUtil.readINT(input); // Name
result[i] = DataInputOutputUtil.readINT(input); // Id
}
return result;
}
@@ -660,12 +655,12 @@ public class FSRecords implements Forceable {
if (input != null) {
try {
final int count = input.readInt();
final int count = DataInputOutputUtil.readINT(input);
names = ArrayUtil.newIntArray(count);
ids = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
final int name = input.readInt();
final int id = input.readInt();
final int name = DataInputOutputUtil.readINT(input);
final int id = DataInputOutputUtil.readINT(input);
if (name == root) {
return id;
}
@@ -683,13 +678,13 @@ public class FSRecords implements Forceable {
int id;
try {
id = createRecord();
output.writeInt(names.length + 1);
DataInputOutputUtil.writeINT(output, names.length + 1);
for (int i = 0; i < names.length; i++) {
output.writeInt(names[i]);
output.writeInt(ids[i]);
DataInputOutputUtil.writeINT(output, names[i]);
DataInputOutputUtil.writeINT(output, ids[i]);
}
output.writeInt(root);
output.writeInt(id);
DataInputOutputUtil.writeINT(output, root);
DataInputOutputUtil.writeINT(output, id);
}
finally {
output.close();
@@ -712,13 +707,13 @@ public class FSRecords implements Forceable {
int[] names;
int[] ids;
try {
count = input.readInt();
count = DataInputOutputUtil.readINT(input);
names = ArrayUtil.newIntArray(count);
ids = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
names[i] = input.readInt();
ids[i] = input.readInt();
names[i] = DataInputOutputUtil.readINT(input);
ids[i] = DataInputOutputUtil.readINT(input);
}
}
finally {
@@ -733,10 +728,10 @@ public class FSRecords implements Forceable {
final DataOutputStream output = writeAttribute(1, CHILDREN_ATT, false);
try {
output.writeInt(count - 1);
DataInputOutputUtil.writeINT(output, count - 1);
for (int i = 0; i < names.length; i++) {
output.writeInt(names[i]);
output.writeInt(ids[i]);
DataInputOutputUtil.writeINT(output, names[i]);
DataInputOutputUtil.writeINT(output, ids[i]);
}
}
finally {
@@ -754,10 +749,12 @@ public class FSRecords implements Forceable {
final DataInputStream input = readAttribute(id, CHILDREN_ATT);
if (input == null) return ArrayUtil.EMPTY_INT_ARRAY;
final int count = input.readInt();
final int count = DataInputOutputUtil.readINT(input);
final int[] result = ArrayUtil.newIntArray(count);
for (int i = 0; i < count; i++) {
result[i] = input.readInt();
int childId = DataInputOutputUtil.readINT(input);
childId = childId >= 0 ? childId + id : -childId;
result[i] = childId;
}
input.close();
return result;
@@ -776,11 +773,12 @@ public class FSRecords implements Forceable {
final DataInputStream input = readAttribute(parentId, CHILDREN_ATT);
if (input == null) return Pair.create(ArrayUtil.EMPTY_STRING_ARRAY, ArrayUtil.EMPTY_INT_ARRAY);
final int count = input.readInt();
final int count = DataInputOutputUtil.readINT(input);
final int[] ids = ArrayUtil.newIntArray(count);
final String[] names = ArrayUtil.newStringArray(count);
for (int i = 0; i < count; i++) {
int id = input.readInt();
int id = DataInputOutputUtil.readINT(input);
id = id >= 0 ? id + parentId : -id;
ids[i] = id;
names[i] = getName(id);
}
@@ -813,13 +811,14 @@ public class FSRecords implements Forceable {
w.lock();
DbConnection.markDirty();
final DataOutputStream record = writeAttribute(id, CHILDREN_ATT, false);
record.writeInt(children.length);
DataInputOutputUtil.writeINT(record, children.length);
for (int child : children) {
if (child == id) {
LOG.error("Cyclic parent child relations");
}
else {
record.writeInt(child);
child = child > id ? child - id : -child;
DataInputOutputUtil.writeINT(record, child);
}
}
record.close();
@@ -1123,8 +1122,8 @@ public class FSRecords implements Forceable {
DataInputStream attrRefs = storage.readStream(recordId);
try {
while (attrRefs.available() > 0) {
final int attIdOnPage = attrRefs.readInt();
final int attrAddress = attrRefs.readInt();
final int attIdOnPage = DataInputOutputUtil.readINT(attrRefs);
final int attrAddress = DataInputOutputUtil.readINT(attrRefs);
if (attIdOnPage == encodedAttrId) return attrAddress;
}
@@ -1136,9 +1135,9 @@ public class FSRecords implements Forceable {
if (toWrite) {
Storage.AppenderStream appender = storage.appendStream(recordId);
appender.writeInt(encodedAttrId);
DataInputOutputUtil.writeINT(appender, encodedAttrId);
int attrAddress = storage.createNewRecord();
appender.writeInt(attrAddress);
DataInputOutputUtil.writeINT(appender, attrAddress);
appender.close();
return attrAddress;
}
@@ -1403,11 +1402,9 @@ public class FSRecords implements Forceable {
final DataInputStream dataInputStream = getAttributesStorage().readStream(attributeRecordId);
try {
final int streamSize = dataInputStream.available();
assert streamSize % 8 == 0;
for (int i = 0; i < streamSize / 8; i++) {
int attId = dataInputStream.readInt();
int attDataRecordId = dataInputStream.readInt();
while(dataInputStream.available() > 0) {
int attId = DataInputOutputUtil.readINT(dataInputStream);
int attDataRecordId = DataInputOutputUtil.readINT(dataInputStream);
assert !usedAttributeRecordIds.contains(attDataRecordId);
usedAttributeRecordIds.add(attDataRecordId);
if (!validAttributeIds.contains(attId)) {
platform/util/src/com/intellij/util/io/storage/AbstractStorage.java
+23 -17
View File
@@ -26,11 +26,15 @@ import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.openapi.util.io.ByteSequence;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.io.*;
import com.intellij.util.io.DataOutputStream;
import com.intellij.util.io.PagePool;
import com.intellij.util.io.RecordDataOutput;
import com.intellij.util.io.UnsyncByteArrayInputStream;
import org.jetbrains.annotations.NonNls;
import java.io.*;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
@SuppressWarnings({"HardCodedStringLiteral"})
public abstract class AbstractStorage implements Disposable, Forceable {
@@ -46,6 +50,7 @@ public abstract class AbstractStorage implements Disposable, Forceable {
protected AbstractRecordsTable myRecordsTable;
protected DataTable myDataTable;
protected PagePool myPool;
private final CapacityAllocationPolicy myCapacityAllocationPolicy;
public static boolean deleteFiles(String storageFilePath) {
final File recordsFile = new File(storageFilePath + INDEX_EXTENSION);
@@ -67,6 +72,19 @@ public abstract class AbstractStorage implements Disposable, Forceable {
}
protected AbstractStorage(String storageFilePath, PagePool pool) throws IOException {
this(storageFilePath, pool, CapacityAllocationPolicy.DEFAULT);
}
protected AbstractStorage(String storageFilePath,
CapacityAllocationPolicy capacityAllocationPolicy) throws IOException {
this(storageFilePath, PagePool.SHARED, capacityAllocationPolicy);
}
protected AbstractStorage(String storageFilePath,
PagePool pool,
CapacityAllocationPolicy capacityAllocationPolicy) throws IOException {
myCapacityAllocationPolicy = capacityAllocationPolicy != null ? capacityAllocationPolicy
: CapacityAllocationPolicy.DEFAULT;
tryInit(storageFilePath, pool, 0);
}
@@ -139,7 +157,7 @@ public abstract class AbstractStorage implements Disposable, Forceable {
if (size > 0) {
assert addr > 0;
final int capacity = calcCapacity(size);
final int capacity = myCapacityAllocationPolicy.calculateCapacity(size);
final long newaddr = newDataTable.allocateSpace(capacity);
final byte[] bytes = new byte[size];
myDataTable.readBytes(addr, bytes);
@@ -202,19 +220,6 @@ public abstract class AbstractStorage implements Disposable, Forceable {
}
}
private static int calcCapacity(int requiredLength) {
return Math.max(64, Math.min(nearestPowerOfTwo(requiredLength * 3 / 2), (requiredLength / 1024 + 1) * 1024));
}
private static int nearestPowerOfTwo(int n) {
int power = 1;
while (n != 0) {
power *= 2;
n /= 2;
}
return power;
}
public StorageDataOutput writeStream(final int record) {
return writeStream(record, false);
}
@@ -289,7 +294,8 @@ public abstract class AbstractStorage implements Disposable, Forceable {
else {
myDataTable.reclaimSpace(currentCapacity);
final int newCapacity = fixedSize ? requiredLength : calcCapacity(requiredLength);
int newCapacity = fixedSize ? requiredLength:myCapacityAllocationPolicy.calculateCapacity(requiredLength);
if (newCapacity < requiredLength) newCapacity = requiredLength;
address = myDataTable.allocateSpace(newCapacity);
myRecordsTable.setAddress(record, address);
myRecordsTable.setCapacity(record, newCapacity);
platform/util/src/com/intellij/util/io/storage/CapacityAllocationPolicy.java
+48
View File
@@ -0,0 +1,48 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.io.storage;
public abstract class CapacityAllocationPolicy {
public abstract int calculateCapacity(int requiredLength);
public static final CapacityAllocationPolicy FIXED = new CapacityAllocationPolicy() {
@Override
public int calculateCapacity(int requiredLength) {
return requiredLength;
}
};
public static final CapacityAllocationPolicy FIVE_PERCENT_FOR_GROWTH = new CapacityAllocationPolicy() {
@Override
public int calculateCapacity(int requiredLength) {
return Math.min((int)(requiredLength * 1.05), (requiredLength / 1024 + 1) * 1024);
}
};
public static final CapacityAllocationPolicy DEFAULT = new CapacityAllocationPolicy() {
@Override
public int calculateCapacity(int requiredLength) {
return Math.max(64, Math.min(Integer.highestOneBit(requiredLength * 3 / 2) << 1, (requiredLength / 1024 + 1) * 1024));
}
};
public static final CapacityAllocationPolicy REASONABLY_SMALL = new CapacityAllocationPolicy() {
@Override
public int calculateCapacity(int requiredLength) { // 20% for growth
return Math.max(8, Math.min((int)(requiredLength * 1.2), (requiredLength / 1024 + 1) * 1024));
}
};
}
platform/util/src/com/intellij/util/io/storage/RefCountingStorage.java
+4
View File
@@ -54,6 +54,10 @@ public class RefCountingStorage extends AbstractStorage {
super(path);
}
public RefCountingStorage(String path, CapacityAllocationPolicy capacityAllocationPolicy) throws IOException {
super(path, capacityAllocationPolicy);
}
public DataInputStream readStream(int record) throws IOException {
if (myDoNotZipCaches) return super.readStream(record);
BufferExposingByteArrayOutputStream stream = internalReadStream(record);
platform/util/src/com/intellij/util/io/storage/Storage.java
+4
View File
@@ -33,6 +33,10 @@ public class Storage extends AbstractStorage {
super(path, pool);
}
public Storage(String path, CapacityAllocationPolicy capacityAllocationPolicy) throws IOException {
super(path, capacityAllocationPolicy);
}
@Override
protected AbstractRecordsTable createRecordsTable(PagePool pool, File recordsFile) throws IOException {
return new RecordsTable(recordsFile, pool);