MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Get rid of Snappy library, 2 (IDEA-189365)

Browse Source
This commit is contained in:
Maxim.Mossienko
2018-04-17 14:56:49 +02:00
parent 5efd39f0a2
commit d21efe177d
5 changed files with 87 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java
+2 -2
View File
@@ -258,14 +258,14 @@ public class SnapshotInputMappings<Key, Value, Input> {
new DataExternalizer<String>() {
@Override
public void save(@NotNull DataOutput out, String value) throws IOException {
out.write((byte[])CompressionUtil.compressCharSequence(value, Charset.defaultCharset()));
out.write((byte[])CompressionUtil.compressStringRawBytes(value));
}
@Override
public String read(@NotNull DataInput in) throws IOException {
byte[] b = new byte[((InputStream)in).available()];
in.readFully(b);
return (String)CompressionUtil.uncompressCharSequence(b, Charset.defaultCharset());
return (String)CompressionUtil.uncompressStringRawBytes(b);
}
}, 4096);
}
platform/util/intellij.platform.util.iml
-1
View File
@@ -20,7 +20,6 @@
<orderEntry type="module" module-name="intellij.platform.util.rt" exported="" />
<orderEntry type="library" name="jna" level="project" />
<orderEntry type="library" name="OroMatcher" level="project" />
<orderEntry type="library" exported="" name="Snappy-Java" level="project" />
<orderEntry type="library" name="imgscalr" level="project" />
<orderEntry type="library" name="xmlgraphics-commons" level="project" />
<orderEntry type="library" name="batik" level="project" />
platform/util/src/com/intellij/openapi/application/PathManager.java
-2
View File
@@ -17,7 +17,6 @@ import net.jpountz.lz4.LZ4Factory;
import org.apache.log4j.Appender;
import org.apache.oro.text.regex.PatternMatcher;
import org.intellij.lang.annotations.Flow;
import org.iq80.snappy.Snappy;
import org.jdom.Document;
import org.jetbrains.annotations.Contract;
import org.jetbrains.annotations.NotNull;
@@ -511,7 +510,6 @@ public class PathManager {
TypeMapper.class, // JNA
FileUtils.class, // JNA (jna-platform)
PatternMatcher.class, // OROMatcher
Snappy.class, // Snappy
LZ4Factory.class, // lz4-java
};
platform/util/src/com/intellij/util/CompressionUtil.java
+56 -89
View File
@@ -16,18 +16,21 @@
package com.intellij.util;
import com.intellij.openapi.util.ThreadLocalCachedByteArray;
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.util.io.DataInputOutputUtil;
import com.intellij.util.io.DataOutputStream;
import com.intellij.util.text.StringFactory;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import org.iq80.snappy.CorruptionException;
import org.iq80.snappy.Snappy;
import net.jpountz.lz4.LZ4FastDecompressor;
import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -40,10 +43,13 @@ public class CompressionUtil {
public static int writeCompressed(@NotNull DataOutput out, @NotNull byte[] bytes, int start, int length) throws IOException {
if (length > COMPRESSION_THRESHOLD) {
byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length));
int compressedSize = Snappy.compress(bytes, start, length, compressedOutputBuffer, 0);
LZ4Compressor compressor = compressor();
byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length));
int compressedSize = compressor.compress(bytes, start, length, compressedOutputBuffer, 0);
if (compressedSize < length) {
DataInputOutputUtil.writeINT(out, -compressedSize);
DataInputOutputUtil.writeINT(out, length - compressedSize);
out.write(compressedOutputBuffer, 0, compressedSize);
return compressedSize;
}
@@ -62,22 +68,13 @@ public class CompressionUtil {
private static final AtomicLong mySizeAfterCompression = new AtomicLong();
public static final boolean DUMP_COMPRESSION_STATS = SystemProperties.getBooleanProperty("idea.dump.compression.stats", false);
public static final boolean USE_SNAPPY = SystemProperties.getBooleanProperty("idea.use.snappy", false);
public static int writeCompressedWithoutOriginalBufferLength(@NotNull DataOutput out, @NotNull byte[] bytes, int length) throws IOException {
long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0;
final byte[] compressedOutputBuffer;
int compressedSize;
if (USE_SNAPPY) {
compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length));
compressedSize = Snappy.compress(bytes, 0, length, compressedOutputBuffer, 0);
} else {
LZ4Compressor compressor = LZ4Factory.fastestJavaInstance().fastCompressor();
compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length));
compressedSize = compressor.compress(bytes, 0, length, compressedOutputBuffer, 0);
}
LZ4Compressor compressor = compressor();
final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length));
int compressedSize = compressor.compress(bytes, 0, length, compressedOutputBuffer, 0);
final long time = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started;
mySizeAfterCompression.addAndGet(compressedSize);
@@ -95,6 +92,10 @@ public class CompressionUtil {
return compressedSize;
}
private static LZ4Compressor compressor() {
return LZ4Factory.fastestJavaInstance().fastCompressor();
}
@NotNull
public static byte[] readCompressedWithoutOriginalBufferLength(@NotNull DataInput in, int originalBufferLength) throws IOException {
int size = DataInputOutputUtil.readINT(in);
@@ -105,13 +106,7 @@ public class CompressionUtil {
int decompressedRequests = myDecompressionRequests.incrementAndGet();
long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0;
final byte[] decompressedResult;
if (USE_SNAPPY) {
decompressedResult = Snappy.uncompress(bytes, 0, size);
} else {
decompressedResult = LZ4Factory.fastestJavaInstance().fastDecompressor().decompress(bytes, 0, originalBufferLength);
}
final byte[] decompressedResult = decompressor().decompress(bytes, 0, originalBufferLength);
long doneTime = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started;
long decompressedSize = myDecompressedSize.addAndGet(size);
@@ -123,13 +118,22 @@ public class CompressionUtil {
return decompressedResult;
}
protected static LZ4FastDecompressor decompressor() {
return LZ4Factory.fastestJavaInstance().fastDecompressor();
}
@NotNull
public static byte[] readCompressed(@NotNull DataInput in) throws IOException {
int size = DataInputOutputUtil.readINT(in);
if (size < 0) {
byte[] bytes = spareBufferLocal.getBuffer(-size);
in.readFully(bytes, 0, -size);
return Snappy.uncompress(bytes, 0, -size);
size = -size;
byte[] bytes = spareBufferLocal.getBuffer(size);
int sizeUncompressed = DataInputOutputUtil.readINT(in) + size;
in.readFully(bytes, 0, size);
byte[] result = new byte[sizeUncompressed];
int decompressed = decompressor().decompress(bytes, 0, result, 0, sizeUncompressed);
assert decompressed == size;
return result;
}
else {
byte[] bytes = new byte[size];
@@ -140,38 +144,6 @@ public class CompressionUtil {
private static final int STRING_COMPRESSION_THRESHOLD = 1024;
@NotNull
public static CharSequence uncompressCharSequence(@NotNull Object string, @NotNull Charset charset) {
if (string instanceof CharSequence) return (CharSequence)string;
byte[] b = (byte[])string;
try {
int uncompressedLength = Snappy.getUncompressedLength(b, 0);
byte[] bytes = spareBufferLocal.getBuffer(uncompressedLength);
int bytesLength = Snappy.uncompress(b, 0, b.length, bytes, 0);
return new String(bytes, 0, bytesLength, charset);
}
catch (CorruptionException ex) {
throw new RuntimeException(ex);
}
}
@NotNull
public static Object compressCharSequence(@NotNull CharSequence string, @NotNull Charset charset) {
if (string.length() < STRING_COMPRESSION_THRESHOLD) {
if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
string = string.toString(); // shrink to size
}
return string;
}
try {
return Snappy.compress(string.toString().getBytes(charset));
}
catch (CorruptionException ex) {
ex.printStackTrace();
return string;
}
}
@NotNull
public static Object compressStringRawBytes(@NotNull CharSequence string) {
int length = string.length();
@@ -182,20 +154,22 @@ public class CompressionUtil {
return string;
}
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(length);
BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(length);
@NotNull DataOutput out = new DataOutputStream(bytes);
DataInputOutputUtil.writeINT(out, length);
for (int i=0; i< length;i++) {
char c = string.charAt(i);
DataInputOutputUtil.writeINT(out, c);
}
byte[] compressedBytes = Snappy.compress(bytes.toByteArray());
return compressedBytes.length < length * 2 ? compressedBytes : string;
}
catch (CorruptionException ex) {
ex.printStackTrace();
return string;
LZ4Compressor compressor = compressor();
int bytesWritten = bytes.size();
ByteBuffer dest = ByteBuffer.wrap(spareBufferLocal.getBuffer(compressor.maxCompressedLength(bytesWritten) + 10));
DataInputOutputUtil.writeINT(dest, length);
DataInputOutputUtil.writeINT(dest, bytesWritten - length);
compressor.compress(ByteBuffer.wrap(bytes.getInternalBuffer(), 0, bytesWritten), dest);
return dest.position() < length * 2 ? Arrays.copyOf(dest.array(), dest.position()) : string;
}
catch (IOException e) {
e.printStackTrace();
@@ -206,28 +180,21 @@ public class CompressionUtil {
@NotNull
public static CharSequence uncompressStringRawBytes(@NotNull Object compressed) {
if (compressed instanceof CharSequence) return (CharSequence)compressed;
byte[] b = (byte[])compressed;
try {
int uncompressedLength = Snappy.getUncompressedLength(b, 0);
byte[] bytes = spareBufferLocal.getBuffer(uncompressedLength);
int bytesLength = Snappy.uncompress(b, 0, b.length, bytes, 0);
ByteArrayInputStream byteStream = new ByteArrayInputStream(bytes, 0, bytesLength);
@NotNull DataInput in = new DataInputStream(byteStream);
ByteBuffer buffer = ByteBuffer.wrap((byte[])compressed);
int len = DataInputOutputUtil.readINT(buffer);
int uncompressedLength = DataInputOutputUtil.readINT(buffer) + len;
ByteBuffer dest = ByteBuffer.wrap(spareBufferLocal.getBuffer(uncompressedLength), 0, uncompressedLength);
decompressor().decompress(buffer, dest);
dest.rewind();
char[] chars = new char[len];
int len = DataInputOutputUtil.readINT(in);
char[] chars = new char[len];
for (int i=0; i<len; i++) {
int c = DataInputOutputUtil.readINT(in);
chars[i] = (char)c;
}
return StringFactory.createShared(chars);
}
catch (CorruptionException ex) {
throw new RuntimeException(ex);
}
catch (IOException e) {
throw new RuntimeException(e);
for (int i=0; i<len; i++) {
int c = DataInputOutputUtil.readINT(dest);
chars[i] = (char)c;
}
return StringFactory.createShared(chars);
}
}
platform/util/src/com/intellij/util/io/DataInputOutputUtil.java
+29
View File
@@ -24,6 +24,7 @@ import org.jetbrains.annotations.Nullable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* @author max
@@ -38,10 +39,38 @@ public class DataInputOutputUtil extends DataInputOutputUtilRt {
return DataInputOutputUtilRt.readINT(record);
}
public static int readINT(@NotNull ByteBuffer byteBuffer) {
final int val = byteBuffer.get() & 0xFF;
if (val < 192) {
return val;
}
int res = val - 192;
for (int sh = 6; ; sh += 7) {
int next = byteBuffer.get() & 0xFF;
res |= (next & 0x7F) << sh;
if ((next & 0x80) == 0) {
return res;
}
}
}
public static void writeINT(@NotNull DataOutput record, int val) throws IOException {
DataInputOutputUtilRt.writeINT(record, val);
}
public static void writeINT(@NotNull ByteBuffer byteBuffer, int val) {
if (0 > val || val >= 192) {
byteBuffer.put( (byte)(192 + (val & 0x3F)));
val >>>= 6;
while (val >= 128) {
byteBuffer.put((byte)((val & 0x7F) | 0x80));
val >>>= 7;
}
}
byteBuffer.put((byte)val);
}
public static long readLONG(@NotNull DataInput record) throws IOException {
final int val = record.readUnsignedByte();
if (val < 192) {