From d21efe177d18eff037ee3a0b7e285e80b977655e Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Tue, 17 Apr 2018 14:54:27 +0200 Subject: [PATCH] Get rid of Snappy library, 2 (IDEA-189365) --- .../util/indexing/SnapshotInputMappings.java | 4 +- platform/util/intellij.platform.util.iml | 1 - .../openapi/application/PathManager.java | 2 - .../com/intellij/util/CompressionUtil.java | 145 +++++++----------- .../intellij/util/io/DataInputOutputUtil.java | 29 ++++ 5 files changed, 87 insertions(+), 94 deletions(-) diff --git a/platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java b/platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java index ebf9e240429a..1103086843c7 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java @@ -258,14 +258,14 @@ public class SnapshotInputMappings { new DataExternalizer() { @Override public void save(@NotNull DataOutput out, String value) throws IOException { - out.write((byte[])CompressionUtil.compressCharSequence(value, Charset.defaultCharset())); + out.write((byte[])CompressionUtil.compressStringRawBytes(value)); } @Override public String read(@NotNull DataInput in) throws IOException { byte[] b = new byte[((InputStream)in).available()]; in.readFully(b); - return (String)CompressionUtil.uncompressCharSequence(b, Charset.defaultCharset()); + return (String)CompressionUtil.uncompressStringRawBytes(b); } }, 4096); } diff --git a/platform/util/intellij.platform.util.iml b/platform/util/intellij.platform.util.iml index ee8827e22446..9aa64a3f67f6 100644 --- a/platform/util/intellij.platform.util.iml +++ b/platform/util/intellij.platform.util.iml @@ -20,7 +20,6 @@ - diff --git a/platform/util/src/com/intellij/openapi/application/PathManager.java b/platform/util/src/com/intellij/openapi/application/PathManager.java index a32ca2b45c5e..0675b5bda6ee 100644 --- a/platform/util/src/com/intellij/openapi/application/PathManager.java +++ b/platform/util/src/com/intellij/openapi/application/PathManager.java @@ -17,7 +17,6 @@ import net.jpountz.lz4.LZ4Factory; import org.apache.log4j.Appender; import org.apache.oro.text.regex.PatternMatcher; import org.intellij.lang.annotations.Flow; -import org.iq80.snappy.Snappy; import org.jdom.Document; import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; @@ -511,7 +510,6 @@ public class PathManager { TypeMapper.class, // JNA FileUtils.class, // JNA (jna-platform) PatternMatcher.class, // OROMatcher - Snappy.class, // Snappy LZ4Factory.class, // lz4-java }; diff --git a/platform/util/src/com/intellij/util/CompressionUtil.java b/platform/util/src/com/intellij/util/CompressionUtil.java index 47a121e12bb2..e5cb74327996 100644 --- a/platform/util/src/com/intellij/util/CompressionUtil.java +++ b/platform/util/src/com/intellij/util/CompressionUtil.java @@ -16,18 +16,21 @@ package com.intellij.util; import com.intellij.openapi.util.ThreadLocalCachedByteArray; +import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; import com.intellij.util.io.DataInputOutputUtil; import com.intellij.util.io.DataOutputStream; import com.intellij.util.text.StringFactory; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; -import org.iq80.snappy.CorruptionException; -import org.iq80.snappy.Snappy; +import net.jpountz.lz4.LZ4FastDecompressor; import org.jetbrains.annotations.NotNull; -import java.io.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.CharBuffer; -import java.nio.charset.Charset; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -40,10 +43,13 @@ public class CompressionUtil { public static int writeCompressed(@NotNull DataOutput out, @NotNull byte[] bytes, int start, int length) throws IOException { if (length > COMPRESSION_THRESHOLD) { - byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length)); - int compressedSize = Snappy.compress(bytes, start, length, compressedOutputBuffer, 0); + LZ4Compressor compressor = compressor(); + + byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length)); + int compressedSize = compressor.compress(bytes, start, length, compressedOutputBuffer, 0); if (compressedSize < length) { DataInputOutputUtil.writeINT(out, -compressedSize); + DataInputOutputUtil.writeINT(out, length - compressedSize); out.write(compressedOutputBuffer, 0, compressedSize); return compressedSize; } @@ -62,22 +68,13 @@ public class CompressionUtil { private static final AtomicLong mySizeAfterCompression = new AtomicLong(); public static final boolean DUMP_COMPRESSION_STATS = SystemProperties.getBooleanProperty("idea.dump.compression.stats", false); - public static final boolean USE_SNAPPY = SystemProperties.getBooleanProperty("idea.use.snappy", false); public static int writeCompressedWithoutOriginalBufferLength(@NotNull DataOutput out, @NotNull byte[] bytes, int length) throws IOException { long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0; - final byte[] compressedOutputBuffer; - int compressedSize; - - if (USE_SNAPPY) { - compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length)); - compressedSize = Snappy.compress(bytes, 0, length, compressedOutputBuffer, 0); - } else { - LZ4Compressor compressor = LZ4Factory.fastestJavaInstance().fastCompressor(); - compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length)); - compressedSize = compressor.compress(bytes, 0, length, compressedOutputBuffer, 0); - } + LZ4Compressor compressor = compressor(); + final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length)); + int compressedSize = compressor.compress(bytes, 0, length, compressedOutputBuffer, 0); final long time = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started; mySizeAfterCompression.addAndGet(compressedSize); @@ -95,6 +92,10 @@ public class CompressionUtil { return compressedSize; } + private static LZ4Compressor compressor() { + return LZ4Factory.fastestJavaInstance().fastCompressor(); + } + @NotNull public static byte[] readCompressedWithoutOriginalBufferLength(@NotNull DataInput in, int originalBufferLength) throws IOException { int size = DataInputOutputUtil.readINT(in); @@ -105,13 +106,7 @@ public class CompressionUtil { int decompressedRequests = myDecompressionRequests.incrementAndGet(); long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0; - final byte[] decompressedResult; - - if (USE_SNAPPY) { - decompressedResult = Snappy.uncompress(bytes, 0, size); - } else { - decompressedResult = LZ4Factory.fastestJavaInstance().fastDecompressor().decompress(bytes, 0, originalBufferLength); - } + final byte[] decompressedResult = decompressor().decompress(bytes, 0, originalBufferLength); long doneTime = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started; long decompressedSize = myDecompressedSize.addAndGet(size); @@ -123,13 +118,22 @@ public class CompressionUtil { return decompressedResult; } + protected static LZ4FastDecompressor decompressor() { + return LZ4Factory.fastestJavaInstance().fastDecompressor(); + } + @NotNull public static byte[] readCompressed(@NotNull DataInput in) throws IOException { int size = DataInputOutputUtil.readINT(in); if (size < 0) { - byte[] bytes = spareBufferLocal.getBuffer(-size); - in.readFully(bytes, 0, -size); - return Snappy.uncompress(bytes, 0, -size); + size = -size; + byte[] bytes = spareBufferLocal.getBuffer(size); + int sizeUncompressed = DataInputOutputUtil.readINT(in) + size; + in.readFully(bytes, 0, size); + byte[] result = new byte[sizeUncompressed]; + int decompressed = decompressor().decompress(bytes, 0, result, 0, sizeUncompressed); + assert decompressed == size; + return result; } else { byte[] bytes = new byte[size]; @@ -140,38 +144,6 @@ public class CompressionUtil { private static final int STRING_COMPRESSION_THRESHOLD = 1024; - @NotNull - public static CharSequence uncompressCharSequence(@NotNull Object string, @NotNull Charset charset) { - if (string instanceof CharSequence) return (CharSequence)string; - byte[] b = (byte[])string; - try { - int uncompressedLength = Snappy.getUncompressedLength(b, 0); - byte[] bytes = spareBufferLocal.getBuffer(uncompressedLength); - int bytesLength = Snappy.uncompress(b, 0, b.length, bytes, 0); - return new String(bytes, 0, bytesLength, charset); - } - catch (CorruptionException ex) { - throw new RuntimeException(ex); - } - } - - @NotNull - public static Object compressCharSequence(@NotNull CharSequence string, @NotNull Charset charset) { - if (string.length() < STRING_COMPRESSION_THRESHOLD) { - if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) { - string = string.toString(); // shrink to size - } - return string; - } - try { - return Snappy.compress(string.toString().getBytes(charset)); - } - catch (CorruptionException ex) { - ex.printStackTrace(); - return string; - } - } - @NotNull public static Object compressStringRawBytes(@NotNull CharSequence string) { int length = string.length(); @@ -182,20 +154,22 @@ public class CompressionUtil { return string; } try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(length); + BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(length); @NotNull DataOutput out = new DataOutputStream(bytes); - DataInputOutputUtil.writeINT(out, length); for (int i=0; i< length;i++) { char c = string.charAt(i); DataInputOutputUtil.writeINT(out, c); } - byte[] compressedBytes = Snappy.compress(bytes.toByteArray()); - return compressedBytes.length < length * 2 ? compressedBytes : string; - } - catch (CorruptionException ex) { - ex.printStackTrace(); - return string; + + LZ4Compressor compressor = compressor(); + int bytesWritten = bytes.size(); + ByteBuffer dest = ByteBuffer.wrap(spareBufferLocal.getBuffer(compressor.maxCompressedLength(bytesWritten) + 10)); + DataInputOutputUtil.writeINT(dest, length); + DataInputOutputUtil.writeINT(dest, bytesWritten - length); + compressor.compress(ByteBuffer.wrap(bytes.getInternalBuffer(), 0, bytesWritten), dest); + + return dest.position() < length * 2 ? Arrays.copyOf(dest.array(), dest.position()) : string; } catch (IOException e) { e.printStackTrace(); @@ -206,28 +180,21 @@ public class CompressionUtil { @NotNull public static CharSequence uncompressStringRawBytes(@NotNull Object compressed) { if (compressed instanceof CharSequence) return (CharSequence)compressed; - byte[] b = (byte[])compressed; - try { - int uncompressedLength = Snappy.getUncompressedLength(b, 0); - byte[] bytes = spareBufferLocal.getBuffer(uncompressedLength); - int bytesLength = Snappy.uncompress(b, 0, b.length, bytes, 0); - ByteArrayInputStream byteStream = new ByteArrayInputStream(bytes, 0, bytesLength); - @NotNull DataInput in = new DataInputStream(byteStream); + + ByteBuffer buffer = ByteBuffer.wrap((byte[])compressed); + int len = DataInputOutputUtil.readINT(buffer); + int uncompressedLength = DataInputOutputUtil.readINT(buffer) + len; + + ByteBuffer dest = ByteBuffer.wrap(spareBufferLocal.getBuffer(uncompressedLength), 0, uncompressedLength); + decompressor().decompress(buffer, dest); + dest.rewind(); + + char[] chars = new char[len]; - int len = DataInputOutputUtil.readINT(in); - char[] chars = new char[len]; - - for (int i=0; i val || val >= 192) { + byteBuffer.put( (byte)(192 + (val & 0x3F))); + val >>>= 6; + while (val >= 128) { + byteBuffer.put((byte)((val & 0x7F) | 0x80)); + val >>>= 7; + } + } + byteBuffer.put((byte)val); + } + public static long readLONG(@NotNull DataInput record) throws IOException { final int val = record.readUnsignedByte(); if (val < 192) {