From 2de36f53d7f0a9eb601529471c81af5b0cd5b41e Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Mon, 18 Jun 2018 11:13:21 +0300 Subject: [PATCH] [platform] introduces compressor API ... as a foundation of future merger/replacement for TarUtil/ZipUtil --- .../src/com/intellij/util/io/Compressor.java | 188 ++++++++++++++++++ .../com/intellij/util/io/CompressorTest.kt | 97 +++++++++ 2 files changed, 285 insertions(+) create mode 100644 platform/util/src/com/intellij/util/io/Compressor.java create mode 100644 platform/util/testSrc/com/intellij/util/io/CompressorTest.kt diff --git a/platform/util/src/com/intellij/util/io/Compressor.java b/platform/util/src/com/intellij/util/io/Compressor.java new file mode 100644 index 000000000000..8b32b8e8ac73 --- /dev/null +++ b/platform/util/src/com/intellij/util/io/Compressor.java @@ -0,0 +1,188 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.io; + +import com.intellij.openapi.util.Condition; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public abstract class Compressor implements Closeable { + public static class Tar extends Compressor { + public Tar(@NotNull File file) throws IOException { + myStream = new TarArchiveOutputStream(new GzipCompressorOutputStream(new FileOutputStream(file))); + myStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + } + + public Tar(@NotNull TarArchiveOutputStream stream) { + myStream = stream; + } + + // + private final TarArchiveOutputStream myStream; + + @Override + protected void writeDirectoryEntry(String name, long timestamp) throws IOException { + TarArchiveEntry e = new TarArchiveEntry(name + '/'); + e.setModTime(timestamp); + myStream.putArchiveEntry(e); + myStream.closeArchiveEntry(); + } + + @Override + protected void writeFileEntry(String name, InputStream source, long length, long timestamp) throws IOException { + TarArchiveEntry e = new TarArchiveEntry(name); + e.setSize(length); + e.setModTime(timestamp); + myStream.putArchiveEntry(e); + FileUtil.copy(source, myStream); + myStream.closeArchiveEntry(); + } + + @Override + public void close() throws IOException { + myStream.close(); + } + // + } + + public static class Zip extends Compressor { + public Zip(@NotNull File file) throws FileNotFoundException { + myStream = new ZipOutputStream(new FileOutputStream(file)); + } + + public Zip(@NotNull ZipOutputStream stream) { + myStream = stream; + } + + // + private final ZipOutputStream myStream; + + @Override + protected void writeDirectoryEntry(String name, long timestamp) throws IOException { + ZipEntry e = new ZipEntry(name + '/'); + e.setMethod(ZipEntry.STORED); + e.setSize(0); + e.setCrc(0); + e.setTime(timestamp); + myStream.putNextEntry(e); + myStream.closeEntry(); + } + + @Override + protected void writeFileEntry(String name, InputStream source, long length, long timestamp) throws IOException { + ZipEntry e = new ZipEntry(name); + if (length == 0) { + e.setMethod(ZipEntry.STORED); + e.setSize(0); + e.setCrc(0); + } + e.setTime(timestamp); + myStream.putNextEntry(e); + FileUtil.copy(source, myStream); + myStream.closeEntry(); + } + + @Override + public void close() throws IOException { + myStream.close(); + } + // + } + + private Condition myFilter = null; + + public Compressor filter(@Nullable Condition filter) { + myFilter = filter; + return this; + } + + public final void addFile(@NotNull String entryName, @NotNull File file) throws IOException { + entryName = entryName(entryName); + if (accepts(entryName)) { + InputStream source = new FileInputStream(file); + try { + writeFileEntry(entryName, source, file.length(), file.lastModified()); + } + finally { + source.close(); + } + } + } + + public final void addFile(@NotNull String entryName, @NotNull byte[] content) throws IOException { + addFile(entryName, content, -1); + } + + public final void addFile(@NotNull String entryName, @NotNull byte[] content, long timestamp) throws IOException { + entryName = entryName(entryName); + if (accepts(entryName)) { + writeFileEntry(entryName, new ByteArrayInputStream(content), content.length, timestamp(timestamp)); + } + } + + public final void addDirectory(@NotNull String entryName) throws IOException { + addDirectory(entryName, -1); + } + + public final void addDirectory(@NotNull String entryName, long timestamp) throws IOException { + entryName = entryName(entryName); + if (accepts(entryName)) { + writeDirectoryEntry(entryName, timestamp(timestamp)); + } + } + + public final void addDirectory(@NotNull File directory) throws IOException { + addRecursively("", directory); + } + + public final void addDirectory(@NotNull String prefix, @NotNull File directory) throws IOException { + addRecursively(entryName(prefix), directory); + } + + // + protected Compressor() { } + + private static String entryName(String name) { + String entryName = StringUtil.trimLeading(StringUtil.trimTrailing(name.replace('\\', '/'), '/'), '/'); + if (StringUtil.isEmpty(entryName)) throw new IllegalArgumentException("Invalid entry name: " + name); + return entryName; + } + + private static long timestamp(long timestamp) { + return timestamp == -1 ? System.currentTimeMillis() : timestamp; + } + + private boolean accepts(String entryName) { + return myFilter == null || myFilter.value(entryName); + } + + private void addRecursively(String prefix, File directory) throws IOException { + File[] children = directory.listFiles(); + if (children != null) { + if (!prefix.isEmpty()) { + addDirectory(prefix, directory.lastModified()); + } + for (File child: children) { + String name = prefix.isEmpty() ? child.getName() : prefix + '/' + child.getName(); + if (child.isDirectory()) { + addRecursively(name, child); + } + else { + addFile(name, child); + } + } + } + } + + protected abstract void writeDirectoryEntry(String name, long timestamp) throws IOException; + protected abstract void writeFileEntry(String name, InputStream source, long length, long timestamp) throws IOException; + // +} \ No newline at end of file diff --git a/platform/util/testSrc/com/intellij/util/io/CompressorTest.kt b/platform/util/testSrc/com/intellij/util/io/CompressorTest.kt new file mode 100644 index 000000000000..f288db1a2c0b --- /dev/null +++ b/platform/util/testSrc/com/intellij/util/io/CompressorTest.kt @@ -0,0 +1,97 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.io + +import com.intellij.testFramework.rules.TempDirectory +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +import org.assertj.core.api.Assertions.assertThat +import org.junit.Rule +import org.junit.Test +import java.io.File +import java.io.FileInputStream +import java.util.zip.ZipInputStream + +class CompressorTest { + @Rule @JvmField var tempDir = TempDirectory() + + @Test fun simpleZip() { + val zip = tempDir.newFile("test.zip") + Compressor.Zip(zip).use { it.addFile("file.txt", "123".toByteArray()) } + assertZip(zip, "file.txt" to "123") + } + + @Test fun simpleZipWithFilters() { + val zip = tempDir.newFile("test.zip") + val set = mutableSetOf() + Compressor.Zip(zip).filter(set::add).use { + it.addFile("file1.txt", "123".toByteArray()) + it.addFile("file2.txt", "456".toByteArray()) + it.addFile("file1.txt", "789".toByteArray()) + } + assertZip(zip, "file1.txt" to "123", "file2.txt" to "456") + } + + @Test fun recursiveZip() { + val dir = tempDir.newFolder("dir") + tempDir.newFile("dir/f1").writeText("1") + tempDir.newFile("dir/f2").writeText("2") + tempDir.newFile("dir/d1/f11").writeText("11") + tempDir.newFile("dir/d1/f12").writeText("12") + tempDir.newFile("dir/d1/d11/f111").writeText("111") + tempDir.newFile("dir/d1/d11/f112").writeText("112") + tempDir.newFile("dir/d2/f21").writeText("21") + tempDir.newFile("dir/d2/f22").writeText("22") + + val zip = tempDir.newFile("test.zip") + Compressor.Zip(zip).use { it.addDirectory(dir) } + assertZip( + zip, + "d1/" to "", "d1/d11/" to "", "d2/" to "", + "f1" to "1", "f2" to "2", + "d1/f11" to "11", "d1/f12" to "12", + "d1/d11/f111" to "111", "d1/d11/f112" to "112", + "d2/f21" to "21", "d2/f22" to "22") + } + + @Test fun recursiveTarWithPrefix() { + val dir = tempDir.newFolder("dir") + tempDir.newFile("dir/f1").writeText("1") + tempDir.newFile("dir/f2").writeText("2") + tempDir.newFile("dir/d1/f11").writeText("11") + tempDir.newFile("dir/d1/f12").writeText("12") + tempDir.newFile("dir/d1/d11/f111").writeText("111") + tempDir.newFile("dir/d1/d11/f112").writeText("112") + tempDir.newFile("dir/d2/f21").writeText("21") + tempDir.newFile("dir/d2/f22").writeText("22") + + val tar = tempDir.newFile("test.tgz") + Compressor.Tar(tar).use { it.addDirectory("tar/", dir) } + assertTar( + tar, + "tar/" to "", "tar/d1/" to "", "tar/d1/d11/" to "", "tar/d2/" to "", + "tar/f1" to "1", "tar/f2" to "2", + "tar/d1/f11" to "11", "tar/d1/f12" to "12", + "tar/d1/d11/f111" to "111", "tar/d1/d11/f112" to "112", + "tar/d2/f21" to "21", "tar/d2/f22" to "22") + } + + @Test fun entryNameTrimming() { + val zip = tempDir.newFile("test.zip") + Compressor.Zip(zip).use { it.addFile("//file.txt//", "123".toByteArray()) } + assertZip(zip, "file.txt" to "123") + } + + private fun assertZip(zip: File, vararg expected: Pair) { + val actual = ZipInputStream(FileInputStream(zip)).use { + generateSequence(it::getNextEntry).map { entry -> entry.name to String(it.readBytes()) }.toList() + } + assertThat(actual).containsExactlyInAnyOrder(*expected) + } + + private fun assertTar(tar: File, vararg expected: Pair) { + val actual = TarArchiveInputStream(GzipCompressorInputStream(FileInputStream(tar))).use { + generateSequence(it::getNextTarEntry).map { entry -> entry.name to String(it.readBytes()) }.toList() + } + assertThat(actual).containsExactlyInAnyOrder(*expected) + } +} \ No newline at end of file