From 6f6cd93cdb354c3d467bcfe242454d9c93248902 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Mon, 14 Dec 2020 12:30:25 +0100 Subject: [PATCH] reorder jars - keep all dirs with resources GitOrigin-RevId: 31caa3c19478452d911607b81942963d45a4f638 --- .../org/jetbrains/intellij/build/io/zip.kt | 26 +++++++++++-- .../intellij/build/tasks/reorderJars.kt | 9 ++++- .../intellij/build/tasks/ReorderJarsTest.kt | 39 ++++++++++++++++++- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt b/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt index 07ff611db79e..5c9ba8eb655d 100644 --- a/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt +++ b/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt @@ -6,6 +6,7 @@ import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream import org.apache.commons.compress.parallel.InputStreamSupplier import java.io.BufferedInputStream +import java.io.InputStream import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes import java.util.* @@ -13,20 +14,39 @@ import java.util.concurrent.Executors import java.util.zip.ZipEntry // symlinks not supported but can be easily implemented - see CollectingVisitor.visitFile -fun zipForWindows(targetFile: Path, dirs: Iterable) { +@JvmOverloads +fun zipForWindows(targetFile: Path, dirs: Iterable, addDirEntries: Boolean = false) { val zipCreator = ParallelScatterZipCreator(Executors.newWorkStealingPool()) // note - dirs contain duplicated directories (you cannot simply add directory entry on visit - uniqueness must be preserved) // anyway, directory entry are not added for (dir in dirs) { - val visitor = CollectingVisitor(zipCreator, dir.toAbsolutePath().normalize()) + val visitor = CollectingVisitor(zipCreator, dir.toAbsolutePath().normalize(), addDirEntries = addDirEntries) Files.walkFileTree(visitor.rootDir, visitor) } ZipArchiveOutputStream(Files.newByteChannel(targetFile, EnumSet.of(StandardOpenOption.WRITE, StandardOpenOption.CREATE))) .use(zipCreator::writeTo) } +private val emptyInputStream = object : InputStream() { + override fun read() = -1 +} + +private val emptyInputStreamSupplier = InputStreamSupplier { + emptyInputStream +} + private class CollectingVisitor(private val zipCreator: ParallelScatterZipCreator, - val rootDir: Path) : SimpleFileVisitor() { + val rootDir: Path, + val addDirEntries: Boolean = false) : SimpleFileVisitor() { + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { + if (addDirEntries) { + val entry = ZipArchiveEntry("${rootDir.relativize(dir).toString().replace('\\', '/')}/") + entry.method = ZipEntry.STORED + zipCreator.addArchiveEntry(entry, emptyInputStreamSupplier) + } + return super.preVisitDirectory(dir, attrs) + } + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { if (attrs.isSymbolicLink) { throw RuntimeException("Symlinks are not allowed for Windows archive") diff --git a/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt b/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt index 6bea2261dd8f..92de502721da 100644 --- a/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt +++ b/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt @@ -247,9 +247,14 @@ private fun reorderJar(jarFile: Path, orderedNames: List, resultJarFile: for (entry in entries) { val name = entry.name if (!entry.isDirectory && !name.endsWith(".class") && !name.endsWith("/package.html") && name != "META-INF/MANIFEST.MF") { - val slashIndex = name.lastIndexOf('/') + var slashIndex = name.lastIndexOf('/') if (slashIndex != -1) { - dirSetWithoutClassFiles.add(name.substring(0, slashIndex)) + while (dirSetWithoutClassFiles.add(name.substring(0, slashIndex))) { + slashIndex = name.lastIndexOf('/', slashIndex - 2) + if (slashIndex == -1) { + break + } + } } } } diff --git a/build/tasks/test/org/jetbrains/intellij/build/tasks/ReorderJarsTest.kt b/build/tasks/test/org/jetbrains/intellij/build/tasks/ReorderJarsTest.kt index 80336e118fc7..ca8792bc8173 100644 --- a/build/tasks/test/org/jetbrains/intellij/build/tasks/ReorderJarsTest.kt +++ b/build/tasks/test/org/jetbrains/intellij/build/tasks/ReorderJarsTest.kt @@ -4,26 +4,63 @@ package org.jetbrains.intellij.build.tasks import com.intellij.testFramework.PlatformTestUtil import com.intellij.testFramework.TemporaryDirectory +import com.intellij.testFramework.rules.InMemoryFsRule import com.intellij.util.lang.JarMemoryLoader import com.intellij.util.lang.JdkZipResourceFile import com.intellij.util.lang.ZipResourceFile import org.apache.commons.compress.archivers.zip.ZipFile import org.assertj.core.api.Assertions.assertThat +import org.jetbrains.intellij.build.io.zipForWindows import org.junit.Rule import org.junit.Test import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.util.zip.ZipEntry +import kotlin.random.Random private val testDataPath: Path get() = Paths.get(PlatformTestUtil.getPlatformTestDataPath(), "plugins/reorderJars") class ReorderJarsTest { + @JvmField + @Rule + val fsRule = InMemoryFsRule() + @JvmField @Rule val tempDir = TemporaryDirectory() + @Test + fun `keep all dirs with resources`() { + // check that not only immediate parent of resource file is preserved, but also any dir in a path + val random = Random(42) + + val rootDir = fsRule.fs.getPath("/dir") + val dir = rootDir.resolve("dir2/dir3") + Files.createDirectories(dir) + Files.write(dir.resolve("resource.txt"), random.nextBytes(random.nextInt(128))) + + val dir2 = rootDir.resolve("anotherDir") + Files.createDirectories(dir2) + Files.write(dir2.resolve("resource2.txt"), random.nextBytes(random.nextInt(128))) + + val archiveFile = fsRule.fs.getPath("/dir/archive.jar") + zipForWindows(archiveFile, listOf(rootDir), addDirEntries = true) + + doReorderJars(mapOf(archiveFile to emptyList()), archiveFile.parent, archiveFile.parent, TaskTest.logger) + ZipFile(Files.newByteChannel(archiveFile)).use { zipFile -> + assertThat(zipFile.entriesInPhysicalOrder.asSequence().map { it.name }.sorted().joinToString(separator = "\n")).isEqualTo(""" + META-INF/jb/${'$'}${'$'}size${'$'}${'$'} + anotherDir/ + anotherDir/resource2.txt + dir2/ + dir2/dir3/ + dir2/dir3/resource.txt + """.trimIndent()) + } + } + @Test fun testReordering() { val path = testDataPath @@ -41,7 +78,7 @@ class ReorderJarsTest { val file = files[0].toPath() assertThat(file.fileName.toString()).isEqualTo("annotations.jar") var data: ByteArray - ZipFile(file.toFile()).use { zipFile2 -> + ZipFile(Files.newByteChannel(file)).use { zipFile2 -> val entries = zipFile2.entriesInPhysicalOrder.toList() assertThat(entries[0].name).isEqualTo(SIZE_ENTRY) val entry = entries[1]