diff --git a/fleet/build/fs/srcJvmMain/fleet/buildtool/fs/ArchiveUtils.kt b/fleet/build/fs/srcJvmMain/fleet/buildtool/fs/ArchiveUtils.kt index 34f73db2d395..2d3ba22c3738 100644 --- a/fleet/build/fs/srcJvmMain/fleet/buildtool/fs/ArchiveUtils.kt +++ b/fleet/build/fs/srcJvmMain/fleet/buildtool/fs/ArchiveUtils.kt @@ -1,5 +1,7 @@ package fleet.buildtool.fs +import fleet.buildtool.fs.FilePermissions.Posix +import fleet.buildtool.fs.ReproducibilityMode.Reproducible.PermissionOption import org.apache.commons.compress.archivers.ArchiveEntry import org.apache.commons.compress.archivers.ArchiveOutputStream import org.apache.commons.compress.archivers.tar.TarArchiveEntry @@ -21,6 +23,7 @@ import java.io.InputStream import java.nio.file.Files import java.nio.file.LinkOption import java.nio.file.Path +import java.nio.file.attribute.FileTime import java.nio.file.attribute.PosixFileAttributeView import java.nio.file.attribute.PosixFilePermission import java.util.zip.* @@ -322,11 +325,6 @@ private fun restorePermissions(destinationFile: Path, unixMode: Int, isSymbolicL /** * Compresses a given [source] folder to an [outputFile] in ZIP format. - * - * TODO: Make it reproducible by using the methods from `zip(Path, Path, Boolean)` method (Make it optional, not to break backwards compatibility): - * 1. Sort entries alphabetically - * 2. Use invariant separators - * 3. Reset lastModified to 0 */ fun zip( source: Path, @@ -334,7 +332,8 @@ fun zip( withTopLevelFolder: Boolean, temporaryDir: Path, logger: Logger, -) = compress(source, outputFile, withTopLevelFolder, ArchiveType.ZIP, compressorName = "", temporaryDir, logger) + reproducibilityMode: ReproducibilityMode, +) = compress(source, outputFile, withTopLevelFolder, ArchiveType.ZIP, compressorName = "", reproducibilityMode, temporaryDir, logger) fun tarGz( source: Path, @@ -342,7 +341,8 @@ fun tarGz( withTopLevelFolder: Boolean, temporaryDir: Path, logger: Logger, -) = tarWithCompression(source, outputFile, withTopLevelFolder, CompressorStreamFactory.GZIP, temporaryDir, logger) + reproducibilityMode: ReproducibilityMode, +) = tarWithCompression(source, outputFile, withTopLevelFolder, CompressorStreamFactory.GZIP, reproducibilityMode, temporaryDir, logger) fun tarZst( source: Path, @@ -350,16 +350,18 @@ fun tarZst( withTopLevelFolder: Boolean, temporaryDir: Path, logger: Logger, -) = tarWithCompression(source, outputFile, withTopLevelFolder, CompressorStreamFactory.ZSTANDARD, temporaryDir, logger) + reproducibilityMode: ReproducibilityMode, +) = tarWithCompression(source, outputFile, withTopLevelFolder, CompressorStreamFactory.ZSTANDARD, reproducibilityMode, temporaryDir, logger) private fun tarWithCompression( source: Path, outputFile: Path, withTopLevelFolder: Boolean, compressorName: String, + reproducibilityMode: ReproducibilityMode, temporaryDir: Path, logger: Logger, -): Path = compress(source, outputFile, withTopLevelFolder, ArchiveType.TAR, compressorName, temporaryDir, logger) +): Path = compress(source, outputFile, withTopLevelFolder, ArchiveType.TAR, compressorName, reproducibilityMode, temporaryDir, logger) @OptIn(ExperimentalPathApi::class) private fun compress( @@ -368,6 +370,7 @@ private fun compress( withTopLevelFolder: Boolean, archiveType: ArchiveType, compressorName: String, + reproducibilityMode: ReproducibilityMode, temporaryDir: Path, logger: Logger, ): Path { @@ -399,19 +402,29 @@ private fun compress( source.isDirectory() -> { val base = when { withTopLevelFolder -> { - outputStream.addEntry(source, source.name, archiveType) + outputStream.addEntry(source, source.name, archiveType, reproducibilityMode) "${source.name}/" } else -> "./" } - source.walk().forEach { // must not follow symlinks - outputStream.addEntry(it, "$base${it.relativeTo(source)}", archiveType) + val fileTree = when (reproducibilityMode) { // must not follow symlinks + is ReproducibilityMode.None -> source.walk() + is ReproducibilityMode.Reproducible -> source.walk().sortedBy { path -> path } + } + + fileTree.forEach { + outputStream.addEntry( + it, + "$base${it.relativeTo(source).invariantSeparatorsPathString}", + archiveType, + reproducibilityMode, + ) } } - else -> outputStream.addEntry(source, source.name, archiveType = archiveType) + else -> outputStream.addEntry(source, source.name, archiveType, reproducibilityMode) } } } @@ -425,17 +438,15 @@ private fun compress( return outputFile } -private fun ArchiveOutputStream.addEntry(path: Path, relativePathInArchive: String, archiveType: ArchiveType) { +private fun ArchiveOutputStream.addEntry(path: Path, relativePathInArchive: String, archiveType: ArchiveType, reproducibilityMode: ReproducibilityMode) { when { path.isSymbolicLink() -> { when (archiveType) { ArchiveType.TAR -> { val entry = TarArchiveEntry(relativePathInArchive, LF_SYMLINK).also { + it.resetLastModifiedTime(reproducibilityMode) it.linkName = path.readSymbolicLink().pathString - when (val permissions = path.getFilePermissions(LinkOption.NOFOLLOW_LINKS)) { - FilePermissions.Other -> {} - is FilePermissions.Posix -> it.mode = permissions.toInt() - } + it.setFilePermissions(path, reproducibilityMode, LinkOption.NOFOLLOW_LINKS) } putArchiveEntry(entry) closeArchiveEntry() @@ -443,10 +454,8 @@ private fun ArchiveOutputStream.addEntry(path: Path, relativePathI ArchiveType.ZIP -> { val link = path.readSymbolicLink().pathString val entry = ZipArchiveEntry(relativePathInArchive).also { - when (val permissions = path.getFilePermissions(LinkOption.NOFOLLOW_LINKS)) { - FilePermissions.Other -> {} - is FilePermissions.Posix -> it.unixMode = UnixStat.LINK_FLAG or permissions.toInt() - } + it.resetLastModifiedTime(reproducibilityMode) + it.setFilePermissions(path, reproducibilityMode, UnixStat.LINK_FLAG, LinkOption.NOFOLLOW_LINKS) it.size = link.toByteArray().size.toLong() } putArchiveEntry(entry) @@ -461,16 +470,12 @@ private fun ArchiveOutputStream.addEntry(path: Path, relativePathI else -> path.inputStream().use { fileIn -> val entry: ArchiveEntry = when (archiveType) { ArchiveType.TAR -> TarArchiveEntry(path, relativePathInArchive).also { - when (val permissions = path.getFilePermissions()) { - FilePermissions.Other -> {} - is FilePermissions.Posix -> it.mode = permissions.toInt() - } + it.resetLastModifiedTime(reproducibilityMode) + it.setFilePermissions(path, reproducibilityMode) } ArchiveType.ZIP -> ZipArchiveEntry(path, relativePathInArchive).also { - when (val permissions = path.getFilePermissions()) { - FilePermissions.Other -> {} - is FilePermissions.Posix -> it.unixMode = permissions.toInt() - } + it.resetLastModifiedTime(reproducibilityMode) + it.setFilePermissions(path, reproducibilityMode) } } putArchiveEntry(entry) @@ -480,10 +485,67 @@ private fun ArchiveOutputStream.addEntry(path: Path, relativePathI } } +private fun TarArchiveEntry.setFilePermissions(path: Path, reproducibilityMode: ReproducibilityMode, vararg linkOption: LinkOption) { + when (val permissions = path.getFilePermissions(*linkOption)) { + FilePermissions.Other -> {} + is FilePermissions.Posix -> when (reproducibilityMode) { + is ReproducibilityMode.None -> // Store real file permissions + mode = permissions.toInt() + is ReproducibilityMode.Reproducible -> when (val permissionOption = reproducibilityMode.permissionOption) { + is PermissionOption.Override -> + mode = permissionOption.permissions.toInt() + PermissionOption.Preserve -> + mode = permissions.toInt() + } + } + } +} + +private fun ZipArchiveEntry.setFilePermissions(path: Path, reproducibilityMode: ReproducibilityMode, additionalFlags: Int? = null, vararg linkOption: LinkOption) { + when (val filePermissions = path.getFilePermissions(*linkOption)) { + FilePermissions.Other -> {} + is FilePermissions.Posix -> when (reproducibilityMode) { + is ReproducibilityMode.None -> // Store real file permissions + storePermissions(additionalFlags, filePermissions.permissions) + is ReproducibilityMode.Reproducible -> when (val permissionOption = reproducibilityMode.permissionOption) { + PermissionOption.Preserve -> // Store real file permissions + storePermissions(additionalFlags, filePermissions.permissions) + is PermissionOption.Override -> // Store artificial file permissions that make the file reproducible + storePermissions(additionalFlags, permissionOption.permissions) + } + } + } +} + +private fun ZipArchiveEntry.storePermissions(additionalFlags: Int? = null, permissions: Set) { + unixMode = when { + additionalFlags != null -> permissions.toInt() or additionalFlags + else -> permissions.toInt() + } +} + +private fun TarArchiveEntry.resetLastModifiedTime(reproducibilityMode: ReproducibilityMode) { + when (reproducibilityMode) { + is ReproducibilityMode.None -> {} + is ReproducibilityMode.Reproducible -> { + lastModifiedTime = FileTime.fromMillis(reproducibilityMode.lastModifiedTime) + } + } +} + +private fun ZipArchiveEntry.resetLastModifiedTime(reproducibilityMode: ReproducibilityMode) { + when (reproducibilityMode) { + is ReproducibilityMode.None -> {} + is ReproducibilityMode.Reproducible -> { + lastModifiedTime = FileTime.fromMillis(reproducibilityMode.lastModifiedTime) + } + } +} + private fun Path.getFilePermissions(vararg options: LinkOption): FilePermissions { return try { - FilePermissions.Posix(getPosixFilePermissions(*options)) + Posix(getPosixFilePermissions(*options)) } catch (_: UnsupportedOperationException) { FilePermissions.Other @@ -530,4 +592,54 @@ private sealed class FilePermissions { private enum class ArchiveType { ZIP, TAR, +} + +/** + * Enum representing modes of reproducibility for file compression operations. + * + * Reproducibility is a measure of ensuring consistent and predictable outcomes when creating compressed archives. + * Depending on the selected mode, adjustments such as sorting file entries, normalizing file attributes, + * or resetting file modification timestamps may be applied during compression to ensure the output archive + * remains consistent across different environments and execution runs. + */ +sealed class ReproducibilityMode { + + /** + * Represents the mode where no changes or adjustments are made to ensure reproducibility. + * Files and directories are processed and compressed as they are, preserving their original attributes. + */ + data object None : ReproducibilityMode() + + /** + * Represents the mode where adjustments are made to ensure reproducible outcomes. + * Files and directories are processed and compressed in a way that should ensure consistent and predictable outcomes across different environments and execution runs. + * + * @param permissionOption an option specifying how file permissions should be handled during compression. Defaults to [PermissionOption.Preserve]. + * - [PermissionOption.Preserve]: preserves the original file permissions. + * This option might not always give a reproducible results, since posix file permissions are different from windows ones, + * and will probably be lost during compression of the same files on Windows. + * - [PermissionOption.Override]: overrides the original file permissions with the specified ones to have consistent results. + * @param lastModifiedTime last modification timestamp to apply to files and directories. + * + * Please make sure that all the parameters are consistent across different environments and execution runs. + */ + data class Reproducible( + val permissionOption: PermissionOption, + val lastModifiedTime: Long = 0L, + ) : ReproducibilityMode() { + + sealed class PermissionOption { + data object Preserve : PermissionOption() + data class Override(val permissions: Set = defaultPermissions) : PermissionOption() + + companion object { + val defaultPermissions: Set = //Default permissions for reproducible archives, 0755 + setOf( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE, + ) + } + } + } } \ No newline at end of file diff --git a/fleet/build/fs/srcJvmTest/fleet/buildtool/fs/ArchiveUtilsTest.kt b/fleet/build/fs/srcJvmTest/fleet/buildtool/fs/ArchiveUtilsTest.kt index dc63a541518d..05411b19ed63 100644 --- a/fleet/build/fs/srcJvmTest/fleet/buildtool/fs/ArchiveUtilsTest.kt +++ b/fleet/build/fs/srcJvmTest/fleet/buildtool/fs/ArchiveUtilsTest.kt @@ -1,5 +1,11 @@ package fleet.buildtool.fs +import fleet.buildtool.fs.ReproducibilityMode.None +import fleet.buildtool.fs.ReproducibilityMode.Reproducible +import fleet.buildtool.fs.ReproducibilityMode.Reproducible.PermissionOption.Override +import fleet.buildtool.fs.ReproducibilityMode.Reproducible.PermissionOption.Preserve +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream import org.junit.Assume.assumeTrue import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -8,6 +14,7 @@ import java.io.IOException import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.nio.file.attribute.FileTime import java.nio.file.attribute.PosixFilePermission import java.util.zip.ZipEntry import java.util.zip.ZipInputStream @@ -167,7 +174,7 @@ class ArchiveUtilsTest { val extractTmpDir = tempDir.resolve("tmp2") // When - zip(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + zip(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, None) extractZip(outputFile, extractDir, stripTopLevelFolder = false, cleanDestination = false, extractTmpDir, logger) @@ -215,7 +222,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - zip(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + zip(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, None) // Then val extractDir = tempDir.resolve("extracted") @@ -571,7 +578,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, None) // Then assertTrue(outputFile.exists()) @@ -588,7 +595,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - tarGz(sourceDir, outputFile, withTopLevelFolder = true, tmpDir, logger) + tarGz(sourceDir, outputFile, withTopLevelFolder = true, tmpDir, logger, None) // Then assertTrue(outputFile.exists()) @@ -602,7 +609,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - tarGz(sourceFile, outputFile, withTopLevelFolder = false, tmpDir, logger) + tarGz(sourceFile, outputFile, withTopLevelFolder = false, tmpDir, logger, None) // Then assertTrue(outputFile.exists()) @@ -617,7 +624,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - val result = tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + val result = tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, reproducibilityMode = None) // Then assertEquals(outputFile, result) @@ -637,7 +644,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, reproducibilityMode = None) // Then val extractDir = tempDir.resolve("extracted") @@ -661,7 +668,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - tarZst(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + tarZst(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, reproducibilityMode = None) // Then assertTrue(outputFile.exists()) @@ -678,7 +685,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - tarZst(sourceDir, outputFile, withTopLevelFolder = true, tmpDir, logger) + tarZst(sourceDir, outputFile, withTopLevelFolder = true, tmpDir, logger, reproducibilityMode = None) // Then assertTrue(outputFile.exists()) @@ -695,7 +702,7 @@ class ArchiveUtilsTest { val extractTmpDir = tempDir.resolve("tmp2") // When - tarZst(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + tarZst(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, reproducibilityMode = None) extractTarZst(outputFile, extractDir, stripTopLevelFolder = false, cleanDestination = false, extractTmpDir, logger) @@ -713,7 +720,7 @@ class ArchiveUtilsTest { sourceDir.resolve("file.txt").writeText("content") val tarGz = tempDir.resolve("archive.tar.gz") val tmpDir = tempDir.resolve("tmp1") - tarGz(sourceDir, tarGz, withTopLevelFolder = true, tmpDir, logger) + tarGz(sourceDir, tarGz, withTopLevelFolder = true, tmpDir, logger, reproducibilityMode = None) val destination = tempDir.resolve("extracted") val extractTmpDir = tempDir.resolve("tmp2") @@ -733,7 +740,7 @@ class ArchiveUtilsTest { sourceDir.resolve("file.txt").writeText("content") val tarGz = tempDir.resolve("archive.tar.gz") val tmpDir = tempDir.resolve("tmp1") - tarGz(sourceDir, tarGz, withTopLevelFolder = true, tmpDir, logger) + tarGz(sourceDir, tarGz, withTopLevelFolder = true, tmpDir, logger, reproducibilityMode = None) val destination = tempDir.resolve("extracted") val extractTmpDir = tempDir.resolve("tmp2") @@ -753,7 +760,7 @@ class ArchiveUtilsTest { sourceDir.resolve("new.txt").writeText("new") val tarGz = tempDir.resolve("archive.tar.gz") val tmpDir = tempDir.resolve("tmp1") - tarGz(sourceDir, tarGz, withTopLevelFolder = false, tmpDir, logger) + tarGz(sourceDir, tarGz, withTopLevelFolder = false, tmpDir, logger, reproducibilityMode = None) val destination = tempDir.resolve("extracted").createDirectories() destination.resolve("old.txt").writeText("old") @@ -776,7 +783,7 @@ class ArchiveUtilsTest { nested.resolve("deep.txt").writeText("deep content") val tarGz = tempDir.resolve("single_symlink.tar.gz") val tmpDir = tempDir.resolve("tmp1") - tarGz(sourceDir, tarGz, withTopLevelFolder = false, tmpDir, logger) + tarGz(sourceDir, tarGz, withTopLevelFolder = false, tmpDir, logger, reproducibilityMode = None) val destination = tempDir.resolve("extracted") val extractTmpDir = tempDir.resolve("tmp2") @@ -834,7 +841,7 @@ class ArchiveUtilsTest { val tmpDir = tempDir.resolve("tmp") // When - tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger) + tarGz(sourceDir, outputFile, withTopLevelFolder = false, tmpDir, logger, reproducibilityMode = None) // Then val extractDir = tempDir.resolve("extracted") @@ -879,7 +886,7 @@ class ArchiveUtilsTest { sourceDir.resolve("file.txt").writeText("zst content") val tarZst = tempDir.resolve("archive.tar.zst") val tmpDir = tempDir.resolve("tmp1") - tarZst(sourceDir, tarZst, withTopLevelFolder = true, tmpDir, logger) + tarZst(sourceDir, tarZst, withTopLevelFolder = true, tmpDir, logger, reproducibilityMode = None) val destination = tempDir.resolve("extracted") val extractTmpDir = tempDir.resolve("tmp2") @@ -899,7 +906,7 @@ class ArchiveUtilsTest { sourceDir.resolve("file.txt").writeText("zst content") val tarZst = tempDir.resolve("archive.tar.zst") val tmpDir = tempDir.resolve("tmp1") - tarZst(sourceDir, tarZst, withTopLevelFolder = true, tmpDir, logger) + tarZst(sourceDir, tarZst, withTopLevelFolder = true, tmpDir, logger, reproducibilityMode = None) val destination = tempDir.resolve("extracted") val extractTmpDir = tempDir.resolve("tmp2") @@ -912,6 +919,86 @@ class ArchiveUtilsTest { assertEquals("zst content", destination.resolve("file.txt").readText()) } + @Test + fun `GIVEN last modified time changed WHEN extract tar zst THEN extraction is Reproducible`() { + val sourceDir = tempDir.resolve("source").createDirectories() + val file = sourceDir.resolve("file.txt").apply { writeText("zst content") } + val archive1 = tempDir.resolve("archive1.tar.zst") + val archive2 = tempDir.resolve("archive2.tar.zst") + val tmpDir = tempDir.resolve("tmp1") + + file.setLastModifiedTime(FileTime.fromMillis(0L)) + val tarZst1 = tarZst(file, archive1, withTopLevelFolder = false, temporaryDir = tmpDir, logger, Reproducible(Preserve)) + val firstSha256 = sha256(tarZst1.readBytesForSha256()) + + file.setLastModifiedTime(FileTime.fromMillis(System.currentTimeMillis())) + val tarZst2 = tarZst(file, archive2, withTopLevelFolder = false, temporaryDir = tmpDir, logger, Reproducible(Preserve)) + val secondSha256 = sha256(tarZst2.readBytesForSha256()) + + assertEquals(firstSha256, secondSha256, "tarZst compression should be Reproducible and not change no matter of system meta information") + } + + @Test + fun `GIVEN order is random WHEN extract tar zst THEN order is alphabetical`() { + val sourceDir = tempDir.resolve("source").createDirectories() + val fileC = sourceDir.resolve("C.txt").apply { writeText("C") } + val dirA = sourceDir.resolve("A").apply { createDirectories() } + val dirAFileA = dirA.resolve("a.txt").apply { writeText("a") } + val fileB = sourceDir.resolve("B.txt").apply { writeText("B") } + val fileA = sourceDir.resolve("A.txt").apply { writeText("A") } + val archiveFile = tempDir.resolve("archive.tar.zst") + val tmpDir = tempDir.resolve("tmp1") + val destination = tempDir.resolve("dest") + + val tarZstArchive = + tarZst(sourceDir, archiveFile, withTopLevelFolder = false, temporaryDir = tmpDir, logger, Reproducible(permissionOption = Override())) + extractTarZst(tarZstArchive, destination, stripTopLevelFolder = false, cleanDestination = false, tmpDir, logger) + + archiveFile.inputStream().buffered().use { bufferedInputStream -> + ZstdCompressorInputStream(bufferedInputStream).use { zstdInputStream -> + TarArchiveInputStream(zstdInputStream).use { tarInputStream -> + var entry = tarInputStream.nextEntry + val entries = mutableListOf() + while (entry != null) { + entries.add(entry.name) + entry = tarInputStream.nextEntry + } + + assertEquals("./A.txt", entries[0]) + assertEquals("./A/a.txt", entries[1]) + assertEquals("./B.txt", entries[2]) + assertEquals("./C.txt", entries[3]) + } + } + } + } + + @Test + fun `GIVEN permissions changed WHEN extract tar zst THEN extraction is Reproducible`() { + assumePosixFileSystem() + + val sourceDir = tempDir.resolve("source").createDirectories() + val file = sourceDir.resolve("file.txt").apply { writeText("zst content") } + val archive1 = tempDir.resolve("archive1.tar.zst") + val archive2 = tempDir.resolve("archive2.tar.zst") + val tmpDir = tempDir.resolve("tmp1") + archive1.deleteIfExists() + archive2.deleteIfExists() + + + val onlyOwnerReadPermissions = setOf(PosixFilePermission.OWNER_READ) + val readPermissions = setOf(PosixFilePermission.OTHERS_READ, PosixFilePermission.OWNER_READ, PosixFilePermission.GROUP_READ) + file.setPosixFilePermissions(readPermissions) + val tarZst1 = tarZst(file, archive1, withTopLevelFolder = false, temporaryDir = tmpDir, logger, Reproducible(Override())) + val firstSha256 = sha256(tarZst1.readBytesForSha256()) + + file.setPosixFilePermissions(onlyOwnerReadPermissions) + val tarZst2 = tarZst(file, archive2, withTopLevelFolder = false, temporaryDir = tmpDir, logger, Reproducible(Override())) + val secondSha256 = sha256(tarZst2.readBytesForSha256()) + + assertEquals(firstSha256, secondSha256, "tarZst compression should be Reproducible and not change no matter of system meta information") + } + // ========== Edge Cases and Error Handling ========== @Test @@ -990,7 +1077,7 @@ class ArchiveUtilsTest { } @Test - fun `tarGz and extractTarGz should preserve file permissions`() { + fun `tarGz and extractTarGz should preserve file permissions when ReproducibilityMode=NONE`() { assumePosixFileSystem() // Given @@ -999,7 +1086,6 @@ class ArchiveUtilsTest { val expectedPerms = setOf( PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, @@ -1010,7 +1096,7 @@ class ArchiveUtilsTest { val archive = tempDir.resolve("perm.tar.gz") val tmp = tempDir.resolve("tmp") - tarGz(sourceDir, archive, withTopLevelFolder = false, tmp, logger) + tarGz(sourceDir, archive, withTopLevelFolder = false, tmp, logger, None) val dest = tempDir.resolve("out-tar") extractTarGz(archive, dest, stripTopLevelFolder = false, cleanDestination = false, tmp, logger) @@ -1022,6 +1108,72 @@ class ArchiveUtilsTest { assertEquals(expectedPerms, actualPerms) } + @Test + fun `tarGz and extractTarGz should preserve file permissions when ReproducibilityMode=Reproducible with preserve`() { + assumePosixFileSystem() + + // Given + val sourceDir = tempDir.resolve("src").createDirectories() + val file = sourceDir.resolve("script.sh").apply { writeText("echo hi") } + val expectedPerms = setOf( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_EXECUTE, + ) + + file.setPosixFilePermissions(expectedPerms) + + val archive = tempDir.resolve("perm.tar.gz") + val tmp = tempDir.resolve("tmp") + tarGz(sourceDir, archive, withTopLevelFolder = false, tmp, logger, Reproducible(Preserve)) + + val dest = tempDir.resolve("out-tar") + extractTarGz(archive, dest, stripTopLevelFolder = false, cleanDestination = false, tmp, logger) + + // Then + val extracted = dest.resolve("script.sh") + assertTrue(extracted.exists()) + val actualPerms = extracted.getPosixFilePermissions() + assertEquals(expectedPerms, actualPerms) + } + + @Test + fun `tarGz and extractTarGz should use 0755 file permissions when ReproducibilityMode=Reproducible with override`() { + assumePosixFileSystem() + + // Given + val sourceDir = tempDir.resolve("src").createDirectories() + val file = sourceDir.resolve("script.sh").apply { writeText("echo hi") } + val defaultPermissions = setOf( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + ) + + val expectedPermissions = setOf( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE, + ) + + file.setPosixFilePermissions(defaultPermissions) + + val archive = tempDir.resolve("perm.tar.gz") + val tmp = tempDir.resolve("tmp") + tarGz(sourceDir, archive, withTopLevelFolder = false, tmp, logger, Reproducible(Override())) + + val dest = tempDir.resolve("out-tar") + extractTarGz(archive, dest, stripTopLevelFolder = false, cleanDestination = false, tmp, logger) + + // Then + val extracted = dest.resolve("script.sh") + assertTrue(extracted.exists()) + val actualPerms = extracted.getPosixFilePermissions() + assertEquals(expectedPermissions, actualPerms) + } + @Test fun `zip and extractZip should preserve file permissions`() { assumePosixFileSystem() @@ -1043,7 +1195,7 @@ class ArchiveUtilsTest { val archive = tempDir.resolve("script.sh") val tmp = tempDir.resolve("tmpZip") - zip(sourceDir, archive, withTopLevelFolder = false, tmp, logger) + zip(sourceDir, archive, withTopLevelFolder = false, tmp, logger, reproducibilityMode = None) val dest = tempDir.resolve("out-zip") extractZip(archive, dest, stripTopLevelFolder = false, cleanDestination = false, tmp, logger) diff --git a/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/AwsFleetS3Client.kt b/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/AwsFleetS3Client.kt index b8ef82ca4c51..ca4e36aee07e 100644 --- a/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/AwsFleetS3Client.kt +++ b/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/AwsFleetS3Client.kt @@ -28,7 +28,6 @@ class AwsFleetS3Client(private val client: AwsClient) : FleetS3Client { override suspend fun getObject(bucket: String, key: String, temporaryDir: Path): Path { val file = createTempFile(temporaryDir, "s3-download-", "") - file.createFile() client.getObject(input = GetObjectRequest { this.bucket = bucket this.key = key diff --git a/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/UploadToS3Utils.kt b/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/UploadToS3Utils.kt index 699461c3e0b6..1d0b1cc138e8 100644 --- a/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/UploadToS3Utils.kt +++ b/fleet/build/s3/srcJvmMain/fleet/buildtool/s3/upload/UploadToS3Utils.kt @@ -1,5 +1,7 @@ package fleet.buildtool.s3.upload +import fleet.buildtool.fs.ReproducibilityMode +import fleet.buildtool.fs.ReproducibilityMode.Reproducible.PermissionOption.Preserve import fleet.buildtool.fs.readBytesForSha256 import fleet.buildtool.fs.sha256 import fleet.buildtool.fs.tarZst @@ -46,6 +48,7 @@ suspend fun uploadToS3( withTopLevelFolder = false, temporaryDir = temporaryDir, logger = logger, + reproducibilityMode = ReproducibilityMode.Reproducible(permissionOption = Preserve) ) }