From 7a601ed2c2fee1e93ffca024e69eb0b108b0a2be Mon Sep 17 00:00:00 2001 From: "Dmitriy.Panov" Date: Fri, 15 Jul 2022 19:15:24 +0200 Subject: [PATCH] CWM-6546 IJI-772 checking OS-specific executable patterns, not only Runtime GitOrigin-RevId: 0d2f0bdd952b1a1d0b8eeef9deb422b82e2de812 --- .../intellij/build/IdeaCommunityBuildTest.kt | 1 - .../intellij/build/impl/BuildTasksImpl.kt | 2 +- .../intellij/build/impl/BundledRuntime.kt | 2 - .../intellij/build/impl/BundledRuntimeImpl.kt | 85 --------------- .../build/impl/LinuxDistributionBuilder.kt | 8 +- .../build/impl/MacDistributionBuilder.kt | 32 +++--- .../intellij/build/impl/MacDmgBuilder.kt | 20 ++-- .../impl/OsSpecificDistributionBuilder.kt | 103 +++++++++++++++++- .../build/impl/WindowsDistributionBuilder.kt | 2 +- 9 files changed, 134 insertions(+), 121 deletions(-) diff --git a/build/tests/testSrc/org/jetbrains/intellij/build/IdeaCommunityBuildTest.kt b/build/tests/testSrc/org/jetbrains/intellij/build/IdeaCommunityBuildTest.kt index 6c4fc5e6ff63..491da2119cd6 100644 --- a/build/tests/testSrc/org/jetbrains/intellij/build/IdeaCommunityBuildTest.kt +++ b/build/tests/testSrc/org/jetbrains/intellij/build/IdeaCommunityBuildTest.kt @@ -3,7 +3,6 @@ package org.jetbrains.intellij.build import com.intellij.openapi.application.PathManager import com.intellij.openapi.util.io.NioFiles -import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot import org.jetbrains.intellij.build.testFramework.createBuildContext import org.jetbrains.intellij.build.testFramework.runTestBuild import org.junit.jupiter.api.Test diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt index 750410f111bf..8f940a43dc71 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt @@ -153,7 +153,7 @@ class BuildTasksImpl(private val context: BuildContext) : BuildTasks { destinationDir = targetDirectory.resolve("jbr"), arch = arch) updateExecutablePermissions(targetDirectory, builder.generateExecutableFilesPatterns(true)) - context.bundledRuntime.checkExecutablePermissions(distribution = targetDirectory, root = "", os = currentOs) + builder.checkExecutablePermissions(targetDirectory, root = "") } else { copyDistFiles(context, targetDirectory) diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntime.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntime.kt index 7b81f1b9b23c..eb6e9a16401a 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntime.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntime.kt @@ -17,7 +17,5 @@ interface BundledRuntime { fun archiveName(prefix: String, arch: JvmArchitecture, os: OsFamily, forceVersionWithUnderscores: Boolean = false): String - fun checkExecutablePermissions(distribution: Path, root: String, os: OsFamily) - fun executableFilesPatterns(os: OsFamily): List } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntimeImpl.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntimeImpl.kt index 676b7cede5c9..05eb8a041f8c 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntimeImpl.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntimeImpl.kt @@ -4,15 +4,10 @@ package org.jetbrains.intellij.build.impl import com.intellij.diagnostic.telemetry.use import com.intellij.openapi.util.SystemInfoRt import com.intellij.openapi.util.io.NioFiles -import com.intellij.util.io.PosixFilePermissionsUtil -import org.apache.commons.compress.archivers.tar.TarArchiveEntry import org.apache.commons.compress.archivers.tar.TarArchiveInputStream -import org.apache.commons.compress.archivers.zip.ZipFile -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.jetbrains.intellij.build.* import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader import org.jetbrains.intellij.build.dependencies.BuildDependenciesExtractOptions -import java.io.BufferedInputStream import java.net.URI import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes @@ -20,7 +15,6 @@ import java.nio.file.attribute.DosFileAttributeView import java.nio.file.attribute.PosixFilePermission.* import java.util.* import java.util.zip.GZIPInputStream -import kotlin.streams.toList class BundledRuntimeImpl(private val context: CompilationContext) : BundledRuntime { companion object { @@ -117,85 +111,6 @@ class BundledRuntimeImpl(private val context: CompilationContext) : BundledRunti return "fastdebug-" } - override fun checkExecutablePermissions(distribution: Path, root: String, os: OsFamily) { - if (os == OsFamily.WINDOWS) { - return - } - - val patterns = executableFilesPatterns(os).map { - FileSystems.getDefault().getPathMatcher("glob:$it") - } - val entries: List - if (Files.isDirectory(distribution)) { - @Suppress("NAME_SHADOWING") val distribution = distribution.resolve(root) - entries = Files.walk(distribution).use { files -> - val expectedExecutables = files.filter { file -> - val relativePath = distribution.relativize(file) - !Files.isDirectory(file) && patterns.any { - it.matches(relativePath) - } - }.toList() - if (expectedExecutables.size < patterns.size) { - context.messages.error("Executable files patterns:\n" + - executableFilesPatterns(os).joinToString(separator = "\n") + - "\nFound files:\n" + - expectedExecutables.joinToString(separator = "\n")) - } - expectedExecutables.stream() - .filter { OWNER_EXECUTE !in Files.getPosixFilePermissions(it) } - .map { distribution.relativize(it).toString() } - .toList() - } - } - else if ("$distribution".endsWith(".tar.gz")) { - entries = TarArchiveInputStream(GzipCompressorInputStream(BufferedInputStream(Files.newInputStream(distribution)))).use { stream -> - val expectedExecutables = mutableListOf() - while (true) { - val entry = (stream.nextEntry ?: break) as TarArchiveEntry - var entryPath = Path.of(entry.name) - if (!root.isEmpty()) { - entryPath = Path.of(root).relativize(entryPath) - } - if (!entry.isDirectory && patterns.any { it.matches(entryPath) }) { - expectedExecutables.add(entry) - } - } - if (expectedExecutables.size < patterns.size) { - context.messages.error("Executable files patterns:\n" + - executableFilesPatterns(os).joinToString(separator = "\n") + - "\nFound files:\n" + - expectedExecutables.joinToString(separator = "\n")) - } - expectedExecutables - .filter { OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(it.mode) } - .map { "${it.name}: mode is 0${Integer.toOctalString(it.mode)}" } - } - } - else { - entries = ZipFile(Files.newByteChannel(distribution)).use { zipFile -> - val expectedExecutables = zipFile.entries.asSequence().filter { entry -> - var entryPath = Path.of(entry.name) - if (!root.isEmpty()) { - entryPath = Path.of(root).relativize(entryPath) - } - !entry.isDirectory && patterns.any { it.matches(entryPath) } - }.toList() - if (expectedExecutables.size < patterns.size) { - context.messages.error("Executable files patterns:\n" + - executableFilesPatterns(os).joinToString(separator = "\n") + - "\nFound files:\n" + - expectedExecutables.joinToString(separator = "\n")) - } - expectedExecutables - .filter { entry -> OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(entry.unixMode) } - .map { "${it.name}: mode is 0${Integer.toOctalString(it.unixMode)}" } - } - } - if (entries.isNotEmpty()) { - context.messages.error("Missing executable permissions in $distribution for:\n" + entries.joinToString(separator = "\n")) - } - } - /** * When changing this list of patterns, also change patch_bin_file in launcher.sh (for remote dev) */ diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt index 80d1fa520056..4499b0923e49 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt @@ -16,7 +16,7 @@ import java.nio.file.* import java.nio.file.attribute.PosixFilePermissions import java.util.concurrent.TimeUnit -class LinuxDistributionBuilder(private val context: BuildContext, +class LinuxDistributionBuilder(override val context: BuildContext, private val customizer: LinuxDistributionCustomizer, private val ideaProperties: Path?) : OsSpecificDistributionBuilder { private val iconPngPath: Path? @@ -76,7 +76,7 @@ class LinuxDistributionBuilder(private val context: BuildContext, val jreDirectoryPath = context.bundledRuntime.extract(getProductPrefix(context), OsFamily.LINUX, arch) val tarGzPath = buildTarGz(jreDirectoryPath, osAndArchSpecificDistPath, suffix) - context.bundledRuntime.checkExecutablePermissions(tarGzPath, rootDirectoryName, OsFamily.LINUX) + checkExecutablePermissions(tarGzPath, rootDirectoryName) if (arch == JvmArchitecture.x64) { buildSnapPackage(jreDirectoryPath, osAndArchSpecificDistPath) @@ -119,11 +119,11 @@ class LinuxDistributionBuilder(private val context: BuildContext, ), convertToUnixLineEndings = true) } - override fun generateExecutableFilesPatterns(includeJre: Boolean): List { + override fun generateExecutableFilesPatterns(includeRuntime: Boolean): List { val patterns = ArrayList() patterns.addAll(listOf("bin/*.sh", "bin/*.py", "bin/fsnotifier*", "bin/remote-dev-server.sh")) patterns.addAll(customizer.extraExecutables) - if (includeJre) { + if (includeRuntime) { patterns.addAll(context.bundledRuntime.executableFilesPatterns(OsFamily.LINUX)) } return patterns diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt index f68f1e7dc111..7fa452699960 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt @@ -36,7 +36,7 @@ import java.util.zip.Deflater import kotlin.io.path.extension import kotlin.io.path.nameWithoutExtension -class MacDistributionBuilder(private val context: BuildContext, +class MacDistributionBuilder(override val context: BuildContext, private val customizer: MacDistributionCustomizer, private val ideaProperties: Path?) : OsSpecificDistributionBuilder { private val targetIcnsFileName: String = "${context.productProperties.baseFileName}.icns" @@ -145,7 +145,7 @@ class MacDistributionBuilder(private val context: BuildContext, macDist = osAndArchSpecificDistPath, runtimeDist = runtimeDist, extraFiles = context.getDistFiles(), - executableFilePatterns = getExecutableFilePatterns(customizer), + executableFilePatterns = generateExecutableFilesPatterns(true), compressionLevel = if (publishArchive) Deflater.DEFAULT_COMPRESSION else Deflater.BEST_SPEED, errorsConsumer = context.messages::warning ) @@ -309,8 +309,21 @@ class MacDistributionBuilder(private val context: BuildContext, } } - override fun generateExecutableFilesPatterns(includeJre: Boolean): List = getExecutableFilePatterns(customizer) -} + override fun generateExecutableFilesPatterns(includeRuntime: Boolean): List { + val executableFilePatterns = mutableListOf( + "bin/*.sh", + "bin/*.py", + "bin/fsnotifier", + "bin/printenv", + "bin/restarter", + "bin/repair", + "MacOS/*" + ) + if (includeRuntime) { + executableFilePatterns += context.bundledRuntime.executableFilesPatterns(OsFamily.MACOS) + } + return executableFilePatterns + customizer.extraExecutables + } private fun createBuildForArchTask(builtinModule: BuiltinModulesFileData?, arch: JvmArchitecture, @@ -406,17 +419,6 @@ private fun propertiesToXml(properties: List, moreProperties: Map = - listOf( - "bin/*.sh", - "bin/*.py", - "bin/fsnotifier", - "bin/printenv", - "bin/restarter", - "bin/repair", - "MacOS/*" - ) + customizer.extraExecutables - internal fun getMacZipRoot(customizer: MacDistributionCustomizer, context: BuildContext): String = "${customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)}/Contents" diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt index 60581853a831..e909621edf2b 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt @@ -17,14 +17,14 @@ import java.nio.file.StandardCopyOption import java.nio.file.attribute.PosixFilePermissions import java.util.concurrent.TimeUnit -internal fun signAndBuildDmg(builtinModule: BuiltinModulesFileData?, - context: BuildContext, - customizer: MacDistributionCustomizer, - macHostProperties: MacHostProperties?, - macZip: Path, - isRuntimeBundled: Boolean, - suffix: String, - notarize: Boolean) { +internal fun MacDistributionBuilder.signAndBuildDmg(builtinModule: BuiltinModulesFileData?, + context: BuildContext, + customizer: MacDistributionCustomizer, + macHostProperties: MacHostProperties?, + macZip: Path, + isRuntimeBundled: Boolean, + suffix: String, + notarize: Boolean) { var javaExePath: String? = null if (isRuntimeBundled) { javaExePath = "../jbr/Contents/Home/bin/java" @@ -63,9 +63,7 @@ internal fun signAndBuildDmg(builtinModule: BuiltinModulesFileData?, require(Files.exists(sitFile)) { "$sitFile wasn't created" } - if (isRuntimeBundled) { - context.bundledRuntime.checkExecutablePermissions(sitFile, zipRoot, OsFamily.MACOS) - } + checkExecutablePermissions(sitFile, zipRoot, isRuntimeBundled) } private fun buildAndSignWithMacBuilderHost(sitFile: Path, diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/OsSpecificDistributionBuilder.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/OsSpecificDistributionBuilder.kt index 7cc056dc6ba9..4c71ec018768 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/OsSpecificDistributionBuilder.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/OsSpecificDistributionBuilder.kt @@ -1,19 +1,120 @@ // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl +import com.intellij.diagnostic.telemetry.useWithScope +import com.intellij.util.io.PosixFilePermissionsUtil +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.archivers.zip.ZipFile +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.JvmArchitecture import org.jetbrains.intellij.build.OsFamily +import org.jetbrains.intellij.build.TraceManager +import java.io.BufferedInputStream +import java.nio.file.FileSystems +import java.nio.file.Files import java.nio.file.Path +import java.nio.file.PathMatcher +import java.nio.file.attribute.PosixFilePermission +import kotlin.io.path.name +import kotlin.streams.toList interface OsSpecificDistributionBuilder { + val context: BuildContext val targetOs: OsFamily fun copyFilesForOsDistribution(targetPath: Path, arch: JvmArchitecture) fun buildArtifacts(osAndArchSpecificDistPath: Path, arch: JvmArchitecture) - fun generateExecutableFilesPatterns(includeJre: Boolean): List = emptyList() + fun generateExecutableFilesPatterns(includeRuntime: Boolean): List = emptyList() fun getArtifactNames(context: BuildContext): List = emptyList() + + fun checkExecutablePermissions(distribution: Path, root: String, includeRuntime: Boolean = true) { + TraceManager.spanBuilder("Permissions check for ${distribution.name}").useWithScope { + val executableFilesPatterns = generateExecutableFilesPatterns(includeRuntime) + val patterns = executableFilesPatterns.map { + FileSystems.getDefault().getPathMatcher("glob:$it") + } + try { + val entries = when { + patterns.isEmpty() -> return + Files.isDirectory(distribution) -> checkDirectory(distribution.resolve(root), patterns) + "$distribution".endsWith(".tar.gz") -> checkTar(distribution, root, patterns) + else -> checkZip(distribution, root, patterns) + } + if (entries.isNotEmpty()) { + context.messages.error("Missing executable permissions in $distribution for:\n" + entries.joinToString(separator = "\n")) + } + } + catch (e: MissingFilesException) { + context.messages.error("Executable files patterns:\n" + + executableFilesPatterns.joinToString(separator = "\n") + + "\nFound files:\n" + + e.found.joinToString(separator = "\n")) + } + } + } + + private fun checkDirectory(distribution: Path, patterns: List): List { + val entries = Files.walk(distribution).use { files -> + val found = files.filter { file -> + val relativePath = distribution.relativize(file) + !Files.isDirectory(file) && patterns.any { + it.matches(relativePath) + } + }.toList() + if (found.size < patterns.size) { + throw MissingFilesException(found) + } + found.stream() + .filter { PosixFilePermission.OWNER_EXECUTE !in Files.getPosixFilePermissions(it) } + .map { distribution.relativize(it).toString() } + .toList() + } + return entries + } + + private fun checkTar(distribution: Path, root: String, patterns: List) = + TarArchiveInputStream(GzipCompressorInputStream(BufferedInputStream(Files.newInputStream(distribution)))).use { stream -> + val found = mutableListOf() + while (true) { + val entry = (stream.nextEntry ?: break) as TarArchiveEntry + var entryPath = Path.of(entry.name) + if (!root.isEmpty()) { + entryPath = Path.of(root).relativize(entryPath) + } + if (!entry.isDirectory && patterns.any { it.matches(entryPath) }) { + found.add(entry) + } + } + if (found.size < patterns.size) { + throw MissingFilesException(found) + } + found + .filter { PosixFilePermission.OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(it.mode) } + .map { "${it.name}: mode is 0${Integer.toOctalString(it.mode)}" } + } + + + private fun checkZip(distribution: Path, root: String, patterns: List) = + ZipFile(Files.newByteChannel(distribution)).use { zipFile -> + val found = zipFile.entries.asSequence().filter { entry -> + var entryPath = Path.of(entry.name) + if (!root.isEmpty()) { + entryPath = Path.of(root).relativize(entryPath) + } + !entry.isDirectory && patterns.any { it.matches(entryPath) } + }.toList() + if (found.size < patterns.size) { + throw MissingFilesException(found) + } + found + .filter { entry -> PosixFilePermission.OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(entry.unixMode) } + .map { "${it.name}: mode is 0${Integer.toOctalString(it.unixMode)}" } + } + + private class MissingFilesException(val found: List) : Exception() } \ No newline at end of file diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt index e0fbac982739..0cc690475602 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt @@ -25,7 +25,7 @@ import java.util.concurrent.ForkJoinTask import java.util.function.BiPredicate internal class WindowsDistributionBuilder( - private val context: BuildContext, + override val context: BuildContext, private val customizer: WindowsDistributionCustomizer, private val ideaProperties: Path?, private val patchedApplicationInfo: String,