CWM-6546 IJI-772 checking OS-specific executable patterns, not only Runtime

GitOrigin-RevId: 0d2f0bdd952b1a1d0b8eeef9deb422b82e2de812

build/tests/testSrc/org/jetbrains/intellij/build/IdeaCommunityBuildTest.kt

@@ -3,7 +3,6 @@ package org.jetbrains.intellij.build
 
 import com.intellij.openapi.application.PathManager
 import com.intellij.openapi.util.io.NioFiles
-import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
 import org.jetbrains.intellij.build.testFramework.createBuildContext
 import org.jetbrains.intellij.build.testFramework.runTestBuild
 import org.junit.jupiter.api.Test

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt

@@ -153,7 +153,7 @@ class BuildTasksImpl(private val context: BuildContext) : BuildTasks {
                                        destinationDir = targetDirectory.resolve("jbr"),
                                        arch = arch)
       updateExecutablePermissions(targetDirectory, builder.generateExecutableFilesPatterns(true))
-      context.bundledRuntime.checkExecutablePermissions(distribution = targetDirectory, root = "", os = currentOs)
+      builder.checkExecutablePermissions(targetDirectory, root = "")
     }
     else {
       copyDistFiles(context, targetDirectory)

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntime.kt

@@ -17,7 +17,5 @@ interface BundledRuntime {
 
   fun archiveName(prefix: String, arch: JvmArchitecture, os: OsFamily, forceVersionWithUnderscores: Boolean = false): String
 
-  fun checkExecutablePermissions(distribution: Path, root: String, os: OsFamily)
-
   fun executableFilesPatterns(os: OsFamily): List<String>
 }

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BundledRuntimeImpl.kt

@@ -4,15 +4,10 @@ package org.jetbrains.intellij.build.impl
 import com.intellij.diagnostic.telemetry.use
 import com.intellij.openapi.util.SystemInfoRt
 import com.intellij.openapi.util.io.NioFiles
-import com.intellij.util.io.PosixFilePermissionsUtil
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
-import org.apache.commons.compress.archivers.zip.ZipFile
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesExtractOptions
-import java.io.BufferedInputStream
 import java.net.URI
 import java.nio.file.*
 import java.nio.file.attribute.BasicFileAttributes
@@ -20,7 +15,6 @@ import java.nio.file.attribute.DosFileAttributeView
 import java.nio.file.attribute.PosixFilePermission.*
 import java.util.*
 import java.util.zip.GZIPInputStream
-import kotlin.streams.toList
 
 class BundledRuntimeImpl(private val context: CompilationContext) : BundledRuntime {
   companion object {
@@ -117,85 +111,6 @@ class BundledRuntimeImpl(private val context: CompilationContext) : BundledRunti
     return "fastdebug-"
   }
 
-  override fun checkExecutablePermissions(distribution: Path, root: String, os: OsFamily) {
-    if (os == OsFamily.WINDOWS) {
-      return
-    }
-
-    val patterns = executableFilesPatterns(os).map {
-      FileSystems.getDefault().getPathMatcher("glob:$it")
-    }
-    val entries: List<String>
-    if (Files.isDirectory(distribution)) {
-      @Suppress("NAME_SHADOWING") val distribution = distribution.resolve(root)
-      entries = Files.walk(distribution).use { files ->
-        val expectedExecutables = files.filter { file ->
-          val relativePath = distribution.relativize(file)
-          !Files.isDirectory(file) && patterns.any {
-            it.matches(relativePath)
-          }
-        }.toList()
-        if (expectedExecutables.size < patterns.size) {
-          context.messages.error("Executable files patterns:\n" +
-                                 executableFilesPatterns(os).joinToString(separator = "\n") +
-                                 "\nFound files:\n" +
-                                 expectedExecutables.joinToString(separator = "\n"))
-        }
-        expectedExecutables.stream()
-          .filter { OWNER_EXECUTE !in Files.getPosixFilePermissions(it) }
-          .map { distribution.relativize(it).toString() }
-          .toList()
-      }
-    }
-    else if ("$distribution".endsWith(".tar.gz")) {
-      entries = TarArchiveInputStream(GzipCompressorInputStream(BufferedInputStream(Files.newInputStream(distribution)))).use { stream ->
-        val expectedExecutables = mutableListOf<TarArchiveEntry>()
-          while (true) {
-            val entry = (stream.nextEntry ?: break) as TarArchiveEntry
-            var entryPath = Path.of(entry.name)
-            if (!root.isEmpty()) {
-              entryPath = Path.of(root).relativize(entryPath)
-            }
-            if (!entry.isDirectory && patterns.any { it.matches(entryPath) }) {
-              expectedExecutables.add(entry)
-            }
-          }
-          if (expectedExecutables.size < patterns.size) {
-            context.messages.error("Executable files patterns:\n" +
-                                   executableFilesPatterns(os).joinToString(separator = "\n") +
-                                   "\nFound files:\n" +
-                                   expectedExecutables.joinToString(separator = "\n"))
-          }
-          expectedExecutables
-            .filter { OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(it.mode) }
-            .map { "${it.name}: mode is 0${Integer.toOctalString(it.mode)}" }
-        }
-    }
-    else {
-      entries = ZipFile(Files.newByteChannel(distribution)).use { zipFile ->
-        val expectedExecutables = zipFile.entries.asSequence().filter { entry ->
-          var entryPath = Path.of(entry.name)
-          if (!root.isEmpty()) {
-            entryPath = Path.of(root).relativize(entryPath)
-          }
-          !entry.isDirectory && patterns.any { it.matches(entryPath) }
-        }.toList()
-        if (expectedExecutables.size < patterns.size) {
-          context.messages.error("Executable files patterns:\n" +
-                                 executableFilesPatterns(os).joinToString(separator = "\n") +
-                                 "\nFound files:\n" +
-                                 expectedExecutables.joinToString(separator = "\n"))
-        }
-        expectedExecutables
-          .filter { entry -> OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(entry.unixMode) }
-          .map { "${it.name}: mode is 0${Integer.toOctalString(it.unixMode)}" }
-      }
-    }
-    if (entries.isNotEmpty()) {
-      context.messages.error("Missing executable permissions in $distribution for:\n" + entries.joinToString(separator = "\n"))
-    }
-  }
-
   /**
    * When changing this list of patterns, also change patch_bin_file in launcher.sh (for remote dev)
    */

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt

@@ -16,7 +16,7 @@ import java.nio.file.*
 import java.nio.file.attribute.PosixFilePermissions
 import java.util.concurrent.TimeUnit
 
-class LinuxDistributionBuilder(private val context: BuildContext,
+class LinuxDistributionBuilder(override val context: BuildContext,
                                private val customizer: LinuxDistributionCustomizer,
                                private val ideaProperties: Path?) : OsSpecificDistributionBuilder {
   private val iconPngPath: Path?
@@ -76,7 +76,7 @@ class LinuxDistributionBuilder(private val context: BuildContext,
 
       val jreDirectoryPath = context.bundledRuntime.extract(getProductPrefix(context), OsFamily.LINUX, arch)
       val tarGzPath = buildTarGz(jreDirectoryPath, osAndArchSpecificDistPath, suffix)
-      context.bundledRuntime.checkExecutablePermissions(tarGzPath, rootDirectoryName, OsFamily.LINUX)
+      checkExecutablePermissions(tarGzPath, rootDirectoryName)
 
       if (arch == JvmArchitecture.x64) {
         buildSnapPackage(jreDirectoryPath, osAndArchSpecificDistPath)
@@ -119,11 +119,11 @@ class LinuxDistributionBuilder(private val context: BuildContext,
     ), convertToUnixLineEndings = true)
   }
 
-  override fun generateExecutableFilesPatterns(includeJre: Boolean): List<String> {
+  override fun generateExecutableFilesPatterns(includeRuntime: Boolean): List<String> {
     val patterns = ArrayList<String>()
     patterns.addAll(listOf("bin/*.sh", "bin/*.py", "bin/fsnotifier*", "bin/remote-dev-server.sh"))
     patterns.addAll(customizer.extraExecutables)
-    if (includeJre) {
+    if (includeRuntime) {
       patterns.addAll(context.bundledRuntime.executableFilesPatterns(OsFamily.LINUX))
     }
     return patterns

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt

@@ -36,7 +36,7 @@ import java.util.zip.Deflater
 import kotlin.io.path.extension
 import kotlin.io.path.nameWithoutExtension
 
-class MacDistributionBuilder(private val context: BuildContext,
+class MacDistributionBuilder(override val context: BuildContext,
                              private val customizer: MacDistributionCustomizer,
                              private val ideaProperties: Path?) : OsSpecificDistributionBuilder {
   private val targetIcnsFileName: String = "${context.productProperties.baseFileName}.icns"
@@ -145,7 +145,7 @@ class MacDistributionBuilder(private val context: BuildContext,
         macDist = osAndArchSpecificDistPath,
         runtimeDist = runtimeDist,
         extraFiles = context.getDistFiles(),
-        executableFilePatterns = getExecutableFilePatterns(customizer),
+        executableFilePatterns = generateExecutableFilesPatterns(true),
         compressionLevel = if (publishArchive) Deflater.DEFAULT_COMPRESSION else Deflater.BEST_SPEED,
         errorsConsumer = context.messages::warning
       )
@@ -309,8 +309,21 @@ class MacDistributionBuilder(private val context: BuildContext,
     }
   }
 
-  override fun generateExecutableFilesPatterns(includeJre: Boolean): List<String> = getExecutableFilePatterns(customizer)
-}
+  override fun generateExecutableFilesPatterns(includeRuntime: Boolean): List<String> {
+    val executableFilePatterns = mutableListOf(
+      "bin/*.sh",
+      "bin/*.py",
+      "bin/fsnotifier",
+      "bin/printenv",
+      "bin/restarter",
+      "bin/repair",
+      "MacOS/*"
+    )
+    if (includeRuntime) {
+      executableFilePatterns += context.bundledRuntime.executableFilesPatterns(OsFamily.MACOS)
+    }
+    return executableFilePatterns + customizer.extraExecutables
+  }
 
   private fun createBuildForArchTask(builtinModule: BuiltinModulesFileData?,
                                      arch: JvmArchitecture,
@@ -406,17 +419,6 @@ private fun propertiesToXml(properties: List<String>, moreProperties: Map<String
   return buff.toString().trim()
 }
 
-private fun getExecutableFilePatterns(customizer: MacDistributionCustomizer): List<String> =
-  listOf(
-    "bin/*.sh",
-    "bin/*.py",
-    "bin/fsnotifier",
-    "bin/printenv",
-    "bin/restarter",
-    "bin/repair",
-    "MacOS/*"
-  ) + customizer.extraExecutables
-
 internal fun getMacZipRoot(customizer: MacDistributionCustomizer, context: BuildContext): String =
   "${customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)}/Contents"
 

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt

@@ -17,14 +17,14 @@ import java.nio.file.StandardCopyOption
 import java.nio.file.attribute.PosixFilePermissions
 import java.util.concurrent.TimeUnit
 
-internal fun signAndBuildDmg(builtinModule: BuiltinModulesFileData?,
-                             context: BuildContext,
-                             customizer: MacDistributionCustomizer,
-                             macHostProperties: MacHostProperties?,
-                             macZip: Path,
-                             isRuntimeBundled: Boolean,
-                             suffix: String,
-                             notarize: Boolean) {
+internal fun MacDistributionBuilder.signAndBuildDmg(builtinModule: BuiltinModulesFileData?,
+                                                    context: BuildContext,
+                                                    customizer: MacDistributionCustomizer,
+                                                    macHostProperties: MacHostProperties?,
+                                                    macZip: Path,
+                                                    isRuntimeBundled: Boolean,
+                                                    suffix: String,
+                                                    notarize: Boolean) {
   var javaExePath: String? = null
   if (isRuntimeBundled) {
     javaExePath = "../jbr/Contents/Home/bin/java"
@@ -63,9 +63,7 @@ internal fun signAndBuildDmg(builtinModule: BuiltinModulesFileData?,
   require(Files.exists(sitFile)) {
     "$sitFile wasn't created"
   }
-  if (isRuntimeBundled) {
-    context.bundledRuntime.checkExecutablePermissions(sitFile, zipRoot, OsFamily.MACOS)
-  }
+  checkExecutablePermissions(sitFile, zipRoot, isRuntimeBundled)
 }
 
 private fun buildAndSignWithMacBuilderHost(sitFile: Path,

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/OsSpecificDistributionBuilder.kt

@@ -1,19 +1,120 @@
 // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.intellij.build.impl
 
+import com.intellij.diagnostic.telemetry.useWithScope
+import com.intellij.util.io.PosixFilePermissionsUtil
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.archivers.zip.ZipFile
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
 import org.jetbrains.intellij.build.BuildContext
 import org.jetbrains.intellij.build.JvmArchitecture
 import org.jetbrains.intellij.build.OsFamily
+import org.jetbrains.intellij.build.TraceManager
+import java.io.BufferedInputStream
+import java.nio.file.FileSystems
+import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.PathMatcher
+import java.nio.file.attribute.PosixFilePermission
+import kotlin.io.path.name
+import kotlin.streams.toList
 
 interface OsSpecificDistributionBuilder {
+  val context: BuildContext
   val targetOs: OsFamily
 
   fun copyFilesForOsDistribution(targetPath: Path, arch: JvmArchitecture)
 
   fun buildArtifacts(osAndArchSpecificDistPath: Path, arch: JvmArchitecture)
 
-  fun generateExecutableFilesPatterns(includeJre: Boolean): List<String> = emptyList()
+  fun generateExecutableFilesPatterns(includeRuntime: Boolean): List<String> = emptyList()
 
   fun getArtifactNames(context: BuildContext): List<String> = emptyList()
+
+  fun checkExecutablePermissions(distribution: Path, root: String, includeRuntime: Boolean = true) {
+    TraceManager.spanBuilder("Permissions check for ${distribution.name}").useWithScope {
+      val executableFilesPatterns = generateExecutableFilesPatterns(includeRuntime)
+      val patterns = executableFilesPatterns.map {
+        FileSystems.getDefault().getPathMatcher("glob:$it")
+      }
+      try {
+        val entries = when {
+          patterns.isEmpty() -> return
+          Files.isDirectory(distribution) -> checkDirectory(distribution.resolve(root), patterns)
+          "$distribution".endsWith(".tar.gz") -> checkTar(distribution, root, patterns)
+          else -> checkZip(distribution, root, patterns)
+        }
+        if (entries.isNotEmpty()) {
+          context.messages.error("Missing executable permissions in $distribution for:\n" + entries.joinToString(separator = "\n"))
+        }
+      }
+      catch (e: MissingFilesException) {
+        context.messages.error("Executable files patterns:\n" +
+                               executableFilesPatterns.joinToString(separator = "\n") +
+                               "\nFound files:\n" +
+                               e.found.joinToString(separator = "\n"))
+      }
+    }
+  }
+
+  private fun checkDirectory(distribution: Path, patterns: List<PathMatcher>): List<String> {
+    val entries = Files.walk(distribution).use { files ->
+      val found = files.filter { file ->
+        val relativePath = distribution.relativize(file)
+        !Files.isDirectory(file) && patterns.any {
+          it.matches(relativePath)
+        }
+      }.toList()
+      if (found.size < patterns.size) {
+        throw MissingFilesException(found)
+      }
+      found.stream()
+        .filter { PosixFilePermission.OWNER_EXECUTE !in Files.getPosixFilePermissions(it) }
+        .map { distribution.relativize(it).toString() }
+        .toList()
+    }
+    return entries
+  }
+
+  private fun checkTar(distribution: Path, root: String, patterns: List<PathMatcher>) =
+    TarArchiveInputStream(GzipCompressorInputStream(BufferedInputStream(Files.newInputStream(distribution)))).use { stream ->
+      val found = mutableListOf<TarArchiveEntry>()
+      while (true) {
+        val entry = (stream.nextEntry ?: break) as TarArchiveEntry
+        var entryPath = Path.of(entry.name)
+        if (!root.isEmpty()) {
+          entryPath = Path.of(root).relativize(entryPath)
+        }
+        if (!entry.isDirectory && patterns.any { it.matches(entryPath) }) {
+          found.add(entry)
+        }
+      }
+      if (found.size < patterns.size) {
+        throw MissingFilesException(found)
+      }
+      found
+        .filter { PosixFilePermission.OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(it.mode) }
+        .map { "${it.name}: mode is 0${Integer.toOctalString(it.mode)}" }
+    }
+
+
+  private fun checkZip(distribution: Path, root: String, patterns: List<PathMatcher>) =
+    ZipFile(Files.newByteChannel(distribution)).use { zipFile ->
+      val found = zipFile.entries.asSequence().filter { entry ->
+        var entryPath = Path.of(entry.name)
+        if (!root.isEmpty()) {
+          entryPath = Path.of(root).relativize(entryPath)
+        }
+        !entry.isDirectory && patterns.any { it.matches(entryPath) }
+      }.toList()
+      if (found.size < patterns.size) {
+        throw MissingFilesException(found)
+      }
+      found
+        .filter { entry -> PosixFilePermission.OWNER_EXECUTE !in PosixFilePermissionsUtil.fromUnixMode(entry.unixMode) }
+        .map { "${it.name}: mode is 0${Integer.toOctalString(it.unixMode)}" }
+    }
+
+  private class MissingFilesException(val found: List<Any>) : Exception()
 }

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt

@@ -25,7 +25,7 @@ import java.util.concurrent.ForkJoinTask
 import java.util.function.BiPredicate
 
 internal class WindowsDistributionBuilder(
-  private val context: BuildContext,
+  override val context: BuildContext,
   private val customizer: WindowsDistributionCustomizer,
   private val ideaProperties: Path?,
   private val patchedApplicationInfo: String,