IJI-932 .dmg build scripts archive created to be used on macOS build agent instead of macbuilder host

GitOrigin-RevId: a0291e4c96a3a9d5d411e4d5d2f8e34325ffdda2

platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDmgBuilder.kt

@@ -10,15 +10,11 @@ import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
 import org.jetbrains.intellij.build.impl.support.RepairUtilityBuilder
 import org.jetbrains.intellij.build.io.runProcess
-import org.jetbrains.jps.api.GlobalOptions
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardCopyOption
 import java.nio.file.attribute.PosixFilePermissions
-import java.util.*
-import kotlin.io.path.createDirectories
-import kotlin.io.path.exists
-import kotlin.io.path.name
+import kotlin.io.path.*
 import kotlin.time.Duration.Companion.hours
 
 internal val BuildContext.publishSitArchive: Boolean
@@ -53,9 +49,9 @@ internal suspend fun signAndBuildDmg(builder: MacDistributionBuilder,
   if (notarize && useNotaryRestApi) {
     notarize(sitFile, context)
   }
-  val useMacHost = macHostProperties?.host != null &&
-                   macHostProperties.userName != null &&
-                   macHostProperties.password != null
+  val useMacHost = !macHostProperties?.host.isNullOrBlank() &&
+                   !macHostProperties?.userName.isNullOrBlank() &&
+                   !macHostProperties?.password.isNullOrBlank()
   if (useMacHost && (useNotaryXcodeApi || !context.isStepSkipped(BuildOptions.MAC_DMG_STEP))) {
     notarizeAndBuildDmgViaMacBuilderHost(
       sitFile, requireNotNull(macHostProperties),
@@ -138,7 +134,7 @@ private suspend fun buildLocally(sitFile: Path,
   Files.createDirectories(tempDir)
   try {
     if (notarize) notarizeSitLocally(sitFile, tempDir, customizer, context)
-    buildDmgLocally(sitFile, tempDir, targetName, customizer, context)
+    buildDmgLocally(sitFile, tempDir, targetName, customizer, context, staple = notarize)
   }
   finally {
     NioFiles.deleteRecursively(tempDir)
@@ -189,41 +185,81 @@ private suspend fun buildDmgLocally(sitFile: Path,
                                     tempDir: Path,
                                     targetName: String,
                                     customizer: MacDistributionCustomizer,
-                                    context: BuildContext) {
+                                    context: BuildContext,
+                                    staple: Boolean) {
   context.executeStep(spanBuilder("build .dmg locally"), BuildOptions.MAC_DMG_STEP) {
+    context.paths.artifactDir.createDirectories()
+    val entrypoint = prepareDmgBuildScripts(context, customizer, tempDir, staple)
     if (!SystemInfoRt.isMac) {
       it.addEvent(".dmg can be built only on macOS")
+      publishDmgBuildScripts(context, entrypoint, tempDir)
       return@executeStep
     }
-    val exploded = tempDir.resolve("${context.fullBuildNumber}.exploded")
-    if (!exploded.exists()) {
-      exploded.createDirectories()
-      runProcess(listOf("unzip", "-q", "-o", "$sitFile", "-d", "$exploded"))
+    val tmpSit = Files.move(sitFile, tempDir.resolve(sitFile.fileName))
+    runProcess(args = listOf("./${entrypoint.name}"), workingDir = tempDir, inheritOut = true)
+    val dmgFile = tempDir.resolve("$targetName.dmg")
+    check(dmgFile.exists()) {
+      "$dmgFile wasn't created"
     }
-    val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png")
-    Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath),
-               dmgImageCopy)
-    val scriptDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts")
-    Files.copy(scriptDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"),
-               StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
-    NioFiles.setExecutable(tempDir.resolve("makedmg.sh"))
-    Files.copy(scriptDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"),
-               StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
-    val artifactDir = context.paths.artifactDir
-    Files.createDirectories(artifactDir)
-    val mountName = "$targetName-${UUID.randomUUID().toString().substring(1..4)}"
-    val dmgFile = artifactDir.resolve("$targetName.dmg")
-    val cleanUpExploded = true
-    runProcess(
-      args = listOf(
-        "./makedmg.sh", mountName, context.fullBuildNumber,
-        "$dmgFile", "${context.fullBuildNumber}.exploded",
-        "$cleanUpExploded",
-        "${context.signMacOsBinaries}" // isContentSigned
-      ),
-      additionalEnvVariables = mapOf(GlobalOptions.BUILD_DATE_IN_SECONDS to "${context.options.buildDateInSeconds}"),
-      workingDir = tempDir
-    )
+    Files.move(dmgFile, context.paths.artifactDir.resolve(dmgFile.name), StandardCopyOption.REPLACE_EXISTING)
     context.notifyArtifactBuilt(dmgFile)
+    Files.move(tmpSit, sitFile)
+  }
+}
+
+private fun prepareDmgBuildScripts(context: BuildContext,
+                                   customizer: MacDistributionCustomizer,
+                                   tempDir: Path, staple: Boolean): Path {
+  NioFiles.deleteRecursively(tempDir)
+  tempDir.createDirectories()
+  val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png")
+  Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath),
+             dmgImageCopy)
+  val scriptsDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts")
+  Files.copy(scriptsDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"),
+             StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
+  NioFiles.setExecutable(tempDir.resolve("makedmg.sh"))
+  Files.copy(scriptsDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"),
+             StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
+  Files.copy(scriptsDir.resolve("staple.sh"), tempDir.resolve("staple.sh"),
+             StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
+  val entrypoint = tempDir.resolve("build.sh")
+  entrypoint.writeText(
+    scriptsDir.resolve("build-template.sh").readText()
+      .resolveTemplateVar("staple", "$staple")
+      .resolveTemplateVar("appName", context.fullBuildNumber)
+      .resolveTemplateVar("contentSigned", "${context.signMacOsBinaries}")
+      .resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}")
+  )
+  NioFiles.setExecutable(entrypoint)
+  return entrypoint
+}
+
+private fun String.resolveTemplateVar(variable: String, value: String): String {
+  val reference = "%$variable%"
+  check(contains(reference)) {
+    "No $reference is found in:\n'$this'"
+  }
+  return replace(reference, value)
+}
+
+private fun publishDmgBuildScripts(context: BuildContext, entrypoint: Path, tempDir: Path) {
+  if (context.publishSitArchive) {
+    val artifactDir = context.paths.artifactDir.resolve("macos-dmg-build")
+    artifactDir.createDirectories()
+    synchronized("$artifactDir".intern()) {
+      tempDir.listDirectoryEntries().forEach {
+        Files.copy(it, artifactDir.resolve(it.name), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
+      }
+      val message = """
+        To build .dmg(s):
+        1. transfer .sit(s) to macOS host;
+        2. transfer ${artifactDir.name}/ content to the same folder;
+        3. execute ${entrypoint.name} from Terminal. 
+        .dmg(s) will be built in the same folder.
+      """.trimIndent()
+      artifactDir.resolve("README.txt").writeText(message)
+      context.messages.info(message)
+    }
   }
 }

platform/build-scripts/tools/mac/scripts/build-template.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+
+set -euo pipefail
+
+# %templates% should be replaced with default values or overridden via environment variables
+export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-%buildDateInSeconds%}"
+staple="${STAPLE:-%staple%}"
+appName="${APP_NAME:-%appName%}"
+contentSigned="${CONTENT_SIGNED:-%contentSigned%}"
+explodedSitDir='exploded'
+cleanUpExploded='true'
+
+function buildDmg() {
+  sit="$1"
+  dmgName="${sit%".sit"}"
+  dmg="$dmgName.dmg"
+  buildDir="$dmgName"
+  mkdir -p "$buildDir/$explodedSitDir"
+  for buildFile in *; do
+    name="${buildFile##*.}"
+    if [[ -f $buildFile && "$name" != "sit" && "$name" != "dmg" ]]; then
+      cp "$buildFile" "$buildDir"
+    fi
+  done
+  (
+    cd "$buildDir"
+    unzip -q -o "../$sit" -d $explodedSitDir
+    bash ./staple.sh $explodedSitDir "$staple"
+    bash ./makedmg.sh "$dmgName" "$appName" "$dmg" $explodedSitDir $cleanUpExploded "$contentSigned"
+    mv "$dmg" "../$dmg"
+  )
+}
+
+for sit in *.sit; do
+  if [ ! -e "$sit" ]; then
+    echo "No .sit found"
+    exit 0
+  fi
+  buildDmg "$sit"
+done

platform/build-scripts/tools/mac/scripts/makedmg.sh

@@ -3,7 +3,6 @@
 
 #immediately exit script with an error if a command fails
 set -euo pipefail
-set -x
 
 cd "$(dirname "$0")"
 
@@ -15,7 +14,7 @@ CLEANUP_EXPLODED=${5:-"true"}
 CONTENT_SIGNED=${6:-"true"}
 
 function log() {
-  echo "$(date '+[%H:%M:%S]') $*"
+  echo "$(date '+[%H:%M:%S]') [$RESULT_DMG] $*"
 }
 
 function retry() {

platform/build-scripts/tools/mac/scripts/staple.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+
+set -euo pipefail
+exploded=$1
+staple=$2
+if [ "$staple" = 'true' ]; then
+  # only unzipped application can be stapled
+  app="$exploded/$(ls "$exploded")"
+  echo "Stapling $app..."
+  xcrun stapler staple "$app"
+else
+  echo "Stapling of $exploded is skipped"
+fi