diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt index bf30063d5118..411556f6c85f 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt @@ -1,7 +1,6 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl -import com.intellij.openapi.util.SystemInfoRt import com.intellij.openapi.util.io.NioFiles import com.intellij.platform.buildData.productInfo.ProductInfoLaunchData import io.opentelemetry.api.trace.Span @@ -36,6 +35,7 @@ import org.jetbrains.intellij.build.io.copyFileToDir import org.jetbrains.intellij.build.io.moveFileToDir import org.jetbrains.intellij.build.io.runProcess import org.jetbrains.intellij.build.io.substituteTemplatePlaceholders +import org.jetbrains.intellij.build.isWindows import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder import org.jetbrains.intellij.build.telemetry.use import java.nio.file.Files @@ -50,6 +50,8 @@ import kotlin.time.Duration.Companion.minutes private const val NO_RUNTIME_SUFFIX = "-no-jbr" private const val EXECUTABLE_TEMPLATE_NAME = "executable-template.sh" +private val BuildSnapSemaphore = Semaphore(Integer.getInteger("intellij.build.unix.snaps.concurrency", 1)) + class LinuxDistributionBuilder( override val context: BuildContext, private val customizer: LinuxDistributionCustomizer, @@ -63,10 +65,6 @@ class LinuxDistributionBuilder( iconPngPath = if (iconPng.isNullOrEmpty()) null else Path.of(iconPng) } - companion object { - private val BuildSnapSemaphore = Semaphore(Integer.getInteger("intellij.build.unix.snaps.concurrency", 1)) - } - override val targetOs: OsFamily get() = OsFamily.LINUX @@ -75,11 +73,11 @@ class LinuxDistributionBuilder( withContext(Dispatchers.IO) { val distBinDir = targetPath.resolve("bin") val sourceBinDir = context.paths.communityHomeDir.resolve("bin/linux") - addNativeLauncher(distBinDir, targetPath, arch) + addNativeLauncher(distBinDir = distBinDir, targetPath = targetPath, arch = arch, context = context) copyFileToDir(NativeBinaryDownloader.getRestarter(context, OsFamily.LINUX, arch), distBinDir) copyFileToDir(sourceBinDir.resolve("${arch.dirName}/fsnotifier"), distBinDir) - generateBuildTxt(context, targetPath) - copyDistFiles(context, targetPath, OsFamily.LINUX, arch, targetLibcImpl) + generateBuildTxt(context = context, targetDirectory = targetPath) + copyDistFiles(context = context, newDir = targetPath, os = OsFamily.LINUX, arch = arch, libcImpl = targetLibcImpl) //todo converting line separators to unix-style make sense only when building Linux distributions under Windows on a local machine; // for real installers we need to checkout all text files with 'lf' separators anyway @@ -89,13 +87,19 @@ class LinuxDistributionBuilder( Files.copy(iconPngPath, distBinDir.resolve("${context.productProperties.baseFileName}.png"), StandardCopyOption.REPLACE_EXISTING) } writeVmOptions(distBinDir) - generateScripts(distBinDir, arch) + generateScripts(distBinDir = distBinDir, arch = arch, targetLibcImpl = targetLibcImpl, context = context) createFrontendContextForLaunchers(context)?.let { clientContext -> writeLinuxVmOptions(distBinDir, clientContext) - generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, clientContext, targetLibcImpl) + generateLauncherScript( + distBinDir = distBinDir, + arch = arch, + nonCustomizableJvmArgs = ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, + context = clientContext, + targetLibcImpl = targetLibcImpl + ) } generateReadme(targetPath) - generateVersionMarker(targetPath) + generateVersionMarker(targetPath, context) customizer.copyAdditionalFiles(context, targetPath, arch) } } @@ -256,11 +260,11 @@ class LinuxDistributionBuilder( .setAttribute("snapName", snapName) .setAttribute("arch", arch.name) .use { span -> - check(!SystemInfoRt.isWindows) { + check(!isWindows) { ".snap package cannot be built on Windows" } check(Docker.isAvailable) { "Docker is required to build .snap package" } - check(iconPngPath != null) { "'iconPngPath' not set" } + requireNotNull(iconPngPath) { "'iconPngPath' not set" } check(!customizer.snapDescription.isNullOrBlank()) { "'snapDescription' not set" } span.addEvent("prepare files") @@ -272,7 +276,7 @@ class LinuxDistributionBuilder( placeholder = "$", values = listOf( "NAME" to productName, - "ICON" to "\${SNAP}/bin/${context.productProperties.baseFileName}.png", + "ICON" to $$"${SNAP}/bin/$${context.productProperties.baseFileName}.png", "SCRIPT" to snapName, "COMMENT" to (appInfo.motto ?: ""), "WM_CLASS" to getLinuxFrameClass(context) @@ -335,7 +339,7 @@ class LinuxDistributionBuilder( val snapArtifactPath = moveFileToDir(resultDir.resolve(snapArtifactName), context.paths.artifactDir) context.notifyArtifactBuilt(snapArtifactPath) - checkExecutablePermissions(snapArtifactPath, root = "", includeRuntime = true, arch, libc = targetLibcImpl) + checkExecutablePermissions(distribution = snapArtifactPath, root = "", includeRuntime = true, arch = arch, libc = targetLibcImpl) val snapProductInfoJsonPath = snapArtifactPath.resolveProductInfoJsonSibling() copyFile(productJsonFile, snapProductInfoJsonPath) @@ -384,103 +388,104 @@ class LinuxDistributionBuilder( writeProductInfoJson(file, json, context) return file } +} - private fun generateVersionMarker(unixDistPath: Path) { - val targetDir = unixDistPath.resolve("lib") - Files.createDirectories(targetDir) - Files.writeString(targetDir.resolve("build-marker-" + context.fullBuildNumber), context.fullBuildNumber) - } +private fun generateVersionMarker(unixDistPath: Path, context: BuildContext) { + val targetDir = unixDistPath.resolve("lib") + Files.createDirectories(targetDir) + Files.writeString(targetDir.resolve("build-marker-" + context.fullBuildNumber), context.fullBuildNumber) +} - private fun generateScripts(distBinDir: Path, arch: JvmArchitecture) { - Files.createDirectories(distBinDir) - val sourceScriptDir = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts") - Files.newDirectoryStream(sourceScriptDir).use { - for (file in it) { - val fileName = file.fileName.toString() - if (fileName != EXECUTABLE_TEMPLATE_NAME) { - copyScript(file, distBinDir.resolve(fileName), additionalTemplateValues = emptyList(), context) - } +private fun generateScripts(distBinDir: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl, context: BuildContext) { + Files.createDirectories(distBinDir) + + val sourceScriptDir = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts") + Files.newDirectoryStream(sourceScriptDir).use { + for (file in it) { + val fileName = file.fileName.toString() + if (fileName != EXECUTABLE_TEMPLATE_NAME) { + copyScript(file, distBinDir.resolve(fileName), additionalTemplateValues = emptyList(), context) } } - - copyInspectScript(context, distBinDir) - - generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = emptyList(), context, targetLibcImpl) } - private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture) { - val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch) - copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName)) - copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html")) - } + copyInspectScript(context, distBinDir) - private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List, context: BuildContext, targetLibcImpl: LinuxLibcImpl) { - val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") + generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = emptyList(), context, targetLibcImpl) +} - val defaultXmxParameter = try { - Files.readAllLines(vmOptionsPath).firstOrNull { it.startsWith("-Xmx") } - ?: throw IllegalStateException("-Xmx was not found in '$vmOptionsPath'") - } - catch (e: NoSuchFileException) { - throw IllegalStateException("File '$vmOptionsPath' should be already generated at this point", e) - } +private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture, context: BuildContext) { + val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch) + copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName)) + copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html")) +} - val classPathJars = context.bootClassPathJarNames - var classPath = "CLASS_PATH=\"\$IDE_HOME/lib/${classPathJars[0]}\"" - for (i in 1 until classPathJars.size) { - classPath += "\nCLASS_PATH=\"\$CLASS_PATH:\$IDE_HOME/lib/${classPathJars[i]}\"" - } +private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List, context: BuildContext, targetLibcImpl: LinuxLibcImpl) { + val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") - val additionalJvmArguments = mutableListOf() - // https://youtrack.jetbrains.com/issue/IDEA-304440 - // "-Djdk.lang.Process.launchMechanism=vfork" - if (targetLibcImpl == LinuxLibcImpl.MUSL) { - additionalJvmArguments.add("-Djdk.lang.Process.launchMechanism=vfork") - } - additionalJvmArguments.addAll(context.getAdditionalJvmArguments(OsFamily.LINUX, arch, isScript = true) + nonCustomizableJvmArgs) + val defaultXmxParameter = try { + Files.readAllLines(vmOptionsPath).firstOrNull { it.startsWith("-Xmx") } + ?: throw IllegalStateException("-Xmx was not found in '$vmOptionsPath'") + } + catch (e: NoSuchFileException) { + throw IllegalStateException("File '$vmOptionsPath' should be already generated at this point", e) + } - val additionalTemplateValues = listOf( - Pair("vm_options", context.productProperties.baseFileName), - Pair("system_selector", context.systemSelector), - Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")), - Pair("ide_default_xmx", defaultXmxParameter.trim()), - Pair("class_path", classPath), - Pair("main_class_name", context.ideMainClassName), - ) + val classPathJars = context.bootClassPathJarNames + var classPath = $$"CLASS_PATH=\"$IDE_HOME/lib/$${classPathJars[0]}\"" + for (i in 1 until classPathJars.size) { + classPath += $$"\nCLASS_PATH=\"$CLASS_PATH:$IDE_HOME/lib/$${classPathJars[i]}\"" + } - val template = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts/${EXECUTABLE_TEMPLATE_NAME}") - copyScript(template, distBinDir.resolve("${context.productProperties.baseFileName}.sh"), additionalTemplateValues, context) + val additionalJvmArguments = mutableListOf() + // https://youtrack.jetbrains.com/issue/IDEA-304440 + // "-Djdk.lang.Process.launchMechanism=vfork" + if (targetLibcImpl == LinuxLibcImpl.MUSL) { + additionalJvmArguments.add("-Djdk.lang.Process.launchMechanism=vfork") } + additionalJvmArguments.addAll(context.getAdditionalJvmArguments(OsFamily.LINUX, arch, isScript = true) + nonCustomizableJvmArgs) + + val additionalTemplateValues = listOf( + Pair("vm_options", context.productProperties.baseFileName), + Pair("system_selector", context.systemSelector), + Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")), + Pair("ide_default_xmx", defaultXmxParameter.trim()), + Pair("class_path", classPath), + Pair("main_class_name", context.ideMainClassName), + ) + + val template = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts/${EXECUTABLE_TEMPLATE_NAME}") + copyScript(template, distBinDir.resolve("${context.productProperties.baseFileName}.sh"), additionalTemplateValues, context) +} - private fun copyScript(sourceFile: Path, targetFile: Path, additionalTemplateValues: List>, context: BuildContext) { - // Until CR (\r) will be removed from the repository checkout, we need to filter it out from Unix-style scripts - // https://youtrack.jetbrains.com/issue/IJI-526/Force-git-to-use-LF-line-endings-in-working-copy-of-via-gitattri - substituteTemplatePlaceholders( - inputFile = sourceFile, - outputFile = targetFile, - placeholder = "__", - values = listOf( - Pair("product_full", context.applicationInfo.fullProductName), - Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)), - Pair("product_vendor", context.applicationInfo.shortCompanyName), - Pair("product_code", context.applicationInfo.productCode), - Pair("script_name", "${context.productProperties.baseFileName}.sh"), - ) + additionalTemplateValues, - mustUseAllPlaceholders = false, - convertToUnixLineEndings = true, - ) - } +private fun copyScript(sourceFile: Path, targetFile: Path, additionalTemplateValues: List>, context: BuildContext) { + // Until CR (\r) will be removed from the repository checkout, we need to filter it out from Unix-style scripts + // https://youtrack.jetbrains.com/issue/IJI-526/Force-git-to-use-LF-line-endings-in-working-copy-of-via-gitattri + substituteTemplatePlaceholders( + inputFile = sourceFile, + outputFile = targetFile, + placeholder = "__", + values = listOf( + Pair("product_full", context.applicationInfo.fullProductName), + Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)), + Pair("product_vendor", context.applicationInfo.shortCompanyName), + Pair("product_code", context.applicationInfo.productCode), + Pair("script_name", "${context.productProperties.baseFileName}.sh"), + ) + additionalTemplateValues, + mustUseAllPlaceholders = false, + convertToUnixLineEndings = true, + ) +} + +private fun writeLinuxVmOptions(distBinDir: Path, context: BuildContext): Path { + val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") + val vmOptions = VmOptionsGenerator.generate(context).asSequence() + sequenceOf("-Dsun.tools.attach.tmp.only=true", "-Dawt.lock.fair=true") + writeVmOptions(file = vmOptionsPath, vmOptions = vmOptions, separator = "\n") + return vmOptionsPath +} - private fun writeLinuxVmOptions(distBinDir: Path, context: BuildContext): Path { - val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") - @Suppress("SpellCheckingInspection") - val vmOptions = VmOptionsGenerator.generate(context).asSequence() + sequenceOf("-Dsun.tools.attach.tmp.only=true", "-Dawt.lock.fair=true") - VmOptionsGenerator.writeVmOptions(vmOptionsPath, vmOptions, separator = "\n") - return vmOptionsPath - } - private fun suffix(arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl): String { - return suffix(arch) + if (targetLibcImpl == LinuxLibcImpl.MUSL) "-musl" else "" - } +private fun suffix(arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl): String { + return suffix(arch) + if (targetLibcImpl == LinuxLibcImpl.MUSL) "-musl" else "" } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt index 58aba76af734..92e8daf77da2 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt @@ -500,7 +500,7 @@ class MacDistributionBuilder( val executable = context.productProperties.baseFileName val fileVmOptions = VmOptionsGenerator.generate(context).asSequence() + sequenceOf("-Dapple.awt.application.appearance=system") val vmOptionsPath = distBinDir.resolve("${executable}.vmoptions") - VmOptionsGenerator.writeVmOptions(vmOptionsPath, fileVmOptions, separator = "\n") + writeVmOptions(file = vmOptionsPath, vmOptions = fileVmOptions, separator = "\n") return vmOptionsPath } @@ -614,11 +614,11 @@ class MacDistributionBuilder( NioFiles.deleteRecursively(tempDir) NioFiles.createDirectories(tempDir) NioFiles.createDirectories(context.paths.artifactDir) - val entrypoint = prepareDmgBuildScripts(tempDir, staple) + val entrypoint = prepareDmgBuildScripts(tempDir = tempDir, staple = staple, customizer = customizer, context = context) if (!SystemInfoRt.isMac) { it.addEvent(".dmg can be built only on macOS") if (publishSitArchive) { - publishDmgBuildScripts(entrypoint, tempDir) + publishDmgBuildScripts(entrypoint, tempDir, context) } return@executeStep } @@ -638,75 +638,76 @@ class MacDistributionBuilder( NioFiles.deleteRecursively(tempDir) } } +} - private fun prepareDmgBuildScripts(tempDir: Path, staple: Boolean): Path { - NioFiles.deleteRecursively(tempDir) - Files.createDirectories(tempDir) - val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png") - Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath), dmgImageCopy) - val scriptsDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts") - Files.copy(scriptsDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - NioFiles.setExecutable(tempDir.resolve("makedmg.sh")) - Files.copy(scriptsDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - Files.copy(scriptsDir.resolve("staple.sh"), tempDir.resolve("staple.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - val entrypoint = tempDir.resolve("build.sh") - Files.writeString( - entrypoint, - Files.readString(scriptsDir.resolve("build-template.sh")) - .resolveTemplateVar("staple", "$staple") - .resolveTemplateVar("appName", context.fullBuildNumber) - .resolveTemplateVar("contentSigned", "${context.isMacCodeSignEnabled}") - .resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}") - ) - NioFiles.setExecutable(entrypoint) - return entrypoint +private fun prepareDmgBuildScripts(tempDir: Path, staple: Boolean, customizer: MacDistributionCustomizer, context: BuildContext): Path { + NioFiles.deleteRecursively(tempDir) + Files.createDirectories(tempDir) + val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png") + Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath), dmgImageCopy) + val scriptsDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts") + Files.copy(scriptsDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + NioFiles.setExecutable(tempDir.resolve("makedmg.sh")) + Files.copy(scriptsDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + Files.copy(scriptsDir.resolve("staple.sh"), tempDir.resolve("staple.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + val entrypoint = tempDir.resolve("build.sh") + Files.writeString( + entrypoint, + Files.readString(scriptsDir.resolve("build-template.sh")) + .resolveTemplateVar("staple", "$staple") + .resolveTemplateVar("appName", context.fullBuildNumber) + .resolveTemplateVar("contentSigned", "${context.isMacCodeSignEnabled}") + .resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}") + ) + NioFiles.setExecutable(entrypoint) + return entrypoint +} + + +private fun String.resolveTemplateVar(variable: String, value: String): String { + val reference = "%$variable%" + check(contains(reference)) { "No $reference is found in:\n'$this'" } + return replace(reference, value) +} + +private fun publishDmgBuildScripts(entrypoint: Path, tempDir: Path, context: BuildContext) { + val artifactDir = context.paths.artifactDir.resolve("macos-dmg-build") + artifactDir.createDirectories() + synchronized("$artifactDir".intern()) { + tempDir.listDirectoryEntries().forEach { + Files.copy(it, artifactDir.resolve(it.name), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + } + val message = """ + To build .dmg(s): + 1. transfer .sit(s) to macOS host; + 2. transfer ${artifactDir.name}/ content to the same folder; + 3. execute ${entrypoint.name} from Terminal. + .dmg(s) will be built in the same folder. + """.trimIndent() + artifactDir.resolve("README.txt").writeText(message) + context.messages.info(message) + context.notifyArtifactBuilt(artifactDir) + } +} + +private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, arch: JvmArchitecture, context: BuildContext) { + if (context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) { + return } - private fun String.resolveTemplateVar(variable: String, value: String): String { - val reference = "%$variable%" - check(contains(reference)) { "No $reference is found in:\n'$this'" } - return replace(reference, value) + val tempSit = Files.createTempDirectory(context.paths.tempDir, "sit-") + try { + spanBuilder("extracting ${sitFile.name}").use(Dispatchers.IO) { + Decompressor.Zip(sitFile) + .withZipExtensions() + .extract(tempSit) + } + RepairUtilityBuilder.generateManifest(context, tempSit.resolve(sitRoot), OsFamily.MACOS, arch) } - - private fun publishDmgBuildScripts(entrypoint: Path, tempDir: Path) { - val artifactDir = context.paths.artifactDir.resolve("macos-dmg-build") - artifactDir.createDirectories() - synchronized("$artifactDir".intern()) { - tempDir.listDirectoryEntries().forEach { - Files.copy(it, artifactDir.resolve(it.name), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - } - val message = """ - To build .dmg(s): - 1. transfer .sit(s) to macOS host; - 2. transfer ${artifactDir.name}/ content to the same folder; - 3. execute ${entrypoint.name} from Terminal. - .dmg(s) will be built in the same folder. - """.trimIndent() - artifactDir.resolve("README.txt").writeText(message) - context.messages.info(message) - context.notifyArtifactBuilt(artifactDir) - } - } - - private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, arch: JvmArchitecture, context: BuildContext) { - if (context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) { - return - } - - val tempSit = Files.createTempDirectory(context.paths.tempDir, "sit-") - try { - spanBuilder("extracting ${sitFile.name}").use(Dispatchers.IO) { - Decompressor.Zip(sitFile) - .withZipExtensions() - .extract(tempSit) - } - RepairUtilityBuilder.generateManifest(context, tempSit.resolve(sitRoot), OsFamily.MACOS, arch) - } - finally { - withContext(Dispatchers.IO + NonCancellable) { - @OptIn(ExperimentalPathApi::class) - tempSit.deleteRecursively() - } + finally { + withContext(Dispatchers.IO + NonCancellable) { + @OptIn(ExperimentalPathApi::class) + tempSit.deleteRecursively() } } } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt index 36eb99acac94..b2a28419531b 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt @@ -10,59 +10,48 @@ import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.name +private const val DEFAULT_MIN_HEAP = "128m" +private const val DEFAULT_MAX_HEAP = "2048m" + +private val COMMON_VM_OPTIONS: List = listOf( + "-XX:JbrShrinkingGcMaxHeapFreeRatio=40", // IJPL-181469. Used in a couple with AppIdleMemoryCleaner.runGc() + "-XX:ReservedCodeCacheSize=512m", + "-XX:+HeapDumpOnOutOfMemoryError", + "-XX:-OmitStackTraceInFastThrow", + "-XX:CICompilerCount=2", + "-XX:+IgnoreUnrecognizedVMOptions", // allowing the JVM to start even with outdated options stuck in user configs + "-ea", + "-Dsun.io.useCanonCaches=false", + "-Dsun.java2d.metal=true", + "-Djbr.catch.SIGABRT=true", + "-Djdk.http.auth.tunneling.disabledSchemes=\"\"", + "-Djdk.attach.allowAttachSelf=true", + "-Djdk.module.illegalAccess.silent=true", + "-Djdk.nio.maxCachedBufferSize=2097152", + "-Djava.util.zip.use.nio.for.zip.file.access=true", // IJPL-149160 + "-Dkotlinx.coroutines.debug=off", +) + +/** duplicates RepositoryHelper.CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY */ +private const val CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY = "intellij.plugins.custom.built.in.repository.url" + object VmOptionsGenerator { - private const val DEFAULT_MIN_HEAP = "128m" - private const val DEFAULT_MAX_HEAP = "2048m" - - @Suppress("SpellCheckingInspection") - private val COMMON_VM_OPTIONS: List = listOf( - "-XX:JbrShrinkingGcMaxHeapFreeRatio=40", // IJPL-181469. Used in a couple with AppIdleMemoryCleaner.runGc() - "-XX:ReservedCodeCacheSize=512m", - "-XX:+HeapDumpOnOutOfMemoryError", - "-XX:-OmitStackTraceInFastThrow", - "-XX:CICompilerCount=2", - "-XX:+IgnoreUnrecognizedVMOptions", // allowing the JVM to start even with outdated options stuck in user configs - "-ea", - "-Dsun.io.useCanonCaches=false", - "-Dsun.java2d.metal=true", - "-Djbr.catch.SIGABRT=true", - "-Djdk.http.auth.tunneling.disabledSchemes=\"\"", - "-Djdk.attach.allowAttachSelf=true", - "-Djdk.module.illegalAccess.silent=true", - "-Djdk.nio.maxCachedBufferSize=2097152", - "-Djava.util.zip.use.nio.for.zip.file.access=true", // IJPL-149160 - "-Dkotlinx.coroutines.debug=off", - ) - - /** duplicates RepositoryHelper.CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY */ - private const val CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY = "intellij.plugins.custom.built.in.repository.url" - fun generate(context: BuildContext): List = generate( context.applicationInfo.isEAP, context.bundledRuntime, context.productProperties.customJvmMemoryOptions, context.productProperties.additionalVmOptions.let { val customPluginRepositoryUrl = computeCustomPluginRepositoryUrl(context) - if (customPluginRepositoryUrl == null) it - else it + "-D${CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY}=${customPluginRepositoryUrl}" + if (customPluginRepositoryUrl == null) { + it + } + else { + it + "-D${CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY}=${customPluginRepositoryUrl}" + } }, context.productProperties.platformPrefix, ) - private fun computeCustomPluginRepositoryUrl(context: BuildContext): String? { - val artifactsServer = context.proprietaryBuildTools.artifactsServer - if (artifactsServer != null && context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins) { - val builtinPluginsRepoUrl = artifactsServer.urlToArtifact(context, "${context.nonBundledPlugins.name}/plugins.xml") - if (builtinPluginsRepoUrl != null) { - if (builtinPluginsRepoUrl.startsWith("http:")) { - context.messages.logErrorAndThrow("Insecure artifact server: ${builtinPluginsRepoUrl}") - } - return builtinPluginsRepoUrl - } - } - return null - } - internal fun generate( isEAP: Boolean, bundledRuntime: BundledRuntime, @@ -76,20 +65,24 @@ object VmOptionsGenerator { memory.putIfAbsent("-Xms", DEFAULT_MIN_HEAP) memory.putIfAbsent("-Xmx", DEFAULT_MAX_HEAP) // must be the same as [com.intellij.diagnostic.MemorySizeConfigurator.DEFAULT_XMX] for ((k, v) in memory) { - result += k + v + result.add(k + v) } - result += COMMON_VM_OPTIONS + result.addAll(COMMON_VM_OPTIONS) if (isMultiRoutingFileSystemEnabledForProduct(platformPrefix)) { - result += MULTI_ROUTING_FILE_SYSTEM_VMOPTIONS + result.addAll(MULTI_ROUTING_FILE_SYSTEM_VMOPTIONS) } result += additionalVmOptions var index = result.indexOf("-ea") - if (index < 0) index = result.indexOfFirst { it.startsWith("-D") } - if (index < 0) index = result.size + if (index < 0) { + index = result.indexOfFirst { it.startsWith("-D") } + } + if (index < 0) { + index = result.size + } result.addAll( index, @@ -109,8 +102,22 @@ object VmOptionsGenerator { return result } +} - internal fun writeVmOptions(file: Path, vmOptions: Sequence, separator: String) { - Files.writeString(file, vmOptions.joinToString(separator, postfix = separator), StandardCharsets.US_ASCII) +private fun computeCustomPluginRepositoryUrl(context: BuildContext): String? { + val artifactsServer = context.proprietaryBuildTools.artifactsServer + if (artifactsServer != null && context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins) { + val builtinPluginsRepoUrl = artifactsServer.urlToArtifact(context, "${context.nonBundledPlugins.name}/plugins.xml") + if (builtinPluginsRepoUrl != null) { + if (builtinPluginsRepoUrl.startsWith("http:")) { + context.messages.logErrorAndThrow("Insecure artifact server: ${builtinPluginsRepoUrl}") + } + return builtinPluginsRepoUrl + } } + return null +} + +internal fun writeVmOptions(file: Path, vmOptions: Sequence, separator: String) { + Files.writeString(file, vmOptions.joinToString(separator, postfix = separator), StandardCharsets.US_ASCII) } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt index e8424a7fe1d7..964860adfbc6 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt @@ -64,6 +64,8 @@ import kotlin.io.path.name import kotlin.io.path.readBytes import kotlin.io.path.readText +private val CompareDistributionsSemaphore = Semaphore(Integer.getInteger("intellij.build.win.compare.concurrency", 1)) + internal class WindowsDistributionBuilder( override val context: BuildContext, private val customizer: WindowsDistributionCustomizer, @@ -75,10 +77,6 @@ internal class WindowsDistributionBuilder( override val targetLibcImpl: LibcImpl get() = WindowsLibcImpl.DEFAULT - companion object { - private val CompareDistributionsSemaphore = Semaphore(Integer.getInteger("intellij.build.win.compare.concurrency", 1)) - } - override suspend fun copyFilesForOsDistribution(targetPath: Path, arch: JvmArchitecture) { val distBinDir = targetPath.resolve("bin") withContext(Dispatchers.IO) { @@ -91,7 +89,7 @@ internal class WindowsDistributionBuilder( copyFileToDir(NativeBinaryDownloader.getRestarter(context, OsFamily.WINDOWS, arch), distBinDir) generateBuildTxt(context, targetPath) - copyDistFiles(context, targetPath, OsFamily.WINDOWS, arch, WindowsLibcImpl.DEFAULT) + copyDistFiles(context = context, newDir = targetPath, os = OsFamily.WINDOWS, arch = arch, libcImpl = WindowsLibcImpl.DEFAULT) Files.writeString(distBinDir.resolve(PROPERTIES_FILE_NAME), StringUtilRt.convertLineSeparators(ideaProperties!!, "\r\n")) @@ -103,11 +101,11 @@ internal class WindowsDistributionBuilder( writeVmOptions(distBinDir) - buildWinLauncher(targetPath, arch, context, copyLicense = true) + buildWinLauncher(winDistPath = targetPath, arch = arch, context = context, copyLicense = true) createFrontendContextForLaunchers(context)?.let { clientContext -> writeWindowsVmOptions(distBinDir, clientContext) - buildWinLauncher(targetPath, arch, clientContext, copyLicense = false) + buildWinLauncher(winDistPath = targetPath, arch = arch, context = clientContext, copyLicense = false) } customizer.copyAdditionalFiles(context, targetPath, arch) @@ -144,7 +142,7 @@ internal class WindowsDistributionBuilder( } copyFileToDir(vcRtDll, osAndArchSpecificDistPath.resolve("bin")) - val (zipWithJbrPath, exePath) = context.executeStep(spanBuilder("build Windows artefacts"), stepId = BuildOptions.WINDOWS_ARTIFACTS_STEP) { + val (zipWithJbrPath, exePath) = context.executeStep(spanBuilder("build Windows artifacts"), stepId = BuildOptions.WINDOWS_ARTIFACTS_STEP) { var zipWithJbrPath: Path? = null var exePath: Path? = null @@ -153,7 +151,14 @@ internal class WindowsDistributionBuilder( if (customizer.buildZipArchiveWithBundledJre && !context.isStepSkipped(BuildOptions.WINDOWS_ZIP_STEP)) { val zipNameSuffix = suffix(arch) + customizer.zipArchiveWithBundledJreSuffix launch(Dispatchers.IO + CoroutineName("build Windows ${zipNameSuffix}.zip distribution")) { - zipWithJbrPath = createBuildWinZipTask(runtimeDir, zipNameSuffix, osAndArchSpecificDistPath, arch, customizer, context) + zipWithJbrPath = createBuildWinZipTask( + runtimeDir = runtimeDir, + zipNameSuffix = zipNameSuffix, + winDistPath = osAndArchSpecificDistPath, + arch = arch, + customizer = customizer, + context = context, + ) } } @@ -166,11 +171,20 @@ internal class WindowsDistributionBuilder( context.executeStep(spanBuilder("build Windows installer").setAttribute("arch", arch.dirName), BuildOptions.WINDOWS_EXE_INSTALLER_STEP) { val productJsonDir = Files.createTempDirectory(context.paths.tempDir, "win-product-info") - val productJsonFile = writeProductJsonFile(productJsonDir, arch) + val productJsonFile = writeProductJsonFile(productJsonDir, arch, context) val installationDirectories = listOf(context.paths.distAllDir, osAndArchSpecificDistPath, runtimeDir) validateProductJson(jsonText = productJsonFile.readText(), installationDirectories, installationArchives = emptyList(), context) launch(Dispatchers.IO + CoroutineName("build Windows ${arch.dirName} installer")) { - exePath = buildNsisInstaller(osAndArchSpecificDistPath, productJsonFile, additionalDirectoryToInclude = productJsonDir, suffix(arch), customizer, runtimeDir, context, arch) + exePath = buildNsisInstaller( + winDistPath = osAndArchSpecificDistPath, + productInfoJsonFile = productJsonFile, + additionalDirectoryToInclude = productJsonDir, + suffix = suffix(arch), + customizer = customizer, + runtimeDir = runtimeDir, + context = context, + arch = arch, + ) } } @@ -190,7 +204,7 @@ internal class WindowsDistributionBuilder( } } - override suspend fun writeProductInfoFile(targetDir: Path, arch: JvmArchitecture): Path = writeProductJsonFile(targetDir, arch) + override suspend fun writeProductInfoFile(targetDir: Path, arch: JvmArchitecture): Path = writeProductJsonFile(targetDir, arch, context) private fun generateScripts(distBinDir: Path, arch: JvmArchitecture) { val fullName = context.applicationInfo.fullProductName @@ -287,7 +301,7 @@ internal class WindowsDistributionBuilder( } val productJsonDir = context.paths.tempDir.resolve("win.dist.product-info.json.zip${zipNameSuffix}") - val productJsonFile = writeProductJsonFile(productJsonDir, arch, withRuntime = runtimeDir != null) + val productJsonFile = writeProductJsonFile(targetDir = productJsonDir, arch = arch, context = context, withRuntime = runtimeDir != null) dirs.add(productJsonDir) copyFile(productJsonFile, targetFileProductInfoJson) @@ -369,7 +383,7 @@ internal class WindowsDistributionBuilder( private fun computeIcoPath(context: BuildContext): Path { val customizer = context.windowsDistributionCustomizer!! val icoPath = (if (context.applicationInfo.isEAP) customizer.icoPathForEAP else null) ?: customizer.icoPath - require(icoPath != null) { "`WindowsDistributionCustomizer#icoPath` must be set" } + requireNotNull(icoPath) { "`WindowsDistributionCustomizer#icoPath` must be set" } return Path.of(icoPath) } @@ -465,40 +479,40 @@ internal class WindowsDistributionBuilder( } } } - - private fun writeWindowsVmOptions(distBinDir: Path, context: BuildContext): Path { - val vmOptionsFile = distBinDir.resolve("${context.productProperties.baseFileName}64.exe.vmoptions") - val vmOptions = VmOptionsGenerator.generate(context).asSequence() - VmOptionsGenerator.writeVmOptions(vmOptionsFile, vmOptions, separator = "\r\n") - return vmOptionsFile - } - - private suspend fun writeProductJsonFile(targetDir: Path, arch: JvmArchitecture, withRuntime: Boolean = true): Path { - val embeddedFrontendLaunchData = generateEmbeddedFrontendLaunchData(arch, OsFamily.WINDOWS, context) { - "bin/${it.productProperties.baseFileName}64.exe.vmoptions" - } - val qodanaCustomLaunchData = generateQodanaLaunchData(context, arch, OsFamily.WINDOWS) - val json = generateProductInfoJson( - relativePathToBin = "bin", - builtinModules = context.builtinModule, - launch = listOf( - ProductInfoLaunchData.create( - os = OsFamily.WINDOWS.osName, - arch = arch.dirName, - launcherPath = "bin/${context.productProperties.baseFileName}64.exe", - javaExecutablePath = if (withRuntime) "jbr/bin/java.exe" else null, - vmOptionsFilePath = "bin/${context.productProperties.baseFileName}64.exe.vmoptions", - bootClassPathJarNames = context.bootClassPathJarNames, - additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch), - mainClass = context.ideMainClassName, - customCommands = listOfNotNull(embeddedFrontendLaunchData, qodanaCustomLaunchData), - ) - ), - context) - val file = targetDir.resolve(PRODUCT_INFO_FILE_NAME) - writeProductInfoJson(file, json, context) - return file - } - - private fun toDosLineEndings(x: String): String = x.replace("\r", "").replace("\n", "\r\n") } + +private fun writeWindowsVmOptions(distBinDir: Path, context: BuildContext): Path { + val vmOptionsFile = distBinDir.resolve("${context.productProperties.baseFileName}64.exe.vmoptions") + val vmOptions = VmOptionsGenerator.generate(context).asSequence() + writeVmOptions(file = vmOptionsFile, vmOptions = vmOptions, separator = "\r\n") + return vmOptionsFile +} + +private suspend fun writeProductJsonFile(targetDir: Path, arch: JvmArchitecture, context: BuildContext, withRuntime: Boolean = true): Path { + val embeddedFrontendLaunchData = generateEmbeddedFrontendLaunchData(arch, OsFamily.WINDOWS, context) { + "bin/${it.productProperties.baseFileName}64.exe.vmoptions" + } + val qodanaCustomLaunchData = generateQodanaLaunchData(context, arch, OsFamily.WINDOWS) + val json = generateProductInfoJson( + relativePathToBin = "bin", + builtinModules = context.builtinModule, + launch = listOf( + ProductInfoLaunchData.create( + os = OsFamily.WINDOWS.osName, + arch = arch.dirName, + launcherPath = "bin/${context.productProperties.baseFileName}64.exe", + javaExecutablePath = if (withRuntime) "jbr/bin/java.exe" else null, + vmOptionsFilePath = "bin/${context.productProperties.baseFileName}64.exe.vmoptions", + bootClassPathJarNames = context.bootClassPathJarNames, + additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch), + mainClass = context.ideMainClassName, + customCommands = listOfNotNull(embeddedFrontendLaunchData, qodanaCustomLaunchData), + ) + ), + context) + val file = targetDir.resolve(PRODUCT_INFO_FILE_NAME) + writeProductInfoJson(file, json, context) + return file +} + +private fun toDosLineEndings(x: String): String = x.replace("\r", "").replace("\n", "\r\n")