[IJI-2419] musl support: installers

GitOrigin-RevId: a1a73dc34ea0ecaba6b562b71dd3d5e36940cecc
This commit is contained in:
Evgenii Ilichev
2025-04-07 12:25:07 +02:00
committed by intellij-monorepo-bot
parent 0f89cdcc86
commit ef6cd83edb
26 changed files with 288 additions and 179 deletions

View File

@@ -152,8 +152,8 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : BaseIde
override fun getRootDirectoryName(appInfo: ApplicationInfoProperties, buildNumber: String): String = "idea-IC-$buildNumber"
override fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture): Sequence<String> =
super.generateExecutableFilesPatterns(context, includeRuntime, arch)
override fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture, targetLibcImpl: LibcImpl): Sequence<String> =
super.generateExecutableFilesPatterns(context, includeRuntime, arch, targetLibcImpl)
.plus(KotlinBinaries.kotlinCompilerExecutables)
.filterNot { it == "plugins/**/*.sh" }
}

View File

@@ -3,9 +3,11 @@ package org.jetbrains.intellij.build.dependencies
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.intellij.build.downloadFileToCacheLocation
import java.nio.file.Files
import java.nio.file.Path
import java.util.concurrent.TimeUnit
import java.util.logging.Logger
/**
@@ -20,7 +22,7 @@ object JdkDownloader {
}
suspend fun getJdkHome(communityRoot: BuildDependenciesCommunityRoot, jdkBuildNumber: String? = null, variation: String? = null, infoLog: (String) -> Unit): Path {
val isMusl = LinuxLibcImpl.isLinuxMusl
val isMusl = isLinuxMusl()
return getJdkHome(
communityRoot = communityRoot,
os = OS.current,
@@ -150,4 +152,37 @@ object JdkDownloader {
}
}
}
@ApiStatus.Internal
fun isLinuxMusl() : Boolean = isRunOnLinuxMusl
private val isRunOnLinuxMusl: Boolean by lazy {
if (OS.current != OS.LINUX) {
false
}
else {
runCatching {
val process = ProcessBuilder()
.command("ldd", "--version")
.redirectErrorStream(true)
.start()
val output = process.inputStream.bufferedReader().use { it.readText().lowercase() }
val lddOutputContainsMusl = if (!process.waitFor(5, TimeUnit.SECONDS)) {
process.destroyForcibly()
false
}
else {
// Check for musl in output
output.contains("musl")
}
Logger.getLogger(JdkDownloader::class.java.name).info("Linux 'ldd --version': ${output}")
lddOutputContainsMusl
}.getOrElse {
Logger.getLogger(JdkDownloader::class.java.name).info("Failed to detect musl libc: ${it.message}")
false
}
}
}
}

View File

@@ -1,44 +0,0 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies
import org.jetbrains.intellij.build.dependencies.JdkDownloader.OS
import java.util.concurrent.TimeUnit
import java.util.logging.Logger
enum class LinuxLibcImpl {
GLIBC,
MUSL;
companion object {
@JvmField
val ALL: List<LinuxLibcImpl> = listOf(*LinuxLibcImpl.entries.toTypedArray())
val isLinuxMusl: Boolean by lazy {
if (OS.current != OS.LINUX) {
false
} else {
runCatching {
val process = ProcessBuilder()
.command("ldd", "--version")
.redirectErrorStream(true)
.start()
val output = process.inputStream.bufferedReader().use { it.readText().lowercase() }
val lddOutputContainsMusl = if (!process.waitFor(5, TimeUnit.SECONDS)) {
process.destroyForcibly()
false
} else {
// Check for musl in output
output.contains("musl")
}
Logger.getLogger(LinuxLibcImpl::class.java.name).info("Linux 'ldd --version': $output")
lddOutputContainsMusl
}.getOrElse {
Logger.getLogger(LinuxLibcImpl::class.java.name).info("Failed to detect musl libc: ${it.message}")
false
}
}
}
}
}

View File

@@ -119,7 +119,7 @@ interface BuildContext : CompilationContext {
/**
* @return sorted [DistFile] collection
*/
fun getDistFiles(os: OsFamily?, arch: JvmArchitecture?): Collection<DistFile>
fun getDistFiles(os: OsFamily?, arch: JvmArchitecture?, libcImpl: LibcImpl?): Collection<DistFile>
fun patchInspectScript(path: Path)
@@ -252,5 +252,6 @@ data class DistFile(
@JvmField val content: DistFileContent,
@JvmField val relativePath: String,
@JvmField val os: OsFamily? = null,
@JvmField val libcImpl: LibcImpl? = null,
@JvmField val arch: JvmArchitecture? = null,
)

View File

@@ -43,10 +43,10 @@ object CommunityRepositoryModules {
spec.bundlingRestrictions.supportedOs = persistentListOf(OsFamily.MACOS)
},
plugin("intellij.webp") { spec ->
spec.withPlatformBin(OsFamily.WINDOWS, JvmArchitecture.x64, "plugins/webp/lib/libwebp/win", "lib/libwebp/win")
spec.withPlatformBin(OsFamily.MACOS, JvmArchitecture.x64, "plugins/webp/lib/libwebp/mac", "lib/libwebp/mac")
spec.withPlatformBin(OsFamily.MACOS, JvmArchitecture.aarch64, "plugins/webp/lib/libwebp/mac", "lib/libwebp/mac")
spec.withPlatformBin(OsFamily.LINUX, JvmArchitecture.x64, "plugins/webp/lib/libwebp/linux", "lib/libwebp/linux")
spec.withPlatformBin(OsFamily.WINDOWS, JvmArchitecture.x64, WindowsLibcImpl.DEFAULT, "plugins/webp/lib/libwebp/win", "lib/libwebp/win")
spec.withPlatformBin(OsFamily.MACOS, JvmArchitecture.x64, MacLibcImpl.DEFAULT, "plugins/webp/lib/libwebp/mac", "lib/libwebp/mac")
spec.withPlatformBin(OsFamily.MACOS, JvmArchitecture.aarch64, MacLibcImpl.DEFAULT, "plugins/webp/lib/libwebp/mac", "lib/libwebp/mac")
spec.withPlatformBin(OsFamily.LINUX, JvmArchitecture.x64, LinuxLibcImpl.GLIBC, "plugins/webp/lib/libwebp/linux", "lib/libwebp/linux")
},
plugin("intellij.webp") { spec ->
spec.bundlingRestrictions.marketplace = true
@@ -272,10 +272,10 @@ object CommunityRepositoryModules {
}
val supportedFfmpegPresets: PersistentList<SupportedDistribution> = persistentListOf(
SupportedDistribution(os = OsFamily.MACOS, arch = JvmArchitecture.x64),
SupportedDistribution(os = OsFamily.MACOS, arch = JvmArchitecture.aarch64),
SupportedDistribution(os = OsFamily.WINDOWS, arch = JvmArchitecture.x64),
SupportedDistribution(os = OsFamily.LINUX, arch = JvmArchitecture.x64),
SupportedDistribution(os = OsFamily.MACOS, arch = JvmArchitecture.x64, MacLibcImpl.DEFAULT),
SupportedDistribution(os = OsFamily.MACOS, arch = JvmArchitecture.aarch64, MacLibcImpl.DEFAULT),
SupportedDistribution(os = OsFamily.WINDOWS, arch = JvmArchitecture.x64, WindowsLibcImpl.DEFAULT),
SupportedDistribution(os = OsFamily.LINUX, arch = JvmArchitecture.x64, LinuxLibcImpl.GLIBC),
)
private fun createAndroidPluginLayout(
@@ -508,7 +508,7 @@ object CommunityRepositoryModules {
spec.withModuleLibrary("ffmpeg-javacpp", "intellij.android.streaming", "javacpp-$javacppVersion.jar")
// include only required as platform-dependent binaries
for ((supportedOs, supportedArch) in supportedFfmpegPresets) {
for ((supportedOs, supportedArch, supportedLibc) in supportedFfmpegPresets) {
val osName = supportedOs.osName.lowercase(Locale.ENGLISH)
val ffmpegLibraryName = "ffmpeg-$osName-$supportedArch"
val javacppLibraryName = "javacpp-$osName-$supportedArch"
@@ -519,7 +519,7 @@ object CommunityRepositoryModules {
spec.withModuleLibrary(javacppLibraryName, "intellij.android.streaming", "${javacppLibraryName}-$javacppVersion.jar")
}
else {
spec.withGeneratedPlatformResources(supportedOs, supportedArch) { targetDir, context ->
spec.withGeneratedPlatformResources(supportedOs, supportedArch, supportedLibc) { targetDir, context ->
val streamingModule = context.projectModel.project.modules.find { it.name == "intellij.android.streaming" }!!
val ffmpegLibrary = streamingModule.libraryCollection.findLibrary(ffmpegLibraryName)!!
val javacppLibrary = streamingModule.libraryCollection.findLibrary(javacppLibraryName)!!

View File

@@ -0,0 +1,47 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import org.jetbrains.intellij.build.dependencies.JdkDownloader
interface LibcImpl {
companion object {
fun current(osFamily: OsFamily): LibcImpl {
return when (osFamily) {
OsFamily.MACOS -> MacLibcImpl.DEFAULT
OsFamily.WINDOWS -> WindowsLibcImpl.DEFAULT
OsFamily.LINUX -> LinuxLibcImpl.current
}
}
}
}
enum class MacLibcImpl : LibcImpl {
DEFAULT;
companion object {
@JvmField
val ALL: List<MacLibcImpl> = listOf(*MacLibcImpl.entries.toTypedArray())
}
}
enum class WindowsLibcImpl : LibcImpl {
DEFAULT;
companion object {
@JvmField
val ALL: List<WindowsLibcImpl> = listOf(*WindowsLibcImpl.entries.toTypedArray())
}
}
enum class LinuxLibcImpl : LibcImpl {
GLIBC,
MUSL;
companion object {
@JvmField
val ALL: List<LinuxLibcImpl> = listOf(*LinuxLibcImpl.entries.toTypedArray())
val current: LinuxLibcImpl = if (JdkDownloader.isLinuxMusl()) MUSL else GLIBC
}
}

View File

@@ -33,7 +33,7 @@ open class LinuxDistributionCustomizer {
*/
var extraExecutables: PersistentList<String> = persistentListOf()
open fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture): Sequence<String> {
open fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture, targetLibcImpl: LibcImpl): Sequence<String> {
val basePatterns = sequenceOf(
"bin/*.sh",
"plugins/**/*.sh",
@@ -42,7 +42,12 @@ open class LinuxDistributionCustomizer {
"bin/${context.productProperties.baseFileName}",
)
val rtPatterns = if (includeRuntime) {
context.bundledRuntime.executableFilesPatterns(OsFamily.LINUX, context.productProperties.runtimeDistribution)
val distribution = if (targetLibcImpl == LinuxLibcImpl.MUSL) {
JetBrainsRuntimeDistribution.LIGHTWEIGHT
} else {
context.productProperties.runtimeDistribution
}
context.bundledRuntime.executableFilesPatterns(OsFamily.LINUX, distribution)
}
else {
emptySequence()

View File

@@ -33,8 +33,11 @@ import org.jetbrains.intellij.build.BuildPaths
import org.jetbrains.intellij.build.BuildPaths.Companion.COMMUNITY_ROOT
import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxDistributionCustomizer
import org.jetbrains.intellij.build.LinuxLibcImpl
import org.jetbrains.intellij.build.MacDistributionCustomizer
import org.jetbrains.intellij.build.MacLibcImpl
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.PluginBuildDescriptor
import org.jetbrains.intellij.build.ProductProperties
@@ -42,6 +45,7 @@ import org.jetbrains.intellij.build.ProprietaryBuildTools
import org.jetbrains.intellij.build.ScrambleTool
import org.jetbrains.intellij.build.SearchableOptionSetDescriptor
import org.jetbrains.intellij.build.WindowsDistributionCustomizer
import org.jetbrains.intellij.build.WindowsLibcImpl
import org.jetbrains.intellij.build.computeAppClassPath
import org.jetbrains.intellij.build.excludedLibJars
import org.jetbrains.intellij.build.generatePluginClassPath
@@ -186,7 +190,9 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
val binDir = Files.createDirectories(runDir.resolve("bin"))
val oldFiles = Files.newDirectoryStream(binDir).use { it.toCollection(HashSet()) }
val osDistributionBuilder = getOsDistributionBuilder(request.os, context)
val libcImpl = LibcImpl.current(OsFamily.currentOs)
val osDistributionBuilder = getOsDistributionBuilder(request.os, libcImpl, context)
if (osDistributionBuilder != null) {
oldFiles.remove(osDistributionBuilder.writeVmOptions(binDir))
// the file cannot be placed right into the distribution as it throws off home dir detection in `PathManager#getHomeDirFor`
@@ -296,7 +302,7 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
spanBuilder("scramble platform").use{
request.scrambleTool?.scramble(platformLayout.await(), context)
}
copyDistFiles(context = context, newDir = runDir, os = request.os, arch = JvmArchitecture.currentJvmArch)
copyDistFiles(context = context, newDir = runDir, os = request.os, arch = JvmArchitecture.currentJvmArch, libcImpl = LibcImpl.current(OsFamily.currentOs))
}
}.invokeOnCompletion {
// close debug logging to prevent locking of the output directory on Windows

View File

@@ -10,6 +10,7 @@ import kotlinx.coroutines.Job
import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.PluginBuildDescriptor
import org.jetbrains.intellij.build.PluginBundlingRestrictions
@@ -55,7 +56,7 @@ internal suspend fun buildPlugins(
val platform = platformLayout.await()
val pluginEntries = spanBuilder("build plugins").setAttribute(AttributeKey.longKey("count"), plugins.size.toLong()).use {
val targetPlatform = SupportedDistribution(OsFamily.currentOs, JvmArchitecture.currentJvmArch)
val targetPlatform = SupportedDistribution(OsFamily.currentOs, JvmArchitecture.currentJvmArch, LibcImpl.current(OsFamily.currentOs))
buildPlugins(
moduleOutputPatcher = moduleOutputPatcher,
plugins = plugins,

View File

@@ -30,7 +30,9 @@ import org.jetbrains.intellij.build.DistFile
import org.jetbrains.intellij.build.FrontendModuleFilter
import org.jetbrains.intellij.build.JarPackagerDependencyHelper
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxDistributionCustomizer
import org.jetbrains.intellij.build.LinuxLibcImpl
import org.jetbrains.intellij.build.MacDistributionCustomizer
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.PLATFORM_LOADER_JAR
@@ -38,7 +40,6 @@ import org.jetbrains.intellij.build.ProductProperties
import org.jetbrains.intellij.build.ProprietaryBuildTools
import org.jetbrains.intellij.build.WindowsDistributionCustomizer
import org.jetbrains.intellij.build.computeAppInfoXml
import org.jetbrains.intellij.build.dependencies.LinuxLibcImpl
import org.jetbrains.intellij.build.impl.PlatformJarNames.PLATFORM_CORE_NIO_FS
import org.jetbrains.intellij.build.impl.plugins.PluginAutoPublishList
import org.jetbrains.intellij.build.io.runProcess
@@ -146,7 +147,7 @@ class BuildContextImpl internal constructor(
productProperties.productLayout.compatiblePluginsToIgnore =
productProperties.productLayout.compatiblePluginsToIgnore.addAll(options.compatiblePluginsToIgnore)
}
check(options.isInDevelopmentMode || bundledRuntime.prefix == productProperties.runtimeDistribution.artifactPrefix || LinuxLibcImpl.isLinuxMusl) {
check(options.isInDevelopmentMode || bundledRuntime.prefix == productProperties.runtimeDistribution.artifactPrefix || LibcImpl.current(OsFamily.currentOs) == LinuxLibcImpl.MUSL) {
"The runtime type doesn't match the one specified in the product properties: ${bundledRuntime.prefix} != ${productProperties.runtimeDistribution.artifactPrefix}"
}
}
@@ -198,7 +199,7 @@ class BuildContextImpl internal constructor(
override fun addDistFile(file: DistFile) {
Span.current().addEvent("add app resource", Attributes.of(AttributeKey.stringKey("file"), file.toString()))
val existing = distFiles.firstOrNull { it.os == file.os && it.arch == file.arch && it.relativePath == file.relativePath }
val existing = distFiles.firstOrNull { it.os == file.os && it.arch == file.arch && it.libcImpl == file.libcImpl && it.relativePath == file.relativePath }
check(existing == null) {
"$file duplicates $existing"
}
@@ -217,11 +218,12 @@ class BuildContextImpl internal constructor(
}
}
override fun getDistFiles(os: OsFamily?, arch: JvmArchitecture?): Collection<DistFile> {
override fun getDistFiles(os: OsFamily?, arch: JvmArchitecture?, libcImpl: LibcImpl?): Collection<DistFile> {
val result = distFiles.filterTo(mutableListOf()) {
(os == null && arch == null) ||
(os == null && arch == null && libcImpl == null) ||
(os == null || it.os == null || it.os == os) &&
(arch == null || it.arch == null || it.arch == arch)
(arch == null || it.arch == null || it.arch == arch) &&
(libcImpl == null || it.libcImpl == null || it.libcImpl == libcImpl)
}
result.sortWith(compareBy({ it.relativePath }, { it.os }, { it.arch }))
return result

View File

@@ -30,9 +30,13 @@ import org.jetbrains.intellij.build.CompilationTasks
import org.jetbrains.intellij.build.DistFileContent
import org.jetbrains.intellij.build.InMemoryDistFileContent
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxLibcImpl
import org.jetbrains.intellij.build.LocalDistFileContent
import org.jetbrains.intellij.build.MacLibcImpl
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.SoftwareBillOfMaterials
import org.jetbrains.intellij.build.WindowsLibcImpl
import org.jetbrains.intellij.build.buildSearchableOptions
import org.jetbrains.intellij.build.executeStep
import org.jetbrains.intellij.build.impl.maven.MavenArtifactData
@@ -114,32 +118,35 @@ internal class BuildTasksImpl(private val context: BuildContextImpl) : BuildTask
JvmArchitecture.currentJvmArch
}
context.options.targetArch = arch
val targetLibcImpl = LibcImpl.current(OsFamily.currentOs)
layoutShared(context)
if (includeBinAndRuntime) {
val propertiesFile = createIdeaPropertyFile(context)
val builder = getOsDistributionBuilder(currentOs, context, propertiesFile)!!
val builder = getOsDistributionBuilder(currentOs, targetLibcImpl, context, propertiesFile)!!
builder.copyFilesForOsDistribution(targetDirectory, arch)
context.bundledRuntime.extractTo(currentOs, arch, targetDirectory.resolve("jbr"))
updateExecutablePermissions(targetDirectory, builder.generateExecutableFilesMatchers(includeRuntime = true, arch).keys)
builder.checkExecutablePermissions(targetDirectory, root = "", includeRuntime = true, arch)
context.bundledRuntime.extractTo(currentOs, arch, targetLibcImpl, targetDirectory.resolve("jbr"))
updateExecutablePermissions(targetDirectory, builder.generateExecutableFilesMatchers(includeRuntime = true, arch, targetLibcImpl).keys)
builder.checkExecutablePermissions(targetDirectory, root = "", includeRuntime = true, arch, targetLibcImpl)
builder.writeProductInfoFile(targetDirectory, arch)
}
else {
copyDistFiles(context, targetDirectory, currentOs, arch)
copyDistFiles(context, targetDirectory, currentOs, arch, targetLibcImpl)
}
}
}
data class SupportedDistribution(@JvmField val os: OsFamily, @JvmField val arch: JvmArchitecture)
data class SupportedDistribution(@JvmField val os: OsFamily, @JvmField val arch: JvmArchitecture, @JvmField val libcImpl: LibcImpl)
@JvmField
val SUPPORTED_DISTRIBUTIONS: List<SupportedDistribution> = listOf(
SupportedDistribution(OsFamily.WINDOWS, JvmArchitecture.x64),
SupportedDistribution(OsFamily.WINDOWS, JvmArchitecture.aarch64),
SupportedDistribution(OsFamily.MACOS, JvmArchitecture.x64),
SupportedDistribution(OsFamily.MACOS, JvmArchitecture.aarch64),
SupportedDistribution(OsFamily.LINUX, JvmArchitecture.x64),
SupportedDistribution(OsFamily.LINUX, JvmArchitecture.aarch64),
SupportedDistribution(OsFamily.WINDOWS, JvmArchitecture.x64, WindowsLibcImpl.DEFAULT),
SupportedDistribution(OsFamily.WINDOWS, JvmArchitecture.aarch64, WindowsLibcImpl.DEFAULT),
SupportedDistribution(OsFamily.MACOS, JvmArchitecture.x64, MacLibcImpl.DEFAULT),
SupportedDistribution(OsFamily.MACOS, JvmArchitecture.aarch64, MacLibcImpl.DEFAULT),
SupportedDistribution(OsFamily.LINUX, JvmArchitecture.x64, LinuxLibcImpl.GLIBC),
SupportedDistribution(OsFamily.LINUX, JvmArchitecture.aarch64, LinuxLibcImpl.GLIBC),
SupportedDistribution(OsFamily.LINUX, JvmArchitecture.x64, LinuxLibcImpl.MUSL),
SupportedDistribution(OsFamily.LINUX, JvmArchitecture.aarch64, LinuxLibcImpl.MUSL),
)
fun createIdeaPropertyFile(context: BuildContext): CharSequence {
@@ -251,6 +258,7 @@ internal suspend fun updateExecutablePermissions(destinationDir: Path, executabl
class DistributionForOsTaskResult(
@JvmField val builder: OsSpecificDistributionBuilder,
@JvmField val arch: JvmArchitecture,
@JvmField val libc: LibcImpl,
@JvmField val outDir: Path
)
@@ -277,7 +285,7 @@ private suspend fun buildOsSpecificDistributions(context: BuildContext): List<Di
spanBuilder("Adjust executable permissions on common dist").use {
val matchers = SUPPORTED_DISTRIBUTIONS.mapNotNull {
getOsDistributionBuilder(it.os, context)
getOsDistributionBuilder(it.os, it.libcImpl, context)
}.flatMap { builder ->
JvmArchitecture.entries.flatMap { arch ->
builder.generateExecutableFilesMatchers(includeRuntime = true, arch).keys
@@ -287,12 +295,12 @@ private suspend fun buildOsSpecificDistributions(context: BuildContext): List<Di
}
supervisorScope {
SUPPORTED_DISTRIBUTIONS.mapNotNull { (os, arch) ->
SUPPORTED_DISTRIBUTIONS.mapNotNull { (os, arch, libcImpl) ->
if (!context.shouldBuildDistributionForOS(os, arch)) {
return@mapNotNull null
}
val builder = getOsDistributionBuilder(os, context, ideaPropertyFileContent) ?: return@mapNotNull null
val builder = getOsDistributionBuilder(os, libcImpl, context, ideaPropertyFileContent) ?: return@mapNotNull null
val stepId = "${os.osId} ${arch.name}"
if (context.options.buildStepsToSkip.contains(stepId)) {
@@ -302,10 +310,10 @@ private suspend fun buildOsSpecificDistributions(context: BuildContext): List<Di
async(CoroutineName("$stepId build step")) {
spanBuilder(stepId).use {
val osAndArchSpecificDistDirectory = getOsAndArchSpecificDistDirectory(os, arch, context)
val osAndArchSpecificDistDirectory = getOsAndArchSpecificDistDirectory(os, arch, libcImpl, context)
builder.buildArtifacts(osAndArchSpecificDistDirectory, arch)
checkClassFiles(osAndArchSpecificDistDirectory, context, isDistAll = false)
DistributionForOsTaskResult(builder, arch, osAndArchSpecificDistDirectory)
DistributionForOsTaskResult(builder, arch, libcImpl, osAndArchSpecificDistDirectory)
}
}
}
@@ -814,7 +822,7 @@ private suspend fun buildCrossPlatformZip(distResults: List<DistributionForOsTas
val runtimeModuleRepositoryPath = if (context.generateRuntimeModuleRepository) {
spanBuilder("generate runtime repository for cross-platform distribution").use {
generateCrossPlatformRepository(context.paths.distAllDir, distResults.filter { it.arch == JvmArchitecture.x64 }.map { it.outDir }, context)
generateCrossPlatformRepository(context.paths.distAllDir, distResults.filter { it.arch == JvmArchitecture.x64 && it.libc != LinuxLibcImpl.MUSL }.map { it.outDir }, context)
}
}
else {
@@ -824,7 +832,7 @@ private suspend fun buildCrossPlatformZip(distResults: List<DistributionForOsTas
val zipFileName = context.productProperties.getCrossPlatformZipFileName(context.applicationInfo, context.buildNumber)
val targetFile = context.paths.artifactDir.resolve(zipFileName)
val extraFiles = mapOf("dependencies.txt" to copyDependenciesFile(context))
crossPlatformZip(context, distResults, targetFile, executableName, productJson, extraFiles, runtimeModuleRepositoryPath)
crossPlatformZip(context, distResults.filter { it.libc != LinuxLibcImpl.MUSL } , targetFile, executableName, productJson, extraFiles, runtimeModuleRepositoryPath)
validateProductJson(targetFile, pathInArchive = "", context)
@@ -868,7 +876,7 @@ private fun checkPlatformSpecificPluginResources(pluginLayouts: List<PluginLayou
}
}
fun getOsDistributionBuilder(os: OsFamily, context: BuildContext, ideaProperties: CharSequence? = null): OsSpecificDistributionBuilder? = when (os) {
fun getOsDistributionBuilder(os: OsFamily, libcImpl: LibcImpl, context: BuildContext, ideaProperties: CharSequence? = null): OsSpecificDistributionBuilder? = when (os) {
OsFamily.WINDOWS -> context.windowsDistributionCustomizer?.let {
WindowsDistributionBuilder(context, customizer = it, ideaProperties)
}
@@ -876,7 +884,7 @@ fun getOsDistributionBuilder(os: OsFamily, context: BuildContext, ideaProperties
MacDistributionBuilder(context, customizer = it, ideaProperties)
}
OsFamily.LINUX -> context.linuxDistributionCustomizer?.let {
LinuxDistributionBuilder(context, customizer = it, ideaProperties)
LinuxDistributionBuilder(context, customizer = it, ideaProperties, targetLibcImpl = libcImpl as LinuxLibcImpl)
}
}
@@ -904,7 +912,7 @@ private fun crossPlatformZip(
) {
val winX64DistDir = distResults.first { it.builder.targetOs == OsFamily.WINDOWS && it.arch == JvmArchitecture.x64 }.outDir
val macArm64DistDir = distResults.first { it.builder.targetOs == OsFamily.MACOS && it.arch == JvmArchitecture.aarch64 }.outDir
val linuxX64DistDir = distResults.first { it.builder.targetOs == OsFamily.LINUX && it.arch == JvmArchitecture.x64 }.outDir
val linuxX64DistDir = distResults.first { it.builder.targetOs == OsFamily.LINUX && it.arch == JvmArchitecture.x64 && it.libc == LinuxLibcImpl.GLIBC }.outDir
val executablePatterns = distResults.flatMap {
it.builder.generateExecutableFilesMatchers(includeRuntime = false, JvmArchitecture.x64).keys +
@@ -1000,7 +1008,7 @@ private fun crossPlatformZip(
}, entryCustomizer)
}
val distFiles = context.getDistFiles(os = null, arch = null)
val distFiles = context.getDistFiles(os = null, arch = null, libcImpl = null)
for (distFile in distFiles) {
// Linux and Windows: we don't add specific dist dirs for ARM, so, copy dist files explicitly
// macOS: we don't copy dist files to avoid extra copy operation
@@ -1101,8 +1109,8 @@ internal suspend fun collectIncludedPluginModules(enabledPluginModules: Collecti
}
}
internal fun copyDistFiles(context: BuildContext, newDir: Path, os: OsFamily, arch: JvmArchitecture) {
for (item in context.getDistFiles(os, arch)) {
internal fun copyDistFiles(context: BuildContext, newDir: Path, os: OsFamily, arch: JvmArchitecture, libcImpl: LibcImpl) {
for (item in context.getDistFiles(os, arch, libcImpl)) {
val targetFile = newDir.resolve(item.relativePath)
Files.createDirectories(targetFile.parent)
if (item.content is LocalDistFileContent) {

View File

@@ -3,6 +3,7 @@ package org.jetbrains.intellij.build.impl
import org.jetbrains.intellij.build.JetBrainsRuntimeDistribution
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.OsFamily
import java.nio.file.Path
@@ -15,15 +16,15 @@ interface BundledRuntime {
/**
* @return a directory, where only one subdirectory is available: 'jbr', which contains specified JBR
*/
suspend fun extract(os: OsFamily, arch: JvmArchitecture, prefix: String = this.prefix): Path
suspend fun extract(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String = this.prefix): Path
suspend fun extractTo(os: OsFamily, arch: JvmArchitecture, destinationDir: Path)
suspend fun extractTo(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, destinationDir: Path)
suspend fun findArchive(os: OsFamily, arch: JvmArchitecture, prefix: String = this.prefix): Path
suspend fun findArchive(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String = this.prefix): Path
fun downloadUrlFor(os: OsFamily, arch: JvmArchitecture, prefix: String = this.prefix): String
fun downloadUrlFor(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String = this.prefix): String
fun archiveName(os: OsFamily, arch: JvmArchitecture, prefix: String = this.prefix, forceVersionWithUnderscores: Boolean = false): String
fun archiveName(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String = this.prefix, forceVersionWithUnderscores: Boolean = false): String
fun executableFilesPatterns(os: OsFamily, distribution: JetBrainsRuntimeDistribution): Sequence<String>
}

View File

@@ -12,12 +12,13 @@ import org.jetbrains.intellij.build.BuildPaths
import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.JetBrainsRuntimeDistribution
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxLibcImpl
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.ProductProperties
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
import org.jetbrains.intellij.build.dependencies.BuildDependenciesExtractOptions
import org.jetbrains.intellij.build.dependencies.DependenciesProperties
import org.jetbrains.intellij.build.dependencies.LinuxLibcImpl
import org.jetbrains.intellij.build.downloadFileToCacheLocation
import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder
import org.jetbrains.intellij.build.telemetry.use
@@ -52,7 +53,7 @@ class BundledRuntimeImpl(
val bundledRuntimePrefix = options.bundledRuntimePrefix
return when {
// no JCEF distribution for musl, see https://github.com/JetBrains/JetBrainsRuntime/releases
LinuxLibcImpl.isLinuxMusl -> "jbrsdk-"
LibcImpl.current(OsFamily.currentOs) == LinuxLibcImpl.MUSL -> JetBrainsRuntimeDistribution.LIGHTWEIGHT.artifactPrefix
// required as a runtime for debugger tests
System.getProperty("intellij.build.jbr.setupSdk", "false").toBoolean() -> "jbrsdk-"
bundledRuntimePrefix != null -> bundledRuntimePrefix
@@ -75,7 +76,8 @@ class BundledRuntimeImpl(
if (result != null) return result
val os = OsFamily.currentOs
val arch = JvmArchitecture.currentJvmArch
val path = extract(os, arch)
val libc = LibcImpl.current(os)
val path = extract(os, arch, libc)
val home = if (os == OsFamily.MACOS) path.resolve("jbr/Contents/Home") else path.resolve("jbr")
val releaseFile = home.resolve("release")
check(Files.exists(releaseFile)) {
@@ -86,12 +88,13 @@ class BundledRuntimeImpl(
}
}
override suspend fun extract(os: OsFamily, arch: JvmArchitecture, prefix: String): Path {
val isMusl = os == OsFamily.LINUX && LinuxLibcImpl.isLinuxMusl
val targetDir = paths.communityHomeDir.resolve("build/download/${prefix}${build}-${os.jbrArchiveSuffix}-${if (isMusl) "musl-" else ""}$arch")
override suspend fun extract(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String): Path {
val isMusl = os == OsFamily.LINUX && libc == LinuxLibcImpl.MUSL
val effectivePrefix = if (libc == LinuxLibcImpl.MUSL) JetBrainsRuntimeDistribution.LIGHTWEIGHT.artifactPrefix else prefix
val targetDir = paths.communityHomeDir.resolve("build/download/${effectivePrefix}${build}-${os.jbrArchiveSuffix}-${if (isMusl) "musl-" else ""}$arch")
val jbrDir = targetDir.resolve("jbr")
val archive = findArchive(os, arch, prefix)
val archive = findArchive(os, arch, libc, effectivePrefix)
BuildDependenciesDownloader.extractFile(
archive, jbrDir,
paths.communityHomeDirRoot,
@@ -108,22 +111,22 @@ class BundledRuntimeImpl(
return targetDir
}
override suspend fun extractTo(os: OsFamily, arch: JvmArchitecture, destinationDir: Path) {
doExtract(findArchive(os, arch, prefix), destinationDir, os)
override suspend fun extractTo(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, destinationDir: Path) {
doExtract(findArchive(os, arch, libc, prefix), destinationDir, os)
}
override fun downloadUrlFor(os: OsFamily, arch: JvmArchitecture, prefix: String): String =
"https://cache-redirector.jetbrains.com/intellij-jbr/${archiveName(os, arch, prefix)}"
override fun downloadUrlFor(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String): String =
"https://cache-redirector.jetbrains.com/intellij-jbr/${archiveName(os, arch, libc, prefix)}"
override suspend fun findArchive(os: OsFamily, arch: JvmArchitecture, prefix: String): Path =
downloadFileToCacheLocation(downloadUrlFor(os, arch, prefix), paths.communityHomeDirRoot)
override suspend fun findArchive(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String): Path =
downloadFileToCacheLocation(downloadUrlFor(os, arch, libc, prefix), paths.communityHomeDirRoot)
/**
* Update this method together with:
* - [UploadingAndSigning.getMissingJbrs]
* - [org.jetbrains.intellij.build.dependencies.JdkDownloader.getUrl]
*/
override fun archiveName(os: OsFamily, arch: JvmArchitecture, prefix: String, forceVersionWithUnderscores: Boolean): String {
override fun archiveName(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, prefix: String, forceVersionWithUnderscores: Boolean): String {
val split = build.split('b')
if (split.size != 2) {
throw IllegalArgumentException("$build doesn't match '<update>b<build_number>' format (e.g.: 17.0.2b387.1)")
@@ -131,7 +134,7 @@ class BundledRuntimeImpl(
val version = if (forceVersionWithUnderscores) split[0].replace(".", "_") else split[0]
val buildNumber = "b${split[1]}"
val archSuffix = getArchSuffix(arch)
val muslSuffix = if (LinuxLibcImpl.isLinuxMusl) "-musl" else ""
val muslSuffix = if (libc == LinuxLibcImpl.MUSL) "-musl" else ""
return "${prefix}${version}-${os.jbrArchiveSuffix}${muslSuffix}-${archSuffix}-${runtimeBuildPrefix()}${buildNumber}.tar.gz"
}

View File

@@ -35,6 +35,8 @@ import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.DistFile
import org.jetbrains.intellij.build.InMemoryDistFileContent
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxLibcImpl
import org.jetbrains.intellij.build.MAVEN_REPO
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.PLATFORM_LOADER_JAR
@@ -266,7 +268,7 @@ private fun writePluginInfo(
specificClasspath?.let { out.write(it) }
out.close()
context.addDistFile(DistFile(content = InMemoryDistFileContent(byteOut.toByteArray()), relativePath = PLUGIN_CLASSPATH, os = supportedDist.os, arch = supportedDist.arch))
context.addDistFile(DistFile(content = InMemoryDistFileContent(byteOut.toByteArray()), relativePath = PLUGIN_CLASSPATH, os = supportedDist.os, arch = supportedDist.arch, libcImpl = supportedDist.libcImpl))
}
}
@@ -281,11 +283,11 @@ fun validateModuleStructure(platform: PlatformLayout, context: BuildContext) {
private fun getPluginDirs(context: BuildContext, isUpdateFromSources: Boolean): List<Pair<SupportedDistribution, Path>> {
if (isUpdateFromSources) {
return listOf(SupportedDistribution(OsFamily.currentOs, JvmArchitecture.currentJvmArch) to context.paths.distAllDir.resolve(PLUGINS_DIRECTORY))
return listOf(SupportedDistribution(OsFamily.currentOs, JvmArchitecture.currentJvmArch, LibcImpl.current(OsFamily.currentOs)) to context.paths.distAllDir.resolve(PLUGINS_DIRECTORY))
}
else {
return SUPPORTED_DISTRIBUTIONS.map {
it to getOsAndArchSpecificDistDirectory(it.os, it.arch, context).resolve(PLUGINS_DIRECTORY)
it to getOsAndArchSpecificDistDirectory(it.os, it.arch, it.libcImpl, context).resolve(PLUGINS_DIRECTORY)
}
}
}
@@ -833,8 +835,8 @@ private suspend fun patchKeyMapWithAltClickReassignedToMultipleCarets(moduleOutp
moduleOutputPatcher.patchModuleOutput(moduleName, relativePath, text)
}
fun getOsAndArchSpecificDistDirectory(osFamily: OsFamily, arch: JvmArchitecture, context: BuildContext): Path {
return context.paths.buildOutputDir.resolve("dist.${osFamily.distSuffix}.${arch.name}")
fun getOsAndArchSpecificDistDirectory(osFamily: OsFamily, arch: JvmArchitecture, libc: LibcImpl, context: BuildContext): Path {
return context.paths.buildOutputDir.resolve("dist.${osFamily.distSuffix}.${arch.name}${if (libc == LinuxLibcImpl.MUSL) { "-musl" } else {""} }")
}
private suspend fun checkOutputOfPluginModules(

View File

@@ -14,7 +14,9 @@ import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxDistributionCustomizer
import org.jetbrains.intellij.build.LinuxLibcImpl
import org.jetbrains.intellij.build.NativeBinaryDownloader
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.executeStep
@@ -50,7 +52,8 @@ private const val EXECUTABLE_TEMPLATE_NAME = "executable-template.sh"
class LinuxDistributionBuilder(
override val context: BuildContext,
private val customizer: LinuxDistributionCustomizer,
private val ideaProperties: CharSequence?
private val ideaProperties: CharSequence?,
override val targetLibcImpl: LinuxLibcImpl,
) : OsSpecificDistributionBuilder {
private val iconPngPath: Path?
@@ -75,7 +78,7 @@ class LinuxDistributionBuilder(
copyFileToDir(NativeBinaryDownloader.getRestarter(context, OsFamily.LINUX, arch), distBinDir)
copyFileToDir(sourceBinDir.resolve("${arch.dirName}/fsnotifier"), distBinDir)
generateBuildTxt(context, targetPath)
copyDistFiles(context, targetPath, OsFamily.LINUX, arch)
copyDistFiles(context, targetPath, OsFamily.LINUX, arch, targetLibcImpl)
//todo converting line separators to unix-style make sense only when building Linux distributions under Windows on a local machine;
// for real installers we need to checkout all text files with 'lf' separators anyway
@@ -88,7 +91,7 @@ class LinuxDistributionBuilder(
generateScripts(distBinDir, arch)
createFrontendContextForLaunchers(context)?.let { clientContext ->
writeLinuxVmOptions(distBinDir, clientContext)
generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, clientContext)
generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, clientContext, targetLibcImpl)
}
generateReadme(targetPath)
generateVersionMarker(targetPath)
@@ -100,7 +103,8 @@ class LinuxDistributionBuilder(
override suspend fun buildArtifacts(osAndArchSpecificDistPath: Path, arch: JvmArchitecture) {
copyFilesForOsDistribution(osAndArchSpecificDistPath, arch)
setLastModifiedTime(osAndArchSpecificDistPath, context)
val executableFileMatchers = generateExecutableFilesMatchers(includeRuntime = true, arch).keys
val targetLibcImpl = this.targetLibcImpl
val executableFileMatchers = generateExecutableFilesMatchers(includeRuntime = true, arch, targetLibcImpl).keys
updateExecutablePermissions(osAndArchSpecificDistPath, executableFileMatchers)
context.executeStep(spanBuilder("Build Linux artifacts").setAttribute("arch", arch.name), BuildOptions.LINUX_ARTIFACTS_STEP) {
if (customizer.buildArtifactWithoutRuntime) {
@@ -121,18 +125,23 @@ class LinuxDistributionBuilder(
}
}
val runtimeDir = context.bundledRuntime.extract(OsFamily.LINUX, arch)
val runtimeDir = context.bundledRuntime.extract(OsFamily.LINUX, arch, targetLibcImpl)
updateExecutablePermissions(runtimeDir, executableFileMatchers)
val tarGzPath: Path? = context.executeStep(
spanBuilder("Build Linux .tar.gz with bundled Runtime")
.setAttribute("arch", arch.name)
.setAttribute("libcImpl", targetLibcImpl.name)
.setAttribute("runtimeDir", runtimeDir.toString()),
"linux_tar_gz_${arch.name}"
) { _ ->
buildTarGz(arch, runtimeDir, osAndArchSpecificDistPath, suffix(arch))
val suffix = suffix(arch) + if (targetLibcImpl == LinuxLibcImpl.MUSL) "-musl" else ""
buildTarGz(arch, runtimeDir, osAndArchSpecificDistPath, suffix)
}
launch(Dispatchers.IO + CoroutineName("build Snap package")) {
buildSnapPackage(runtimeDir, osAndArchSpecificDistPath, arch)
if (targetLibcImpl != LinuxLibcImpl.MUSL) {
launch(Dispatchers.IO + CoroutineName("build Snap package")) {
buildSnapPackage(runtimeDir, osAndArchSpecificDistPath, arch, targetLibcImpl)
}
}
if (tarGzPath != null ) {
@@ -168,8 +177,8 @@ class LinuxDistributionBuilder(
)
}
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture): Sequence<String> {
return customizer.generateExecutableFilesPatterns(context, includeRuntime, arch)
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence<String> {
return customizer.generateExecutableFilesPatterns(context, includeRuntime, arch, libc)
}
private val rootDirectoryName: String
@@ -197,11 +206,11 @@ class LinuxDistributionBuilder(
spanBuilder("build Linux tar.gz")
.setAttribute("runtimeDir", runtimeDir?.toString() ?: "")
.use(Dispatchers.IO) {
val executableFileMatchers = generateExecutableFilesMatchers(includeRuntime = runtimeDir != null, arch).keys
val executableFileMatchers = generateExecutableFilesMatchers(includeRuntime = runtimeDir != null, arch, this@LinuxDistributionBuilder.targetLibcImpl).keys
tar(tarPath, tarRoot, dirs, executableFileMatchers, context.options.buildDateInSeconds)
validateProductJson(tarPath, tarRoot, context)
context.notifyArtifactBuilt(tarPath)
checkExecutablePermissions(tarPath, rootDirectoryName, includeRuntime = runtimeDir != null, arch)
checkExecutablePermissions(tarPath, rootDirectoryName, includeRuntime = runtimeDir != null, arch, this@LinuxDistributionBuilder.targetLibcImpl)
}
tarPath
}
@@ -222,7 +231,7 @@ class LinuxDistributionBuilder(
return "${snapName}_${snapVersion}_${getSnapArchName(arch)}.snap"
}
private suspend fun buildSnapPackage(runtimeDir: Path, unixDistPath: Path, arch: JvmArchitecture) = BuildSnapSemaphore.withPermit {
private suspend fun buildSnapPackage(runtimeDir: Path, unixDistPath: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl) = BuildSnapSemaphore.withPermit {
if (!context.options.buildUnixSnaps) {
Span.current().addEvent("Linux .snap package build is disabled")
return
@@ -318,7 +327,7 @@ class LinuxDistributionBuilder(
)
val snapArtifactPath = moveFileToDir(resultDir.resolve(snapArtifactName), context.paths.artifactDir)
context.notifyArtifactBuilt(snapArtifactPath)
checkExecutablePermissions(snapArtifactPath, root = "", includeRuntime = true, arch)
checkExecutablePermissions(snapArtifactPath, root = "", includeRuntime = true, arch, libc = targetLibcImpl)
}
}
@@ -385,7 +394,7 @@ class LinuxDistributionBuilder(
copyInspectScript(context, distBinDir)
generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = emptyList(), context)
generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = emptyList(), context, targetLibcImpl)
}
private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture) {
@@ -394,7 +403,7 @@ class LinuxDistributionBuilder(
copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html"))
}
private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List<String>, context: BuildContext) {
private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List<String>, context: BuildContext, targetLibcImpl: LinuxLibcImpl) {
val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions")
val defaultXmxParameter = try {
@@ -411,7 +420,14 @@ class LinuxDistributionBuilder(
classPath += "\nCLASS_PATH=\"\$CLASS_PATH:\$IDE_HOME/lib/${classPathJars[i]}\""
}
val additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.LINUX, arch, isScript = true) + nonCustomizableJvmArgs
val additionalJvmArguments = mutableListOf<String>()
// https://youtrack.jetbrains.com/issue/IDEA-304440
// "-Djdk.lang.Process.launchMechanism=vfork"
if (targetLibcImpl == LinuxLibcImpl.MUSL) {
additionalJvmArguments.add("-Djdk.lang.Process.launchMechanism=vfork")
}
additionalJvmArguments.addAll(context.getAdditionalJvmArguments(OsFamily.LINUX, arch, isScript = true) + nonCustomizableJvmArgs)
val additionalTemplateValues = listOf(
Pair("vm_options", context.productProperties.baseFileName),
Pair("system_selector", context.systemSelector),

View File

@@ -21,8 +21,10 @@ import org.jetbrains.intellij.build.BuildPaths
import org.jetbrains.intellij.build.DistFile
import org.jetbrains.intellij.build.InMemoryDistFileContent
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LocalDistFileContent
import org.jetbrains.intellij.build.MacDistributionCustomizer
import org.jetbrains.intellij.build.MacLibcImpl
import org.jetbrains.intellij.build.NativeBinaryDownloader
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.executeStep
@@ -70,6 +72,9 @@ class MacDistributionBuilder(
override val targetOs: OsFamily
get() = OsFamily.MACOS
override val targetLibcImpl: LibcImpl
get() = MacLibcImpl.DEFAULT
private val targetIcnsFileName: String
get() = "${context.productProperties.baseFileName}.icns"
@@ -137,7 +142,7 @@ class MacDistributionBuilder(
// if copyDistFiles false, it means that we will copy dist files directly without a stage dir
if (copyDistFiles) {
copyDistFiles(context, macDistDir, OsFamily.MACOS, arch)
copyDistFiles(context, macDistDir, OsFamily.MACOS, arch, MacLibcImpl.DEFAULT)
}
customizer.copyAdditionalFiles(context, macDistDir, arch)
@@ -154,7 +159,7 @@ class MacDistributionBuilder(
val executableFileMatchers = generateExecutableFilesMatchers(includeRuntime = true, arch).keys
updateExecutablePermissions(osAndArchSpecificDistPath, executableFileMatchers)
val runtimeDir = context.bundledRuntime.extract(OsFamily.MACOS, arch)
val runtimeDir = context.bundledRuntime.extract(OsFamily.MACOS, arch, MacLibcImpl.DEFAULT)
updateExecutablePermissions(runtimeDir, executableFileMatchers)
if (context.isMacCodeSignEnabled) {
@@ -172,7 +177,7 @@ class MacDistributionBuilder(
val macZipWithoutRuntime = macZip.resolveSibling(macZip.nameWithoutExtension + NO_RUNTIME_SUFFIX + ".zip")
val zipRoot = getMacZipRoot(customizer, context)
val compressionLevel = if (publishSitArchive || publishZipOnly) Deflater.DEFAULT_COMPRESSION else Deflater.BEST_SPEED
val extraFiles = context.getDistFiles(OsFamily.MACOS, arch)
val extraFiles = context.getDistFiles(OsFamily.MACOS, arch, MacLibcImpl.DEFAULT)
val directories = listOf(context.paths.distAllDir, osAndArchSpecificDistPath, runtimeDir)
val builder = this@MacDistributionBuilder
val productJson = generateProductJson(context, arch, withRuntime = true)
@@ -312,7 +317,7 @@ class MacDistributionBuilder(
}
}
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture): Sequence<String> =
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence<String> =
customizer.generateExecutableFilesPatterns(context, includeRuntime, arch)
private suspend fun buildForArch(arch: JvmArchitecture, macZip: Path, macZipWithoutRuntime: Path?) {
@@ -574,7 +579,7 @@ class MacDistributionBuilder(
}
val zipRoot = getMacZipRoot(customizer, context)
checkExecutablePermissions(sitFile, zipRoot, isRuntimeBundled, arch)
checkExecutablePermissions(sitFile, zipRoot, isRuntimeBundled, arch, targetLibcImpl)
if (isRuntimeBundled) {
generateIntegrityManifest(sitFile, zipRoot, arch, context)

View File

@@ -11,6 +11,7 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.dependencies.TeamCityHelper
import org.jetbrains.intellij.build.io.runProcess
@@ -28,6 +29,7 @@ import kotlin.io.path.name
interface OsSpecificDistributionBuilder {
val context: BuildContext
val targetOs: OsFamily
val targetLibcImpl: LibcImpl
suspend fun copyFilesForOsDistribution(targetPath: Path, arch: JvmArchitecture)
@@ -35,11 +37,11 @@ interface OsSpecificDistributionBuilder {
suspend fun writeProductInfoFile(targetDir: Path, arch: JvmArchitecture): Path
fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture): Sequence<String> = emptySequence()
fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence<String> = emptySequence()
fun generateExecutableFilesMatchers(includeRuntime: Boolean, arch: JvmArchitecture): Map<PathMatcher, String> {
fun generateExecutableFilesMatchers(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl = targetLibcImpl): Map<PathMatcher, String> {
val fileSystem = FileSystems.getDefault()
return generateExecutableFilesPatterns(includeRuntime, arch)
return generateExecutableFilesPatterns(includeRuntime, arch, libc)
.distinct()
.map(FileUtil::toSystemIndependentName)
.associateBy {
@@ -47,9 +49,9 @@ interface OsSpecificDistributionBuilder {
}
}
suspend fun checkExecutablePermissions(distribution: Path, root: String, includeRuntime: Boolean = true, arch: JvmArchitecture) {
suspend fun checkExecutablePermissions(distribution: Path, root: String, includeRuntime: Boolean = true, arch: JvmArchitecture, libc: LibcImpl) {
spanBuilder("Permissions check for ${distribution.name}").use {
val patterns = generateExecutableFilesMatchers(includeRuntime, arch)
val patterns = generateExecutableFilesMatchers(includeRuntime, arch, libc)
val matchedFiles = when {
patterns.isEmpty() -> return@use
SystemInfoRt.isWindows && distribution.isDirectory() -> return@use

View File

@@ -18,6 +18,7 @@ import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.CustomAssetDescriptor
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LazySource
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.PluginBundlingRestrictions
import org.jetbrains.intellij.build.io.copyDir
@@ -257,8 +258,8 @@ class PluginLayout(val mainModule: String, @Internal @JvmField val auto: Boolean
}
}
fun withGeneratedPlatformResources(os: OsFamily, arch: JvmArchitecture, generator: ResourceGenerator) {
val key = SupportedDistribution(os, arch)
fun withGeneratedPlatformResources(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, generator: ResourceGenerator) {
val key = SupportedDistribution(os, arch, libc)
val newValue = layout.platformResourceGenerators[key]?.let { it + generator } ?: persistentListOf(generator)
layout.platformResourceGenerators += key to newValue
}
@@ -327,8 +328,8 @@ class PluginLayout(val mainModule: String, @Internal @JvmField val auto: Boolean
}
}
fun withPlatformBin(os: OsFamily, arch: JvmArchitecture, binPathRelativeToCommunity: String, outputPath: String, skipIfDoesntExist: Boolean = false) {
withGeneratedPlatformResources(os, arch) { targetDir, context ->
fun withPlatformBin(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl, binPathRelativeToCommunity: String, outputPath: String, skipIfDoesntExist: Boolean = false) {
withGeneratedPlatformResources(os, arch, libc) { targetDir, context ->
copyBinaryResource(binPathRelativeToCommunity, outputPath, skipIfDoesntExist, targetDir, context)
}
}

View File

@@ -28,10 +28,12 @@ import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.BuildPaths.Companion.ULTIMATE_HOME
import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.CompilationTasks
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxLibcImpl
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.TestingOptions
import org.jetbrains.intellij.build.TestingTasks
import org.jetbrains.intellij.build.causal.CausalProfilingOptions
import org.jetbrains.intellij.build.dependencies.LinuxLibcImpl
import org.jetbrains.intellij.build.dependencies.TeamCityHelper
import org.jetbrains.intellij.build.io.ZipEntryProcessorResult
import org.jetbrains.intellij.build.io.readZipFile
@@ -1132,7 +1134,7 @@ internal class TestingTasksImpl(context: CompilationContext, private val options
}
args += "-classpath"
val classpathForTests = if (LinuxLibcImpl.isLinuxMusl) {
val classpathForTests = if (LibcImpl.current(OsFamily.currentOs) == LinuxLibcImpl.MUSL) {
prepareMuslClassPath(classpath)
} else {
classpath

View File

@@ -24,9 +24,11 @@ import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.FileSet
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.NativeBinaryDownloader
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.WindowsDistributionCustomizer
import org.jetbrains.intellij.build.WindowsLibcImpl
import org.jetbrains.intellij.build.executeStep
import org.jetbrains.intellij.build.impl.OsSpecificDistributionBuilder.Companion.suffix
import org.jetbrains.intellij.build.impl.client.createFrontendContextForLaunchers
@@ -71,6 +73,9 @@ internal class WindowsDistributionBuilder(
override val targetOs: OsFamily
get() = OsFamily.WINDOWS
override val targetLibcImpl: LibcImpl
get() = WindowsLibcImpl.DEFAULT
companion object {
private val CompareDistributionsSemaphore = Semaphore(Integer.getInteger("intellij.build.win.compare.concurrency", 1))
}
@@ -87,7 +92,7 @@ internal class WindowsDistributionBuilder(
copyFileToDir(NativeBinaryDownloader.getRestarter(context, OsFamily.WINDOWS, arch), distBinDir)
generateBuildTxt(context, targetPath)
copyDistFiles(context, targetPath, OsFamily.WINDOWS, arch)
copyDistFiles(context, targetPath, OsFamily.WINDOWS, arch, WindowsLibcImpl.DEFAULT)
Files.writeString(distBinDir.resolve(PROPERTIES_FILE_NAME), StringUtilRt.convertLineSeparators(ideaProperties!!, "\r\n"))
@@ -129,7 +134,7 @@ internal class WindowsDistributionBuilder(
override suspend fun buildArtifacts(osAndArchSpecificDistPath: Path, arch: JvmArchitecture) {
copyFilesForOsDistribution(osAndArchSpecificDistPath, arch)
val runtimeDir = context.bundledRuntime.extract(OsFamily.WINDOWS, arch)
val runtimeDir = context.bundledRuntime.extract(OsFamily.WINDOWS, arch, WindowsLibcImpl.DEFAULT)
@Suppress("SpellCheckingInspection")
val vcRtDll = runtimeDir.resolve("jbr/bin/msvcp140.dll")

View File

@@ -23,7 +23,7 @@ internal fun getIncludedModules(entries: Sequence<DistributionFileEntry>): Seque
}
internal fun buildJarContentReport(contentReport: ContentReport, zipFileWriter: ZipFileWriter, buildPaths: BuildPaths, context: BuildContext) {
zipFileWriter.uncompressedData("platform.yaml", buildPlatformContentReport(contentReport, buildPaths, context.getDistFiles(os = null, arch = null)))
zipFileWriter.uncompressedData("platform.yaml", buildPlatformContentReport(contentReport, buildPaths, context.getDistFiles(os = null, arch = null, libcImpl = null)))
zipFileWriter.uncompressedData("bundled-plugins.yaml", buildPluginContentReport(contentReport.bundledPlugins, buildPaths))
zipFileWriter.uncompressedData("non-bundled-plugins.yaml", buildPluginContentReport(contentReport.nonBundledPlugins, buildPaths))
}
@@ -135,6 +135,7 @@ private fun buildPlatformContentReport(contentReport: ContentReport, buildPaths:
writer.writeStringField("name", item.relativePath)
item.os?.let { writer.writeStringField("os", it.osId) }
item.arch?.let { writer.writeStringField("arch", it.dirName) }
item.libcImpl?.let { writer.writeStringField("libc", it.toString()) }
writer.writeEndObject()
}

View File

@@ -38,7 +38,7 @@ internal suspend fun generateRuntimeModuleRepository(entries: Sequence<Distribut
val repositoryEntries = ArrayList<RuntimeModuleRepositoryEntry>()
val osSpecificDistPaths = listOf(null to context.paths.distAllDir) +
SUPPORTED_DISTRIBUTIONS.map { it to getOsAndArchSpecificDistDirectory(osFamily = it.os, arch = it.arch, context = context) }
SUPPORTED_DISTRIBUTIONS.map { it to getOsAndArchSpecificDistDirectory(osFamily = it.os, arch = it.arch, libc = it.libcImpl, context = context) }
for (entry in entries) {
val (distribution, rootPath) = osSpecificDistPaths.find { entry.path.startsWith(it.second) } ?: continue
@@ -56,7 +56,7 @@ internal suspend fun generateRuntimeModuleRepository(entries: Sequence<Distribut
}
else {
for (distribution in SUPPORTED_DISTRIBUTIONS) {
val targetDirectory = getOsAndArchSpecificDistDirectory(osFamily = distribution.os, arch = distribution.arch, context = context)
val targetDirectory = getOsAndArchSpecificDistDirectory(osFamily = distribution.os, arch = distribution.arch, libc = distribution.libcImpl, context = context)
val actualEntries = repositoryEntries.filter { it.distribution == null || it.distribution == distribution }
generateRepositoryForDistribution(
targetDirectory = targetDirectory,

View File

@@ -24,6 +24,7 @@ import org.apache.maven.model.io.xpp3.MavenXpp3Reader
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LibraryLicense
import org.jetbrains.intellij.build.LibraryUpstream
import org.jetbrains.intellij.build.OsFamily
@@ -242,7 +243,7 @@ class SoftwareBillOfMaterialsImpl(
}
document.documentDescribes.add(rootPackage)
val runtimePackage = if (distribution.builder.isRuntimeBundled(it.path)) {
document.runtimePackage(distribution.builder.targetOs, distribution.arch)
document.runtimePackage(distribution.builder.targetOs, distribution.arch, libc = distribution.libc)
}
else {
null
@@ -261,17 +262,17 @@ class SoftwareBillOfMaterialsImpl(
* Used until external document reference for Runtime is supplied,
* then should be replaced with [addRuntimeDocumentRef]
*/
private suspend fun SpdxDocument.runtimePackage(os: OsFamily, arch: JvmArchitecture): SpdxPackage {
val checksums = Checksums(context.bundledRuntime.findArchive(os = os, arch = arch))
private suspend fun SpdxDocument.runtimePackage(os: OsFamily, arch: JvmArchitecture, libc: LibcImpl): SpdxPackage {
val checksums = Checksums(context.bundledRuntime.findArchive(os = os, arch = arch, libc = libc))
val version = context.bundledRuntime.build
val runtimeArchivePackage = spdxPackageForFile(
this,
name = context.bundledRuntime.archiveName(os = os, arch = arch),
name = context.bundledRuntime.archiveName(os = os, arch = arch, libc = libc),
sha256sum = checksums.sha256sum,
sha1sum = checksums.sha1sum
) {
setVersionInfo(version)
setDownloadLocation(context.bundledRuntime.downloadUrlFor(os = os, arch = arch))
setDownloadLocation(context.bundledRuntime.downloadUrlFor(os = os, arch = arch, libc = libc))
}
claimContainedFiles(spdxPackage = runtimeArchivePackage, document = this, license = license)
/**
@@ -358,8 +359,8 @@ class SoftwareBillOfMaterialsImpl(
private suspend fun generateFromContentReport(): List<Path> {
return SUPPORTED_DISTRIBUTIONS
.filter { (os, arch) -> context.shouldBuildDistributionForOS(os, arch) }
.map { (os, arch) ->
val distributionDir = getOsAndArchSpecificDistDirectory(osFamily = os, arch = arch, context = context)
.map { (os, arch, libc ) ->
val distributionDir = getOsAndArchSpecificDistDirectory(osFamily = os, arch = arch, libc = libc, context = context)
val name = context.productProperties.getBaseArtifactName(context) + "-${distributionDir.name}"
val document = spdxDocument(name)
val rootPackage = spdxPackage(document, name) {
@@ -371,7 +372,7 @@ class SoftwareBillOfMaterialsImpl(
generate(
document = document,
rootPackage = rootPackage,
runtimePackage = document.runtimePackage(os, arch),
runtimePackage = document.runtimePackage(os, arch, libc),
distributionDir = distributionDir,
// distributions weren't built
claimContainedFiles = false,

View File

@@ -79,7 +79,7 @@ internal class RuntimeModuleRepositoryChecker private constructor(
}
return SUPPORTED_DISTRIBUTIONS
.mapNotNull { distribution ->
val osSpecificDistPath = getOsAndArchSpecificDistDirectory(distribution.os, distribution.arch, context)
val osSpecificDistPath = getOsAndArchSpecificDistDirectory(distribution.os, distribution.arch, distribution.libcImpl, context)
if (osSpecificDistPath.resolve(MODULE_DESCRIPTORS_JAR_PATH).exists()) {
{ RuntimeModuleRepositoryChecker(commonDistPath, osSpecificDistPath, context) }
}

View File

@@ -12,6 +12,7 @@ data class FileEntry(
@JvmField val os: String? = null,
@JvmField val arch: String? = null,
@JvmField val libc: String? = null,
/**
* The list of included in the file project libraries.

View File

@@ -21,21 +21,29 @@ class BundledRuntimeTest {
for (prefix in JetBrainsRuntimeDistribution.ALL) {
for (os in OsFamily.ALL) {
for (arch in JvmArchitecture.ALL) {
val home = try {
bundledRuntime.extract(os, arch, prefix.artifactPrefix)
}
catch (t: Throwable) {
throw IllegalStateException(
"Unable to download JBR for os $os, arch $arch, type $prefix (classifier '${prefix.classifier}', artifact prefix '${prefix.artifactPrefix}': ${t.message}",
t)
val libcList = when (os) {
OsFamily.LINUX -> LinuxLibcImpl.ALL
OsFamily.WINDOWS -> WindowsLibcImpl.ALL
OsFamily.MACOS -> MacLibcImpl.ALL
}
for (libc in libcList) {
val home = try {
bundledRuntime.extract(os, arch, libc, prefix.artifactPrefix)
}
catch (t: Throwable) {
throw IllegalStateException(
"Unable to download JBR for os $os, arch $arch, type $prefix (classifier '${prefix.classifier}', artifact prefix '${prefix.artifactPrefix}': ${t.message}",
t
)
}
// do not cache, takes too much space. Do not delete current jbr for this os and arch
if (currentJbr.startsWith(home)) {
spottedCurrentJbrInDownloadVariants = true
}
else {
FileUtil.delete(home)
// do not cache, takes too much space. Do not delete current jbr for this os and arch
if (currentJbr.startsWith(home)) {
spottedCurrentJbrInDownloadVariants = true
}
else {
FileUtil.delete(home)
}
}
}
}