[build scripts] refactoring signing tools to support macOS launch requirements (IJPL-187902 prerequisite)

+ cleanup (minor optimization; typos; formatting)

GitOrigin-RevId: 9093f1d2c6dc60b2bfe9ff65e2fbea069a16aec8
This commit is contained in:
Roman Shevchenko
2025-06-24 21:13:58 +00:00
committed by intellij-monorepo-bot
parent 4280df9a08
commit be5adcdb35
9 changed files with 244 additions and 478 deletions
@@ -1,8 +1,10 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
/**
* Full name of a keychain identity (Applications > Utilities > Keychain Access).
* More info in the `SIGNING IDENTITIES` section of the `man codesign` Terminal command output.
*/
data class MacOsCodesignIdentity(val value: String)
data class MacOsCodesignIdentity(val teamName: String, val teamID: String) {
val certificateID: String = "Developer ID Application: ${teamName} (${teamID})"
}
@@ -1,4 +1,4 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import io.opentelemetry.api.common.AttributeKey
@@ -10,7 +10,8 @@ import org.jetbrains.intellij.build.fus.FeatureUsageStatisticsProperties
import java.nio.file.Path
/**
* Describes proprietary tools which are used to build the product. Pass the instance of this class to [org.jetbrains.intellij.build.impl.BuildContextImpl.Companion.createContext] method.
* Describes proprietary tools which are used to build the product.
* Pass the instance of this class to [org.jetbrains.intellij.build.impl.BuildContextImpl.Companion.createContext] method.
*/
data class ProprietaryBuildTools(
/**
@@ -24,18 +25,12 @@ data class ProprietaryBuildTools(
val scrambleTool: ScrambleTool?,
/**
* Describes address and credentials of Mac machine which is used to sign and build *.dmg installer for macOS. If `null` only *.sit
* archive will be built.
*/
val macOsCodesignIdentity: MacOsCodesignIdentity?,
/**
* Describes a server that can be used to download built artifacts to install plugins into IDE
* Describes a server that can be used to download built artifacts to install plugins into IDE.
*/
val artifactsServer: ArtifactsServer?,
/**
* Properties required to bundle a default version of feature usage statistics allowlist into the IDE
* Properties required to bundle a default version of feature usage statistics allowlist into the IDE.
*/
val featureUsageStatisticsProperties: List<FeatureUsageStatisticsProperties>?,
@@ -47,38 +42,30 @@ data class ProprietaryBuildTools(
) {
companion object {
internal val DUMMY_SIGN_TOOL: SignTool = object : SignTool {
override val signNativeFileMode: SignNativeFileMode
get() = SignNativeFileMode.DISABLED
override val macOsCodesignIdentity: MacOsCodesignIdentity? get() = null
override val signNativeFileMode: SignNativeFileMode get() = SignNativeFileMode.DISABLED
override suspend fun signFiles(files: List<Path>, context: BuildContext?, options: PersistentMap<String, String>) {
Span.current().addEvent(
"files won't be signed", Attributes.of(
Span.current().addEvent("files won't be signed", Attributes.of(
AttributeKey.stringArrayKey("files"), files.map(Path::toString),
AttributeKey.stringKey("reason"), "sign tool isn't defined",
)
)
))
}
override suspend fun signFilesWithGpg(files: List<Path>, context: BuildContext) {
signFiles(files, context, persistentMapOf())
}
override suspend fun getPresignedLibraryFile(path: String, libName: String, libVersion: String, context: BuildContext): Path? {
override suspend fun getPresignedLibraryFile(path: String, libName: String, libVersion: String, context: BuildContext): Path {
error("Must be not called if signNativeFileMode equals to $signNativeFileMode")
}
override suspend fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path? {
return null
}
override suspend fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path? = null
}
val DUMMY: ProprietaryBuildTools = ProprietaryBuildTools(
signTool = DUMMY_SIGN_TOOL,
scrambleTool = null,
macOsCodesignIdentity = null,
artifactsServer = null,
featureUsageStatisticsProperties = null,
licenseServerHost = null
DUMMY_SIGN_TOOL, scrambleTool = null, artifactsServer = null, featureUsageStatisticsProperties = null, licenseServerHost = null
)
}
}
@@ -1,4 +1,4 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import kotlinx.collections.immutable.PersistentMap
@@ -12,6 +12,12 @@ interface SignTool {
const val LIB_VERSION_OPTION_NAME: String = "libVersion"
}
/**
* Describes address and credentials of the macOS machine which should be used to sign and build DMG images.
* When `null`, only SIT archives will be built.
*/
val macOsCodesignIdentity: MacOsCodesignIdentity?
val signNativeFileMode: SignNativeFileMode
suspend fun signFiles(files: List<Path>, context: BuildContext?, options: PersistentMap<String, String>)
@@ -16,7 +16,6 @@ import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.selects.onTimeout
@@ -212,13 +211,7 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
val platformLayoutAwaited = platformLayout.await()
val (platformDistributionEntries, classPath) = spanBuilder("layout platform").use {
layoutPlatform(
runDir = runDir,
platformLayout = platformLayoutAwaited,
searchableOptionSet = searchableOptionSet,
context = context,
moduleOutputPatcher = moduleOutputPatcher,
)
layoutPlatform(runDir, platformLayoutAwaited, searchableOptionSet, context, moduleOutputPatcher)
}
if (request.writeCoreClasspath) {
@@ -244,16 +237,7 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
}
val pluginDistributionEntriesDeferred = async(CoroutineName("build plugins")) {
buildPlugins(
request = request,
context = context,
runDir = runDir,
platformLayout = platformLayout,
artifactTask = artifactTask,
searchableOptionSet = searchableOptionSet,
buildPlatformJob = platformDistributionEntriesDeferred,
moduleOutputPatcher = moduleOutputPatcher,
)
buildPlugins(request, context, runDir, platformLayout, artifactTask, searchableOptionSet, platformDistributionEntriesDeferred, moduleOutputPatcher)
}
launch {
@@ -266,7 +250,7 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
val out = DataOutputStream(byteOut)
val pluginCount = pluginEntries.size + (additionalEntries?.size ?: 0)
platformDistributionEntriesDeferred.join()
writePluginClassPathHeader(out = out, isJarOnly = !request.isUnpackedDist, pluginCount = pluginCount, moduleOutputPatcher = moduleOutputPatcher, context = context)
writePluginClassPathHeader(out, isJarOnly = !request.isUnpackedDist, pluginCount, moduleOutputPatcher, context)
out.write(mainData)
additionalData?.let { out.write(it) }
out.close()
@@ -276,7 +260,9 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
if (context.generateRuntimeModuleRepository) {
launch {
val allDistributionEntries = platformDistributionEntriesDeferred.await().asSequence() + pluginDistributionEntriesDeferred.await().first.asSequence().flatMap { it.second }
val allDistributionEntries =
platformDistributionEntriesDeferred.await().asSequence() +
pluginDistributionEntriesDeferred.await().first.asSequence().flatMap { it.second }
spanBuilder("generate runtime repository").use(Dispatchers.IO) {
generateRuntimeModuleRepositoryForDevBuild(allDistributionEntries, runDir, context)
}
@@ -284,12 +270,7 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
}
launch {
computeIdeFingerprint(
platformDistributionEntriesDeferred = platformDistributionEntriesDeferred,
pluginDistributionEntriesDeferred = pluginDistributionEntriesDeferred,
runDir = runDir,
homePath = request.projectDir,
)
computeIdeFingerprint(platformDistributionEntriesDeferred, pluginDistributionEntriesDeferred, runDir, homePath = request.projectDir)
}
launch(Dispatchers.IO) {
@@ -301,7 +282,7 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
spanBuilder("scramble platform").use{
request.scrambleTool?.scramble(platformLayout.await(), context)
}
copyDistFiles(context = context, newDir = runDir, os = request.os, arch = JvmArchitecture.currentJvmArch, libcImpl = LibcImpl.current(OsFamily.currentOs))
copyDistFiles(context, runDir, request.os, JvmArchitecture.currentJvmArch, LibcImpl.current(OsFamily.currentOs))
}
}.invokeOnCompletion {
// close debug logging to prevent locking of the output directory on Windows
@@ -370,7 +351,7 @@ private suspend fun compileIfNeeded(context: BuildContext) {
private suspend fun collectModulesToCompileForDistribution(context: BuildContext): MutableSet<String> {
val result = java.util.LinkedHashSet<String>()
val productLayout = context.productProperties.productLayout
collectIncludedPluginModules(enabledPluginModules = context.getBundledPluginModules(), result = result, context = context)
collectIncludedPluginModules(context.getBundledPluginModules(), result, context)
collectPlatformModules(to = result)
result.addAll(productLayout.productApiModules)
result.addAll(productLayout.productImplementationModules)
@@ -450,8 +431,9 @@ private suspend fun createBuildContext(
): BuildContext {
return coroutineScope {
val buildOptionsTemplate = request.buildOptionsTemplate
val useCompiledClassesFromProjectOutput = buildOptionsTemplate == null ||
(buildOptionsTemplate.useCompiledClassesFromProjectOutput && buildOptionsTemplate.unpackCompiledClassesArchives)
val useCompiledClassesFromProjectOutput =
buildOptionsTemplate == null ||
(buildOptionsTemplate.useCompiledClassesFromProjectOutput && buildOptionsTemplate.unpackCompiledClassesArchives)
val classOutDir = if (useCompiledClassesFromProjectOutput) {
request.productionClassOutput.parent
}
@@ -543,25 +525,11 @@ private suspend fun createBuildContext(
}
BuildContextImpl(
compilationContext = compilationContext,
productProperties = productProperties.await(),
windowsDistributionCustomizer = WindowsDistributionCustomizer(),
linuxDistributionCustomizer = LinuxDistributionCustomizer(),
macDistributionCustomizer = MacDistributionCustomizer(),
compilationContext, productProperties.await(), WindowsDistributionCustomizer(), LinuxDistributionCustomizer(), MacDistributionCustomizer(),
proprietaryBuildTools = if (request.scrambleTool == null) ProprietaryBuildTools.DUMMY else ProprietaryBuildTools(
ProprietaryBuildTools.DUMMY_SIGN_TOOL, request.scrambleTool, featureUsageStatisticsProperties = null, artifactsServer = null, licenseServerHost = null
),
jarCacheManager = jarCacheManager,
proprietaryBuildTools = if (request.scrambleTool == null) {
ProprietaryBuildTools.DUMMY
}
else {
ProprietaryBuildTools(
scrambleTool = request.scrambleTool,
signTool = ProprietaryBuildTools.DUMMY_SIGN_TOOL,
macOsCodesignIdentity = null,
featureUsageStatisticsProperties = null,
artifactsServer = null,
licenseServerHost = null,
)
},
)
}
}
@@ -616,14 +584,7 @@ private suspend fun layoutPlatform(
context: BuildContext,
moduleOutputPatcher: ModuleOutputPatcher,
): Pair<List<DistributionFileEntry>, Set<Path>> {
val entries = layoutPlatformDistribution(
moduleOutputPatcher = moduleOutputPatcher,
targetDirectory = runDir,
platform = platformLayout,
searchableOptionSet = searchableOptionSet,
copyFiles = true,
context = context,
)
val entries = layoutPlatformDistribution(moduleOutputPatcher, runDir, platformLayout, searchableOptionSet, copyFiles = true, context)
lateinit var sortedClassPath: Set<Path>
coroutineScope {
launch {
@@ -664,6 +625,5 @@ private fun computeAdditionalModulesFingerprint(additionalModules: List<String>)
}
}
private fun getCommunityHomePath(homePath: Path): Path {
return if (Files.isDirectory(homePath.resolve("community"))) homePath.resolve("community") else homePath
}
private fun getCommunityHomePath(homePath: Path): Path =
if (Files.isDirectory(homePath.resolve("community"))) homePath.resolve("community") else homePath
@@ -276,7 +276,7 @@ private suspend fun buildOsSpecificDistributions(context: BuildContext): List<Di
// todo exclude plugins - layoutAdditionalResources should perform codesign -
// that's why we process files and zip in plugins (but not JARs)
// and also kotlin compiler includes JNA
recursivelySignMacBinaries(root = file, context)
recursivelySignMacBinaries(coroutineScope = this, file, context)
}
}
}
@@ -1,6 +1,4 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet", "ReplacePutWithAssignment")
package org.jetbrains.intellij.build.impl
import com.dynatrace.hash4j.hashing.HashStream64
@@ -20,7 +18,6 @@ import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.DirSource
import org.jetbrains.intellij.build.FrontendModuleFilter
import org.jetbrains.intellij.build.InMemoryContentSource
@@ -46,6 +43,8 @@ import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleLibraryFi
import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleOutputEntry
import org.jetbrains.intellij.build.impl.projectStructureMapping.ProjectLibraryEntry
import org.jetbrains.intellij.build.inferModuleSources
import org.jetbrains.intellij.build.io.WRITE_OPEN_OPTION
import org.jetbrains.intellij.build.io.writeToFileChannelFully
import org.jetbrains.intellij.build.jarCache.JarCacheManager
import org.jetbrains.intellij.build.jarCache.NonCachingJarCacheManager
import org.jetbrains.intellij.build.jarCache.SourceBuilder
@@ -56,6 +55,7 @@ import org.jetbrains.jps.model.library.JpsOrderRootType
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.model.module.JpsModuleReference
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
import java.nio.file.FileSystemException
import java.nio.file.FileSystems
import java.nio.file.Files
@@ -158,13 +158,9 @@ class JarPackager private constructor(
private val helper = (context as BuildContextImpl).jarPackagerDependencyHelper
companion object {
suspend fun pack(
includedModules: Collection<ModuleItem>,
outputDir: Path,
context: BuildContext,
) {
val packager = JarPackager(outDir = outputDir, context = context, platformLayout = null, isRootDir = false, moduleOutputPatcher = ModuleOutputPatcher())
packager.computeModuleSources(includedModules = includedModules, layout = null, searchableOptionSet = null)
suspend fun pack(includedModules: Collection<ModuleItem>, outputDir: Path, context: BuildContext) {
val packager = JarPackager(outputDir, context, platformLayout = null, isRootDir = false, ModuleOutputPatcher())
packager.computeModuleSources(includedModules, layout = null, searchableOptionSet = null)
buildJars(
assets = packager.assets.values,
layout = null,
@@ -189,15 +185,15 @@ class JarPackager private constructor(
searchableOptionSet: SearchableOptionSetDescriptor? = null,
context: BuildContext,
): Collection<DistributionFileEntry> {
val packager = JarPackager(outDir = outputDir, context = context, platformLayout = platformLayout, isRootDir = isRootDir, moduleOutputPatcher = moduleOutputPatcher)
packager.computeModuleSources(includedModules = includedModules, layout = layout, searchableOptionSet = searchableOptionSet)
val packager = JarPackager(outputDir, context, platformLayout, isRootDir, moduleOutputPatcher)
packager.computeModuleSources(includedModules, layout, searchableOptionSet)
packager.computeModuleCustomLibrarySources(layout)
val frontendModuleFilter = context.getFrontendModuleFilter()
val libraryToMerge = packager.computeProjectLibrariesSources(outDir = outputDir, layout = layout, copiedFiles = packager.copiedFiles, frontendModuleFilter = frontendModuleFilter)
val libraryToMerge = packager.computeProjectLibrariesSources(outputDir, layout, packager.copiedFiles, frontendModuleFilter)
if (isRootDir) {
for ((jarName, predicate) in predefinedMergeRules) {
packager.mergeLibsByPredicate(jarName = jarName, libraryToMerge = libraryToMerge, outputDir = outputDir, predicate = predicate, frontendModuleFilter = frontendModuleFilter)
packager.mergeLibsByPredicate(jarName, libraryToMerge, outputDir, predicate, frontendModuleFilter)
}
if (!libraryToMerge.isEmpty()) {
@@ -215,7 +211,7 @@ class JarPackager private constructor(
else if (!libraryToMerge.isEmpty()) {
val mainJarName = (layout as PluginLayout).getMainJarName()
check(includedModules.any { it.relativeOutputFile == mainJarName })
packager.projectLibsToSourceWithMappings(uberJarFile = outputDir.resolve(mainJarName), libraryToMerge = libraryToMerge)
packager.projectLibsToSourceWithMappings(uberJarFile = outputDir.resolve(mainJarName), libraryToMerge)
}
val cacheManager = if (dryRun || context !is BuildContextImpl) NonCachingJarCacheManager else context.jarCacheManager
@@ -233,16 +229,14 @@ class JarPackager private constructor(
return coroutineScope {
if (buildAssetResult.sourceToNativeFiles.isNotEmpty()) {
launch(CoroutineName("pack native presigned files")) {
packNativePresignedFiles(nativeFiles = buildAssetResult.sourceToNativeFiles, dryRun = dryRun, context = context, toRelativePath = { libName, fileName ->
"lib/$libName/$fileName"
})
packNativePresignedFiles(buildAssetResult.sourceToNativeFiles, dryRun, context, toRelativePath = { libName, fileName -> "lib/$libName/$fileName" })
}
}
val list = mutableListOf<DistributionFileEntry>()
val hasher = Hashing.xxh3_64().hashStream()
for (item in packager.assets.values) {
computeDistributionFileEntries(asset = item, hasher = hasher, list = list, dryRun = dryRun, buildAssetResult = buildAssetResult)
computeDistributionFileEntries(item, hasher, list, dryRun, buildAssetResult)
}
list
}
@@ -254,7 +248,7 @@ class JarPackager private constructor(
// First, check the content. This is done prior to everything else since we might configure a custom relativeOutputFile.
if (layout is PluginLayout) {
computeModuleSourcesByContent(helper = helper, context = context, layout = layout, addedModules = addedModules, jarPackager = this, searchableOptionSet = searchableOptionSet)
computeModuleSourcesByContent(helper, context, layout, addedModules, jarPackager = this, searchableOptionSet)
}
for (item in includedModules) {
@@ -273,15 +267,7 @@ class JarPackager private constructor(
return
}
inferModuleSources(
layout = layout,
addedModules = addedModules,
platformLayout = platformLayout!!,
helper = helper,
jarPackager = this,
searchableOptionSet = searchableOptionSet,
context = context,
)
inferModuleSources(layout, addedModules, platformLayout!!, helper, jarPackager = this, searchableOptionSet, context)
}
internal suspend fun computeSourcesForModule(item: ModuleItem, layout: BaseLayout?, searchableOptionSet: SearchableOptionSetDescriptor?) {
@@ -302,7 +288,7 @@ class JarPackager private constructor(
val outFile = outDir.resolve(item.relativeOutputFile)
val asset = if (packToDir) {
assets.computeIfAbsent(moduleOutputRoots.single()) { file ->
AssetDescriptor(isDir = true, file = file, relativePath = "")
AssetDescriptor(isDir = true, file, relativePath = "")
}
}
else {
@@ -313,8 +299,8 @@ class JarPackager private constructor(
val moduleSources = asset.includedModules.computeIfAbsent(item) { mutableListOf() }
for (entry in patchedContent) {
moduleSources.add(InMemoryContentSource(relativePath = entry.key, data = entry.value))
for ((relativePath, data) in patchedContent) {
moduleSources.add(InMemoryContentSource(relativePath, data))
}
val jarAsset = lazy(LazyThreadSafetyMode.NONE) {
@@ -327,7 +313,7 @@ class JarPackager private constructor(
}
if (searchableOptionSet != null) {
addSearchableOptionSources(layout = layout, moduleName = moduleName, module = module, sources = jarAsset.value.sources, searchableOptionSet = searchableOptionSet)
addSearchableOptionSources(layout, moduleName, module, jarAsset.value.sources, searchableOptionSet)
}
val excludes = if (extraExcludes.isEmpty()) {
@@ -353,14 +339,7 @@ class JarPackager private constructor(
}
if (layout != null && (layout !is PluginLayout || !layout.modulesWithExcludedModuleLibraries.contains(moduleName))) {
computeSourcesForModuleLibs(
item = item,
layout = layout,
module = module,
copiedFiles = copiedFiles,
asset = jarAsset,
withTests = useTestModuleOutput
)
computeSourcesForModuleLibs(item, layout, module, copiedFiles, jarAsset, useTestModuleOutput)
}
}
@@ -376,12 +355,7 @@ class JarPackager private constructor(
}
else {
val targetFile = outDir.resolveSibling(relativePath)
val assetDescriptor = AssetDescriptor(
isDir = false,
file = targetFile,
relativePath = relativePath,
useCacheAsTargetFile = false,
)
val assetDescriptor = AssetDescriptor(isDir = false, targetFile, relativePath, useCacheAsTargetFile = false)
customAsset.getSources(context)?.let { assetDescriptor.sources.addAll(it) }
val existing = assets.putIfAbsent(targetFile, assetDescriptor)
require(existing == null) {
@@ -430,7 +404,7 @@ class JarPackager private constructor(
val moduleName = module.name
val includeProjectLib = if (layout is PluginLayout) layout.auto else item.reason == ModuleIncludeReasons.PRODUCT_MODULES
val excluded = if (layout is PluginLayout) (layout.excludedLibraries.get(moduleName) ?: emptyList()) + (layout.excludedLibraries.get(null) ?: emptyList()) else emptySet()
val excluded = if (layout is PluginLayout) (layout.excludedLibraries[moduleName] ?: emptyList()) + (layout.excludedLibraries[null] ?: emptyList()) else emptySet()
for (element in helper.getLibraryDependencies(module, withTests = withTests)) {
var projectLibraryData: ProjectLibraryData? = null
val libRef = element.libraryReference
@@ -441,11 +415,11 @@ class JarPackager private constructor(
continue
}
if (helper.hasLibraryInDependencyChainOfModuleDependencies(dependentModule = module, libraryName = libName, siblings = layout.includedModules, withTests = withTests)) {
if (helper.hasLibraryInDependencyChainOfModuleDependencies(module, libName, layout.includedModules, withTests)) {
continue
}
projectLibraryData = ProjectLibraryData(libraryName = libName, reason = "<- $moduleName")
projectLibraryData = ProjectLibraryData(libName, reason = "<- $moduleName")
}
else if (platformLayout != null && platformLayout.isLibraryAlwaysPackedIntoPlugin(libName)) {
platformLayout.findProjectLibrary(libName)?.let {
@@ -456,7 +430,7 @@ class JarPackager private constructor(
continue
}
projectLibraryData = ProjectLibraryData(libraryName = libName, reason = "<- $moduleName (always packed into plugin)")
projectLibraryData = ProjectLibraryData(libName, reason = "<- $moduleName (always packed into plugin)")
}
else {
continue
@@ -470,21 +444,16 @@ class JarPackager private constructor(
}
if (item.reason == ModuleIncludeReasons.PRODUCT_MODULES) {
packLibFilesIntoModuleJar(asset = asset.value, item = item, files = library.getPaths(JpsOrderRootType.COMPILED), projectLibraryData = projectLibraryData, library = library)
packLibFilesIntoModuleJar(asset.value, item, library.getPaths(JpsOrderRootType.COMPILED), projectLibraryData, library)
}
else {
fun addLibrary(relativeOutputFile: String, files: List<Path>) {
filesToSourceWithMapping(
asset = getJarAsset(targetFile = outDir.resolve(relativeOutputFile), relativeOutputFile = relativeOutputFile),
files = files,
library = library,
relativeOutputFile = relativeOutputFile,
projectLibraryData = projectLibraryData,
)
val asset = getJarAsset(targetFile = outDir.resolve(relativeOutputFile), relativeOutputFile)
filesToSourceWithMapping(asset, files, library, relativeOutputFile, projectLibraryData)
}
val targetFile = outDir.resolve(item.relativeOutputFile)
val files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile)
val files = getLibraryFiles(library, copiedFiles, targetFile)
if (layout is PluginLayout && item.relativeOutputFile == layout.getMainJarName()) {
if (files.size > 1) {
for (i in (files.size - 1) downTo 0) {
@@ -492,12 +461,12 @@ class JarPackager private constructor(
val fileName = file.fileName.toString()
if (fileName.endsWith("-rt.jar") || fileName.startsWith("maven-")) {
files.removeAt(i)
addLibrary(relativeOutputFile = removeVersionFromJar(fileName), files = listOf(file))
addLibrary(relativeOutputFile = removeVersionFromJar(fileName), listOf(file))
}
}
}
addLibrary(relativeOutputFile = removeVersionFromJar(nameToJarFileName(getLibraryFileName(library))), files = files)
addLibrary(relativeOutputFile = removeVersionFromJar(nameToJarFileName(getLibraryFileName(library))), files)
}
else {
for (i in (files.size - 1) downTo 0) {
@@ -505,11 +474,11 @@ class JarPackager private constructor(
val fileName = file.fileName.toString()
if (isSeparateJar(fileName)) {
files.removeAt(i)
addLibrary(relativeOutputFile = removeVersionFromJar(fileName), files = listOf(file))
addLibrary(relativeOutputFile = removeVersionFromJar(fileName), listOf(file))
}
}
packLibFilesIntoModuleJar(asset = asset.value, item = item, files = files, projectLibraryData = projectLibraryData, library = library)
packLibFilesIntoModuleJar(asset.value, item, files, projectLibraryData, library)
}
}
}
@@ -530,18 +499,10 @@ class JarPackager private constructor(
file = file,
distributionFileEntryProducer = { size, hash, targetFile ->
if (projectLibraryData == null) {
ModuleLibraryFileEntry(
path = targetFile,
moduleName = item.moduleName,
libraryName = libraryName,
libraryFile = file,
size = size,
hash = hash,
relativeOutputFile = item.relativeOutputFile,
)
ModuleLibraryFileEntry(targetFile, item.moduleName, libraryName, file, size, hash, item.relativeOutputFile)
}
else {
ProjectLibraryEntry(path = targetFile, data = projectLibraryData, libraryFile = file, hash = hash, size = size, relativeOutputFile = item.relativeOutputFile)
ProjectLibraryEntry(targetFile, projectLibraryData, file, hash, size, item.relativeOutputFile)
}
},
isPreSignedAndExtractedCandidate = isLibPreSigned(library),
@@ -553,8 +514,7 @@ class JarPackager private constructor(
private fun computeModuleCustomLibrarySources(layout: BaseLayout) {
for (item in layout.includedModuleLibraries) {
val library = context.findRequiredModule(item.moduleName).libraryCollection.libraries
.find { getLibraryFileName(it) == item.libraryName }
val library = context.findRequiredModule(item.moduleName).libraryCollection.libraries.find { getLibraryFileName(it) == item.libraryName }
?: throw IllegalArgumentException("Cannot find library ${item.libraryName} in '${item.moduleName}' module")
var relativePath = item.relativeOutputPath
@@ -576,19 +536,14 @@ class JarPackager private constructor(
}
}
filesToSourceWithMapping(
asset = getJarAsset(targetFile = targetFile, relativeOutputFile = relativePath),
files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile),
library = library,
relativeOutputFile = relativePath,
projectLibraryData = null,
)
val asset = getJarAsset(targetFile, relativePath)
val files = getLibraryFiles(library, copiedFiles, targetFile)
filesToSourceWithMapping(asset, files, library, relativePath, projectLibraryData = null)
}
}
private fun alreadyHasLibrary(layout: BaseLayout, libraryName: String): Boolean {
return layout.includedModuleLibraries.any { it.libraryName == libraryName && !it.extraCopy }
}
private fun alreadyHasLibrary(layout: BaseLayout, libraryName: String): Boolean =
layout.includedModuleLibraries.any { it.libraryName == libraryName && !it.extraCopy }
private fun mergeLibsByPredicate(
jarName: String,
@@ -603,7 +558,7 @@ class JarPackager private constructor(
val (key, value) = iterator.next()
if (predicate(key.name, frontendModuleFilter)) {
iterator.remove()
result.put(key, value)
result[key] = value
}
}
if (result.isEmpty()) {
@@ -615,13 +570,8 @@ class JarPackager private constructor(
private fun projectLibsToSourceWithMappings(uberJarFile: Path, libraryToMerge: Map<JpsLibrary, List<Path>>) {
val descriptor = getJarAsset(targetFile = uberJarFile, relativeOutputFile = "")
for ((library, files) in libraryToMerge) {
filesToSourceWithMapping(
asset = descriptor,
files = files,
library = library,
relativeOutputFile = null,
projectLibraryData = libToMetadata.get(library) ?: throw IllegalStateException("Metadata not found for ${library.name}"),
)
val projectLibraryData = libToMetadata.get(library) ?: throw IllegalStateException("Metadata not found for ${library.name}")
filesToSourceWithMapping(descriptor, files, library, relativeOutputFile = null, projectLibraryData)
}
}
@@ -640,7 +590,7 @@ class JarPackager private constructor(
for (libraryData in projectLibs) {
val library = context.project.libraryCollection.findLibrary(libraryData.libraryName)
?: throw IllegalArgumentException("Cannot find library ${libraryData.libraryName} in the project")
libToMetadata.put(library, libraryData)
libToMetadata[library] = libraryData
val libName = library.name
var packMode = libraryData.packMode
if (packMode == LibraryPackMode.MERGED) {
@@ -654,7 +604,7 @@ class JarPackager private constructor(
val outPath = libraryData.outPath
if (packMode == LibraryPackMode.MERGED && outPath == null) {
toMerge.put(library, getLibraryFiles(library, copiedFiles, targetFile = null))
toMerge[library] = getLibraryFiles(library, copiedFiles, targetFile = null)
continue
}
@@ -662,13 +612,9 @@ class JarPackager private constructor(
if (outPath != null) {
if (outPath.endsWith(".jar")) {
val targetFile = outDir.resolve(outPath)
filesToSourceWithMapping(
asset = getJarAsset(targetFile = targetFile, relativeOutputFile = outPath),
files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile),
library = library,
relativeOutputFile = outPath,
projectLibraryData = libraryData,
)
val asset = getJarAsset(targetFile, outPath)
val files = getLibraryFiles(library, copiedFiles, targetFile)
filesToSourceWithMapping(asset, files, library, outPath, libraryData)
continue
}
@@ -676,19 +622,14 @@ class JarPackager private constructor(
}
fun addLibrary(targetFile: Path, relativeOutputFile: String, files: List<Path>) {
filesToSourceWithMapping(
asset = getJarAsset(targetFile = targetFile, relativeOutputFile = relativeOutputFile),
files = files,
library = library,
relativeOutputFile = relativeOutputFile,
projectLibraryData = libraryData,
)
val asset = getJarAsset(targetFile, relativeOutputFile)
filesToSourceWithMapping(asset, files, library, relativeOutputFile, libraryData)
}
if (packMode == LibraryPackMode.STANDALONE_MERGED) {
val targetFile = libOutputDir.resolve(nameToJarFileName(libName))
val relativeOutputFile = if (outDir == libOutputDir) "" else outDir.relativize(targetFile).invariantSeparatorsPathString
addLibrary(targetFile = targetFile, relativeOutputFile = relativeOutputFile, files = getLibraryFiles(library, copiedFiles, targetFile))
addLibrary(targetFile, relativeOutputFile, getLibraryFiles(library, copiedFiles, targetFile))
}
else {
for (file in library.getPaths(JpsOrderRootType.COMPILED)) {
@@ -699,7 +640,7 @@ class JarPackager private constructor(
val targetFile = libOutputDir.resolve(fileName)
val relativeOutputFile = if (outDir == libOutputDir) "" else outDir.relativize(targetFile).invariantSeparatorsPathString
addLibrary(targetFile = targetFile, relativeOutputFile = relativeOutputFile, files = listOf(file))
addLibrary(targetFile, relativeOutputFile, listOf(file))
}
}
}
@@ -729,25 +670,11 @@ class JarPackager private constructor(
optimizeConfigId = libraryName.takeIf { isRootDir && libraryName == "jsvg" },
distributionFileEntryProducer = { size, hash, targetFile ->
if (moduleName == null) {
ProjectLibraryEntry(
path = targetFile,
data = projectLibraryData ?: throw IllegalStateException("Metadata not specified for $libraryName"),
libraryFile = file,
hash = hash,
size = size,
relativeOutputFile = relativeOutputFile,
)
val data = projectLibraryData ?: throw IllegalStateException("Metadata not specified for $libraryName")
ProjectLibraryEntry(targetFile, data, file, hash, size, relativeOutputFile)
}
else {
ModuleLibraryFileEntry(
path = targetFile,
moduleName = moduleName,
libraryName = getLibraryFilename(library),
libraryFile = file,
hash = hash,
size = size,
relativeOutputFile = relativeOutputFile,
)
ModuleLibraryFileEntry(targetFile, moduleName, getLibraryFilename(library), file, size, hash, relativeOutputFile)
}
},
filter = ::defaultLibrarySourcesNamesFilter,
@@ -756,10 +683,8 @@ class JarPackager private constructor(
}
}
private fun getJarAsset(targetFile: Path, relativeOutputFile: String): AssetDescriptor {
return assets.computeIfAbsent(targetFile) {
createAssetDescriptor(targetFile = targetFile, relativeOutputFile = relativeOutputFile)
}
private fun getJarAsset(targetFile: Path, relativeOutputFile: String): AssetDescriptor = assets.computeIfAbsent(targetFile) {
createAssetDescriptor(targetFile = targetFile, relativeOutputFile = relativeOutputFile)
}
}
@@ -769,11 +694,10 @@ private val agentLibrariesNotForcedInSeparateJars = listOf(
"code-prompt-agents"
)
private fun isSeparateJar(fileName: String): Boolean {
return fileName.endsWith("-rt.jar") ||
(fileName.contains("-agent") && agentLibrariesNotForcedInSeparateJars.none { fileName.contains(it) }) ||
fileName.startsWith("maven-")
}
private fun isSeparateJar(fileName: String): Boolean =
fileName.endsWith("-rt.jar") ||
(fileName.contains("-agent") && agentLibrariesNotForcedInSeparateJars.none { fileName.contains(it) }) ||
fileName.startsWith("maven-")
private data class AssetDescriptor(
@JvmField val isDir: Boolean,
@@ -810,9 +734,7 @@ private fun getLibraryFiles(library: JpsLibrary, copiedFiles: MutableMap<CopiedF
return files
}
private fun nameToJarFileName(name: String): String {
return "${sanitizeFileName(name.lowercase(), replacement = "-")}.jar"
}
private fun nameToJarFileName(name: String): String = "${sanitizeFileName(name.lowercase(), replacement = "-")}.jar"
@Suppress("SpellCheckingInspection", "RedundantSuppression")
private val excludedFromMergeLibs = setOf(
@@ -822,17 +744,16 @@ private val excludedFromMergeLibs = setOf(
"protobuf", // https://youtrack.jetbrains.com/issue/IDEA-268753
)
private fun isLibraryMergeable(libName: String): Boolean {
return !excludedFromMergeLibs.contains(libName) &&
!(libName.startsWith("kotlin-") && !libName.startsWith("kotlin-test-")) &&
!libName.startsWith("kotlinc.") &&
!libName.contains("-agent-") &&
!libName.startsWith("rd-") &&
!libName.contains("annotations", ignoreCase = true) &&
!libName.startsWith("junit", ignoreCase = true) &&
!libName.startsWith("cucumber-", ignoreCase = true) &&
!libName.contains("groovy", ignoreCase = true)
}
private fun isLibraryMergeable(libName: String): Boolean =
!excludedFromMergeLibs.contains(libName) &&
!(libName.startsWith("kotlin-") && !libName.startsWith("kotlin-test-")) &&
!libName.startsWith("kotlinc.") &&
!libName.contains("-agent-") &&
!libName.startsWith("rd-") &&
!libName.contains("annotations", ignoreCase = true) &&
!libName.startsWith("junit", ignoreCase = true) &&
!libName.startsWith("cucumber-", ignoreCase = true) &&
!libName.contains("groovy", ignoreCase = true)
internal val commonModuleExcludes: List<PathMatcher> = FileSystems.getDefault().let { fs ->
listOf(
@@ -846,20 +767,6 @@ internal val commonModuleExcludes: List<PathMatcher> = FileSystems.getDefault().
)
}
suspend fun moduleOutputAsSource(context: CompilationContext, module: JpsModule, excludes: List<PathMatcher> = commonModuleExcludes): List<Source> {
return context.getModuleOutputRoots(module).map { moduleOutput ->
check(Files.exists(moduleOutput)) {
"${module.name} module output directory doesn't exist: $moduleOutput"
}
if (moduleOutput.toString().endsWith(".jar")) {
ZipSource(file = moduleOutput, distributionFileEntryProducer = null, filter = createModuleSourcesNamesFilter(excludes))
}
else {
DirSource(dir = moduleOutput, excludes = excludes)
}
}
}
fun createModuleSourcesNamesFilter(excludes: List<PathMatcher>): (String) -> Boolean = { name ->
val p = Path.of(name)
excludes.none { it.matches(p) }
@@ -888,15 +795,7 @@ private suspend fun buildJars(
val list = withContext(Dispatchers.IO) {
assets.map { asset ->
async(CoroutineName("build jar for ${asset.relativePath}")) {
buildAsset(
asset = asset,
isCodesignEnabled = isCodesignEnabled,
context = context,
cache = cache,
useCacheAsTargetFile = useCacheAsTargetFile,
layout = layout,
helper = helper,
)
buildAsset(asset, isCodesignEnabled, context, cache, useCacheAsTargetFile, layout, helper)
}
}
}
@@ -909,7 +808,7 @@ private suspend fun buildJars(
sourceToNativeFiles.putAll(item.sourceToNativeFiles)
sourceToMetadata.putAll(item.sourceToMetadata)
}
return BuildAssetResult(sourceToNativeFiles = sourceToNativeFiles.ifEmpty { emptyMap() }, sourceToMetadata = sourceToMetadata)
return BuildAssetResult(sourceToNativeFiles = sourceToNativeFiles.ifEmpty { emptyMap() }, sourceToMetadata)
}
private data class SizeAndHash(@JvmField val size: Int, @JvmField val hash: Long)
@@ -946,7 +845,7 @@ private suspend fun buildAsset(
}
}
}
return BuildAssetResult(sourceToNativeFiles = emptyMap(), sourceToMetadata = sourceToMetadata)
return BuildAssetResult(sourceToNativeFiles = emptyMap(), sourceToMetadata)
}
val sources = if (includedModules.isEmpty()) {
@@ -1002,16 +901,12 @@ private suspend fun buildAsset(
}
override suspend fun produce(targetFile: Path) {
buildJar(
targetFile = targetFile,
sources = sources,
nativeFileHandler = nativeFileHandler,
addDirEntries = includedModules.any { helper.isTestPluginModule(moduleName = it.key.moduleName, module = null) },
)
val addDirEntries = includedModules.any { helper.isTestPluginModule(moduleName = it.key.moduleName, module = null) }
buildJar(targetFile, sources, nativeFileHandler, addDirEntries)
}
override fun consumeInfo(source: Source, size: Int, hash: Long) {
val old = sourceToMetadata.putIfAbsent(source, SizeAndHash(size = size, hash = hash))
val old = sourceToMetadata.putIfAbsent(source, SizeAndHash(size, hash))
require(old == null) {
"Source is duplicated: new $source, old: $old"
}
@@ -1020,7 +915,7 @@ private suspend fun buildAsset(
)
}
return BuildAssetResult(sourceToNativeFiles = nativeFileHandler?.sourceToNativeFiles ?: emptyMap(), sourceToMetadata = sourceToMetadata)
return BuildAssetResult(sourceToNativeFiles = nativeFileHandler?.sourceToNativeFiles ?: emptyMap(), sourceToMetadata)
}
private fun emptyBuildJarsResult() = BuildAssetResult(sourceToNativeFiles = emptyMap(), sourceToMetadata = emptyMap())
@@ -1040,21 +935,16 @@ private fun checkAssetUniqueness(assets: Collection<AssetDescriptor>) {
private class NativeFileHandlerImpl(private val context: BuildContext) : NativeFileHandler {
override val sourceToNativeFiles = HashMap<ZipSource, List<String>>()
override fun isNative(name: String): Boolean {
@Suppress("SpellCheckingInspection", "RedundantSuppression")
return isMacLibrary(name) ||
name.endsWith(".exe") ||
name.endsWith(".dll") ||
name.endsWith("pty4j-unix-spawn-helper") ||
name.endsWith("icudtl.dat")
}
@Suppress("SpellCheckingInspection", "RedundantSuppression")
override fun isNative(name: String): Boolean =
isMacLibrary(name) ||
name.endsWith(".exe") ||
name.endsWith(".dll") ||
name.endsWith("pty4j-unix-spawn-helper") ||
name.endsWith("icudtl.dat")
override fun isCompatibleWithTargetPlatform(name: String): Boolean {
if (!isNative(name)) {
return true
}
return NativeFilesMatcher.isCompatibleWithTargetPlatform(name, context.options.targetOs, context.options.targetArch)
}
override fun isCompatibleWithTargetPlatform(name: String): Boolean =
!isNative(name) || NativeFilesMatcher.isCompatibleWithTargetPlatform(name, context.options.targetOs, context.options.targetArch)
@Suppress("SpellCheckingInspection")
override suspend fun sign(name: String, dataSupplier: () -> ByteBuffer): Path? {
@@ -1081,7 +971,17 @@ private class NativeFileHandlerImpl(private val context: BuildContext) : NativeF
}
data.reset()
return signData(data, context)
val options = macSigningOptions("application/x-mac-app-bin", context)
val file = Files.createTempFile(context.paths.tempDir, "", "")
FileChannel.open(file, WRITE_OPEN_OPTION).use { fileChannel ->
writeToFileChannelFully(fileChannel, data)
}
context.proprietaryBuildTools.signTool.signFiles(listOf(file), context, options)
if (!context.options.isInDevelopmentMode) {
check(isSigned(file)) { "Missing signature for $file ($name)" }
}
return file
}
}
@@ -1119,9 +1019,8 @@ private fun createModuleSource(module: JpsModule, outputDir: Path, excludes: Lis
}
}
private fun createAssetDescriptor(relativeOutputFile: String, targetFile: Path): AssetDescriptor {
return AssetDescriptor(isDir = false, file = targetFile, relativePath = relativeOutputFile)
}
private fun createAssetDescriptor(relativeOutputFile: String, targetFile: Path): AssetDescriptor =
AssetDescriptor(isDir = false, targetFile, relativeOutputFile)
private fun computeDistributionFileEntries(
asset: AssetDescriptor,
@@ -1135,7 +1034,7 @@ private fun computeDistributionFileEntries(
hasher.reset()
if (!dryRun) {
for (source in sources) {
val info = buildAssetResult.sourceToMetadata.get(source) ?: continue
val info = buildAssetResult.sourceToMetadata[source] ?: continue
size += info.size
hasher.putInt(size)
hasher.putLong(info.hash)
@@ -1145,16 +1044,7 @@ private fun computeDistributionFileEntries(
hasher.putInt(sources.size)
val hash = hasher.asLong
list.add(
ModuleOutputEntry(
path = asset.effectiveFile,
moduleName = module.moduleName,
size = size,
hash = hash,
relativeOutputFile = module.relativeOutputFile,
reason = module.reason,
)
)
list.add(ModuleOutputEntry(asset.effectiveFile, module.moduleName, size, hash, module.relativeOutputFile, module.reason))
}
for (source in asset.sources) {
@@ -1165,4 +1055,4 @@ private fun computeDistributionFileEntries(
list.add(CustomAssetEntry(path = asset.effectiveFile, hash = 0))
}
}
}
}
@@ -217,10 +217,10 @@ class MacDistributionBuilder(
val binariesToSign = customizer.getBinariesToSign(context, arch).map(osAndArchSpecificDistPath::resolve)
val matchers = generateExecutableFilesMatchers(includeRuntime = false, arch).keys
withContext(Dispatchers.IO) {
signMacBinaries(files = binariesToSign, context)
signMacBinaries(binariesToSign, context)
for (dir in listOf(osAndArchSpecificDistPath, runtimeDist)) {
launch(CoroutineName("recursively signing macOS binaries in $dir")) {
recursivelySignMacBinaries(root = dir, context, matchers)
recursivelySignMacBinaries(coroutineScope = this, dir, context, matchers)
}
}
}
@@ -552,9 +552,6 @@ class MacDistributionBuilder(
private val publishSitArchive: Boolean
get() = !context.isStepSkipped(BuildOptions.MAC_SIT_PUBLICATION_STEP)
private val signMacOsBinaries: Boolean
get() = !context.isStepSkipped(BuildOptions.MAC_SIGN_STEP)
private suspend fun signAndBuildDmg(macZip: Path, isRuntimeBundled: Boolean, suffix: String, arch: JvmArchitecture, notarize: Boolean) {
require(Files.isRegularFile(macZip))
@@ -563,7 +560,7 @@ class MacDistributionBuilder(
Files.move(macZip, sitFile, StandardCopyOption.REPLACE_EXISTING)
if (context.isMacCodeSignEnabled) {
context.proprietaryBuildTools.signTool.signFiles(listOf(sitFile), context, signingOptions("application/x-mac-app-zip", context))
context.proprietaryBuildTools.signTool.signFiles(listOf(sitFile), context, macSigningOptions("application/x-mac-app-zip", context))
}
if (notarize) {
@@ -631,7 +628,7 @@ class MacDistributionBuilder(
Files.readString(scriptsDir.resolve("build-template.sh"))
.resolveTemplateVar("staple", "$staple")
.resolveTemplateVar("appName", context.fullBuildNumber)
.resolveTemplateVar("contentSigned", "${signMacOsBinaries}")
.resolveTemplateVar("contentSigned", "${context.isMacCodeSignEnabled}")
.resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}")
)
NioFiles.setExecutable(entrypoint)
@@ -1,5 +1,4 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplacePutWithAssignment")
package org.jetbrains.intellij.build.impl
import com.intellij.openapi.util.SystemInfoRt
@@ -24,13 +23,23 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.io.*
import org.jetbrains.intellij.build.io.AddDirEntriesMode
import org.jetbrains.intellij.build.io.PackageIndexBuilder
import org.jetbrains.intellij.build.io.WRITE_OPEN_OPTION
import org.jetbrains.intellij.build.io.ZipEntryProcessorResult
import org.jetbrains.intellij.build.io.readZipFile
import org.jetbrains.intellij.build.io.suspendAwareReadZipFile
import org.jetbrains.intellij.build.io.writeToFileChannelFully
import org.jetbrains.intellij.build.io.writeZipUsingTempFile
import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder
import org.jetbrains.intellij.build.telemetry.use
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
import java.nio.channels.SeekableByteChannel
import java.nio.file.*
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.PathMatcher
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.attribute.FileTime
import java.util.EnumSet
@@ -40,14 +49,11 @@ import kotlin.io.path.extension
import kotlin.io.path.name
import kotlin.io.path.relativeTo
internal fun isMacLibrary(name: String): Boolean {
return name.endsWith(".jnilib") ||
name.endsWith(".dylib") ||
name.endsWith(".so") ||
name.endsWith(".tbd")
}
internal fun isMacLibrary(name: String): Boolean =
name.endsWith(".jnilib") || name.endsWith(".dylib") || name.endsWith(".so") || name.endsWith(".tbd")
internal fun CoroutineScope.recursivelySignMacBinaries(
internal fun recursivelySignMacBinaries(
coroutineScope: CoroutineScope,
root: Path,
context: BuildContext,
executableFileMatchers: Collection<PathMatcher> = emptyList(),
@@ -62,23 +68,24 @@ internal fun CoroutineScope.recursivelySignMacBinaries(
if (name.endsWith(".jar") || name.endsWith(".zip")) {
archives.add(file)
}
else if (isMacLibrary(name) ||
executableFileMatchers.any { it.matches(relativePath) } ||
(SystemInfoRt.isUnix && Files.isExecutable(file))) {
else if (
isMacLibrary(name) ||
executableFileMatchers.any { it.matches(relativePath) } ||
(SystemInfoRt.isUnix && Files.isExecutable(file))
) {
binaries.add(file)
}
return FileVisitResult.CONTINUE
}
})
launch(CoroutineName("signing macOS binaries")) {
signMacBinaries(binaries.filter {
isMacBinary(it) && !isSigned(it)
}, context)
coroutineScope.launch(CoroutineName("signing macOS binaries")) {
val binariesToSign = binaries.filter { isMacBinary(it) && !isSigned(it) }
signMacBinaries(binariesToSign, context)
}
for (file in archives) {
launch(CoroutineName("signing macOS binaries in ${file.relativeTo(root)}")) {
coroutineScope.launch(CoroutineName("signing macOS binaries in ${file.relativeTo(root)}")) {
signAndRepackZipIfMacSignaturesAreMissing(file, context)
}
}
@@ -99,8 +106,10 @@ private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context
if (!isSigned(byteChannel, name)) {
data.reset()
val fileToBeSigned = Files.createTempFile(context.paths.tempDir, name.replace('/', '-').takeLast(128), "")
writeToFile(fileToBeSigned, data)
filesToBeSigned.put(name, fileToBeSigned)
FileChannel.open(fileToBeSigned, WRITE_OPEN_OPTION).use { fileChannel ->
writeToFileChannelFully(fileChannel, data)
}
filesToBeSigned[name] = fileToBeSigned
}
}
}
@@ -109,9 +118,10 @@ private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context
return
}
signMacBinaries(files = filesToBeSigned.values.toList(), context = context, checkPermissions = false)
signMacBinaries(filesToBeSigned.values.toList(), context, checkPermissions = false)
copyZipReplacing(origin = zip, entries = filesToBeSigned, context)
copyZipReplacing(origin = zip, entries = filesToBeSigned, context = context)
for (file in filesToBeSigned.values) {
Files.deleteIfExists(file)
}
@@ -139,20 +149,16 @@ private suspend fun copyZipReplacing(origin: Path, entries: Map<String, Path>, c
}
}
internal fun signingOptions(contentType: String, context: BuildContext): PersistentMap<String, String> {
val certificateID = context.proprietaryBuildTools.macOsCodesignIdentity?.value
check(certificateID != null || context.isStepSkipped(BuildOptions.MAC_SIGN_STEP)) {
"Missing certificate ID"
}
internal fun macSigningOptions(contentType: String, context: BuildContext): PersistentMap<String, String> {
val certificateID = context.proprietaryBuildTools.signTool.macOsCodesignIdentity?.certificateID
check(certificateID != null || context.isStepSkipped(BuildOptions.MAC_SIGN_STEP)) { "Missing certificate ID" }
val entitlements = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts/entitlements.xml")
check(entitlements.exists()) {
"Missing $entitlements file"
}
check(entitlements.exists()) { "Missing $entitlements file" }
return persistentMapOf(
"mac_codesign_options" to "runtime",
"mac_codesign_identity" to "$certificateID",
"mac_codesign_entitlements" to "$entitlements",
"mac_codesign_force" to "true", // true if omitted
"mac_codesign_force" to "true",
"contentType" to contentType
)
}
@@ -160,8 +166,8 @@ internal fun signingOptions(contentType: String, context: BuildContext): Persist
internal suspend fun signMacBinaries(
files: List<Path>,
context: BuildContext,
checkPermissions: Boolean = true,
additionalOptions: Map<String, String> = emptyMap(),
checkPermissions: Boolean = true,
) {
if (files.isEmpty() || !context.isMacCodeSignEnabled) {
return
@@ -180,9 +186,9 @@ internal suspend fun signMacBinaries(
val span = spanBuilder("sign binaries for macOS distribution")
span.setAttribute("contentType", "application/x-mac-app-bin")
span.setAttribute(AttributeKey.stringArrayKey("files"), files.map { it.name })
val options = signingOptions(contentType = "application/x-mac-app-bin", context = context).putAll(m = additionalOptions)
val options = macSigningOptions(contentType = "application/x-mac-app-bin", context).putAll(additionalOptions)
span.use {
context.proprietaryBuildTools.signTool.signFiles(files = files, context = context, options = options)
context.proprietaryBuildTools.signTool.signFiles(files, context, options)
if (!permissions.isEmpty()) {
// SRE-1223 workaround
files.forEach {
@@ -190,47 +196,28 @@ internal suspend fun signMacBinaries(
}
}
val missingSignature = files.filter { !isSigned(it) }
check(missingSignature.isEmpty()) {
"Missing signature for:\n" + missingSignature.joinToString(separator = "\n\t")
if (!context.options.isInDevelopmentMode) {
val missingSignature = files.filter { !isSigned(it) }
check(missingSignature.isEmpty()) {
"Missing signature for:\n" + missingSignature.joinToString(separator = "\n\t")
}
}
}
}
internal suspend fun signData(data: ByteBuffer, context: BuildContext): Path {
val options = signingOptions("application/x-mac-app-bin", context)
val file = Files.createTempFile(context.paths.tempDir, "", "")
writeToFile(file, data)
context.proprietaryBuildTools.signTool.signFiles(files = listOf(file), context = context, options = options)
check(isSigned(file)) { "Missing signature for $file" }
return file
}
private fun writeToFile(file: Path?, data: ByteBuffer) {
FileChannel.open(file, WRITE_OPEN_OPTION).use { fileChannel ->
writeToFileChannelFully(channel = fileChannel, data = data)
}
}
private fun isMacBinary(path: Path): Boolean = isMacBinary(Files.newByteChannel(path))
internal suspend fun isSigned(path: Path): Boolean {
return withContext(Dispatchers.IO) {
Files.newByteChannel(path).use {
isSigned(byteChannel = it, binaryId = path.toString())
}
internal suspend fun isSigned(path: Path): Boolean = withContext(Dispatchers.IO) {
Files.newByteChannel(path).use {
isSigned(byteChannel = it, binaryId = path.toString())
}
}
internal fun isMacBinary(byteChannel: SeekableByteChannel): Boolean {
return detectFileType(byteChannel).first == FileType.MachO
}
private fun isMacBinary(byteChannel: SeekableByteChannel): Boolean =
detectFileType(byteChannel).first == FileType.MachO
private fun detectFileType(byteChannel: SeekableByteChannel): Pair<FileType, EnumSet<FileProperties>> {
return byteChannel.use {
it.DetectFileType()
}
private fun detectFileType(byteChannel: SeekableByteChannel): Pair<FileType, EnumSet<FileProperties>> = byteChannel.use {
it.DetectFileType()
}
/**
@@ -1,7 +1,5 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:JvmName("JarBuilder")
@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog")
package org.jetbrains.intellij.build
import io.opentelemetry.api.common.AttributeKey
@@ -40,15 +38,15 @@ internal interface NativeFileHandler {
}
suspend fun buildJar(targetFile: Path, sources: List<Source>, compress: Boolean = false) {
buildJar(targetFile = targetFile, sources = sources, compress = compress, nativeFileHandler = null)
buildJar(targetFile, sources, nativeFileHandler = null, addDirEntries = false, compress)
}
internal suspend fun buildJar(
targetFile: Path,
sources: Collection<Source>,
nativeFileHandler: NativeFileHandler?,
addDirEntries: Boolean,
compress: Boolean = false,
nativeFileHandler: NativeFileHandler? = null,
addDirEntries: Boolean = false,
) {
val packageIndexBuilder = if (compress) null else PackageIndexBuilder(if (addDirEntries) AddDirEntriesMode.ALL else AddDirEntriesMode.NONE)
Files.createDirectories(targetFile.parent)
@@ -61,17 +59,7 @@ internal suspend fun buildJar(
val filesToMerge = mutableListOf<CharSequence>()
for (source in sources) {
writeSource(
source = source,
zipCreator = zipCreator,
uniqueNames = uniqueNames,
packageIndexBuilder = packageIndexBuilder,
targetFile = targetFile,
sources = sources,
nativeFileHandler = nativeFileHandler,
compress = compress,
filesToMerge = filesToMerge,
)
writeSource(source, zipCreator, uniqueNames, packageIndexBuilder, targetFile, sources, nativeFileHandler, compress, filesToMerge)
}
if (filesToMerge.isNotEmpty()) {
@@ -91,7 +79,6 @@ private suspend fun writeSource(
compress: Boolean,
filesToMerge: MutableList<CharSequence>,
) {
val indexWriter = packageIndexBuilder?.indexWriter
when (source) {
is DirSource -> {
val includeManifest = sources.size == 1
@@ -110,7 +97,6 @@ private suspend fun writeSource(
})
val normalizedDir = source.dir.toAbsolutePath().normalize()
archiver.setRootDir(normalizedDir, source.prefix)
indexWriter
archiveDir(
startDir = normalizedDir,
addFile = { archiver.addFile(it, zipCreator) },
@@ -139,18 +125,7 @@ private suspend fun writeSource(
is ZipSource -> {
val sourceFile = source.file
try {
handleZipSource(
source = source,
sourceFile = sourceFile,
nativeFileHandler = nativeFileHandler,
uniqueNames = uniqueNames,
sources = sources,
packageIndexBuilder = packageIndexBuilder,
zipCreator = zipCreator,
compress = compress,
targetFile = targetFile,
filesToMerge = filesToMerge,
)
handleZipSource(source, sourceFile, nativeFileHandler, uniqueNames, sources, packageIndexBuilder, zipCreator, compress, targetFile, filesToMerge)
}
catch (e: IOException) {
if (e.message?.contains("No space left on device") == true) {
@@ -171,17 +146,7 @@ private suspend fun writeSource(
is LazySource -> {
for (subSource in source.getSources()) {
require(subSource !== source)
writeSource(
source = subSource,
zipCreator = zipCreator,
uniqueNames = uniqueNames,
packageIndexBuilder = packageIndexBuilder,
targetFile = targetFile,
sources = sources,
nativeFileHandler = nativeFileHandler,
compress = compress,
filesToMerge = filesToMerge,
)
writeSource(subSource, zipCreator, uniqueNames, packageIndexBuilder, targetFile, sources, nativeFileHandler, compress, filesToMerge)
}
}
@@ -233,14 +198,14 @@ private suspend fun handleZipSource(
}
}
if (checkCoverageAgentManifest(name = name, sourceFile = sourceFile, targetFile = targetFile, dataSupplier = dataSupplier, writeData = ::writeZipData)) {
if (checkCoverageAgentManifest(name, sourceFile, targetFile, dataSupplier, ::writeZipData)) {
return@suspendAwareReadZipFile
}
val includeManifest = sources.size == 1
val isIncluded = source.filter(name) && (includeManifest || name != "META-INF/MANIFEST.MF")
if (!isIncluded || isDuplicated(uniqueNames = uniqueNames, name = name, sourceFile = sourceFile)) {
if (!isIncluded || isDuplicated(uniqueNames, name, sourceFile)) {
return@suspendAwareReadZipFile
}
@@ -318,6 +283,7 @@ private fun isDuplicated(uniqueNames: MutableMap<String, Path>, name: String, so
@Suppress("SpellCheckingInspection")
private fun getIgnoredNames(): Set<String> {
val set = mutableListOf<String>()
// compilation cache on TC
set.add(".hash")
set.add("classpath.index")
@@ -325,6 +291,7 @@ private fun getIgnoredNames(): Set<String> {
set.add("pom.xml")
set.add("about.html")
set.add("module-info.class")
// default is ok (modules not used)
set.add("META-INF/versions/9/kotlin/reflect/jvm/internal/impl/serialization/deserialization/builtins/BuiltInsResourceLoader.class")
set.add("META-INF/versions/9/org/apache/xmlbeans/impl/tool/MavenPluginResolver.class")
@@ -344,9 +311,11 @@ private fun getIgnoredNames(): Set<String> {
// duplicates in maven-resolver-transport-http and maven-resolver-transport-file
set.add("META-INF/sisu/javax.inject.Named")
// duplicates in recommenders-jayes-io-2.5.5 and recommenders-jayes-2.5.5.jar
set.add("OSGI-INF/l10n/bundle.properties")
// groovy
// Groovy
set.add("META-INF/groovy-release-info.properties")
set.add("native-image")
@@ -357,6 +326,7 @@ private fun getIgnoredNames(): Set<String> {
@Suppress("SpellCheckingInspection")
set.add(".gitkeep")
set.add(INDEX_FILENAME)
for (originalName in sequenceOf("NOTICE", "README", "LICENSE", "DEPENDENCIES", "CHANGES", "THIRD_PARTY_LICENSES", "COPYING")) {
for (name in sequenceOf(originalName, originalName.lowercase())) {
set.add(name)
@@ -367,71 +337,38 @@ private fun getIgnoredNames(): Set<String> {
set.add("META-INF/$name.md")
}
}
set.add("kotlinx/coroutines/debug/internal/ByteBuddyDynamicAttach.class")
set.add("kotlin/coroutines/jvm/internal/DebugProbesKt.class")
/**
* A merging build politic breaks Graal VM Truffle-based plugins in an inconsistant way, so it's better
* to provide a correctly merged version in the plugin.
* A merging build politic breaks Graal VM Truffle-based plugins in an inconsistant way,
* so it's better to provide a correctly merged version in the plugin.
*/
set.add("META-INF/services/com.oracle.truffle.api.provider.TruffleLanguageProvider")
return java.util.Set.copyOf(set)
}
private val ignoredNames = getIgnoredNames()
private val moduleInfoPattern = Regex("META-INF/versions/\\d+/module-info\\.class")
fun defaultLibrarySourcesNamesFilter(name: String): Boolean {
@Suppress("SpellCheckingInspection")
return !ignoredNames.contains(name) &&
!name.matches(moduleInfoPattern) &&
!name.endsWith(".kotlin_metadata") &&
!name.startsWith("license/") &&
!name.startsWith("META-INF/license/") &&
!name.startsWith("META-INF/LICENSE-") &&
!name.startsWith("native-image/") &&
// Class 'jakarta.json.JsonValue' not found while looking for field 'jakarta.json.JsonValue NULL'
//!name.startsWith("com/jayway/jsonpath/spi/json/JakartaJsonProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/json/JsonOrgJsonProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/json/TapestryJsonProvider") &&
//
//!name.startsWith("com/jayway/jsonpath/spi/mapper/JakartaMappingProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/mapper/JsonOrgMappingProvider") &&
//!name.startsWith("com/jayway/jsonpath/spi/mapper/TapestryMappingProvider") &&
//!name.startsWith("io/opentelemetry/exporter/internal/grpc/") &&
//!name.startsWith("io/opentelemetry/exporter/internal/okhttp/") &&
//// com.thaiopensource.datatype.xsd.regex.xerces2 is used instead
//!name.startsWith("com/thaiopensource/datatype/xsd/regex/xerces/RegexEngineImpl") &&
//!name.startsWith("com/thaiopensource/relaxng/util/JingTask") &&
//!name.startsWith("com/thaiopensource/validate/schematron") &&
//!name.startsWith("com/thoughtworks/xstream/core/util/ISO8601JodaTimeConverter") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/BEAStaxDriver") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/AbstractXppDomDriver") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Xom") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Dom4") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/JDom2") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/KXml2") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Wstx") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/Xpp3") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/xppdom") &&
//!name.startsWith("com/thoughtworks/xstream/io/xml/XppDom") &&
//!name.startsWith("com/michaelbaranov/microba/jgrpah/birdview/Birdview") &&
// XmlRPC lib
!name.startsWith("org/xml/sax/") &&
!name.startsWith("META-INF/versions/9/org/apache/logging/log4j/") &&
!name.startsWith("META-INF/versions/9/org/bouncycastle/") &&
!name.startsWith("META-INF/versions/10/org/bouncycastle/") &&
!name.startsWith("META-INF/versions/15/org/bouncycastle/") &&
!name.startsWith("kotlinx/coroutines/repackaged/") &&
!name.startsWith("native/") &&
!name.startsWith("licenses/") &&
!name.startsWith("META-INF/INDEX.LIST") &&
(!name.startsWith("META-INF/") || (!name.endsWith(".DSA") && !name.endsWith(".SF") && !name.endsWith(".RSA"))) &&
// we replace lib class with our own patched version
!name.startsWith("net/sf/cglib/core/AbstractClassGenerator")
}
fun defaultLibrarySourcesNamesFilter(name: String): Boolean =
!ignoredNames.contains(name) &&
!name.matches(moduleInfoPattern) &&
!name.endsWith(".kotlin_metadata") &&
!name.startsWith("license/") &&
!name.startsWith("licenses/") &&
!name.startsWith("native/") &&
!name.startsWith("META-INF/license/") &&
!name.startsWith("META-INF/LICENSE-") &&
!name.startsWith("native-image/") &&
!name.startsWith("org/xml/sax/") && // XmlRPC lib
!name.startsWith("META-INF/versions/9/org/apache/logging/log4j/") &&
!name.startsWith("META-INF/versions/9/org/bouncycastle/") &&
!name.startsWith("META-INF/versions/10/org/bouncycastle/") &&
!name.startsWith("META-INF/versions/15/org/bouncycastle/") &&
!name.startsWith("kotlinx/coroutines/repackaged/") &&
!name.startsWith("META-INF/INDEX.LIST") &&
(!name.startsWith("META-INF/") || (!name.endsWith(".DSA") && !name.endsWith(".SF") && !name.endsWith(".RSA"))) &&
!name.startsWith("net/sf/cglib/core/AbstractClassGenerator") // we replace the lib class with our own patched version