IJPL-216902 IJ-MR-181153 prepare to unify module sets

GitOrigin-RevId: 36addf20739ebb3576e641edc02773523324197f
This commit is contained in:
Vladimir Krivosheev
2025-11-09 21:03:42 +00:00
committed by intellij-monorepo-bot
parent 06b4848406
commit 1255e77c41
65 changed files with 5707 additions and 1383 deletions
+2
View File
@@ -989,10 +989,12 @@
<module fileurl="file://$PROJECT_DIR$/platform/bootstrap/dev/intellij.platform.bootstrap.dev.iml" filepath="$PROJECT_DIR$/platform/bootstrap/dev/intellij.platform.bootstrap.dev.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/buildData/intellij.platform.buildData.iml" filepath="$PROJECT_DIR$/platform/buildData/intellij.platform.buildData.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/intellij.platform.buildScripts.iml" filepath="$PROJECT_DIR$/platform/build-scripts/intellij.platform.buildScripts.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/api/intellij.platform.buildScripts.api.iml" filepath="$PROJECT_DIR$/platform/build-scripts/api/intellij.platform.buildScripts.api.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/bazel/intellij.platform.buildScripts.bazel.iml" filepath="$PROJECT_DIR$/platform/build-scripts/bazel/intellij.platform.buildScripts.bazel.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/codeOptimizer/intellij.platform.buildScripts.codeOptimizer.iml" filepath="$PROJECT_DIR$/platform/build-scripts/codeOptimizer/intellij.platform.buildScripts.codeOptimizer.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/dev-server/intellij.platform.buildScripts.devBuildServer.iml" filepath="$PROJECT_DIR$/platform/build-scripts/dev-server/intellij.platform.buildScripts.devBuildServer.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/downloader/intellij.platform.buildScripts.downloader.iml" filepath="$PROJECT_DIR$/platform/build-scripts/downloader/intellij.platform.buildScripts.downloader.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/product-dsl/intellij.platform.buildScripts.productDsl.iml" filepath="$PROJECT_DIR$/platform/build-scripts/product-dsl/intellij.platform.buildScripts.productDsl.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/testFramework/intellij.platform.buildScripts.testFramework.iml" filepath="$PROJECT_DIR$/platform/build-scripts/testFramework/intellij.platform.buildScripts.testFramework.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/tests/intellij.platform.buildScripts.tests.iml" filepath="$PROJECT_DIR$/platform/build-scripts/tests/intellij.platform.buildScripts.tests.iml" />
<module fileurl="file://$PROJECT_DIR$/platform/build-scripts/usages/intellij.platform.buildScripts.usages.iml" filepath="$PROJECT_DIR$/platform/build-scripts/usages/intellij.platform.buildScripts.usages.iml" />
+2
View File
@@ -486,10 +486,12 @@ platform/bootstrap
platform/bootstrap/coroutine
platform/bootstrap/dev
platform/build-scripts
platform/build-scripts/api
platform/build-scripts/codeOptimizer
platform/build-scripts/dev-server
platform/build-scripts/downloader
platform/build-scripts/icons
platform/build-scripts/product-dsl
platform/build-scripts/testFramework
platform/build-scripts/tests
platform/build-scripts/usages
@@ -40,12 +40,6 @@ internal suspend fun createCommunityBuildContext(
)
open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrainsProductProperties() {
override val moduleSetsProviders: List<ModuleSetProvider>
get() = listOf(CommunityModuleSets)
override val baseFileName: String
get() = "idea"
init {
configurePropertiesForAllEditionsOfIntelliJIdea(this)
platformPrefix = "Idea"
@@ -65,11 +59,11 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai
productLayout.prepareCustomPluginRepositoryForPublishedPlugins = false
productLayout.buildAllCompatiblePlugins = true
productLayout.pluginLayouts = CommunityRepositoryModules.COMMUNITY_REPOSITORY_PLUGINS.addAll(listOf(
productLayout.pluginLayouts = CommunityRepositoryModules.COMMUNITY_REPOSITORY_PLUGINS + persistentListOf(
JavaPluginLayout.javaPlugin(),
CommunityRepositoryModules.androidPlugin(allPlatforms = true),
CommunityRepositoryModules.groovyPlugin(),
))
)
productLayout.addPlatformSpec { layout, _ ->
layout.withModule("intellij.platform.structuralSearch")
@@ -78,12 +72,12 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai
productLayout.skipUnresolvedContentModules = true
mavenArtifacts.forIdeModules = true
mavenArtifacts.additionalModules = mavenArtifacts.additionalModules.addAll(MAVEN_ARTIFACTS_ADDITIONAL_MODULES)
mavenArtifacts.squashedModules = mavenArtifacts.squashedModules.addAll(persistentListOf(
mavenArtifacts.additionalModules += MAVEN_ARTIFACTS_ADDITIONAL_MODULES
mavenArtifacts.squashedModules += persistentListOf(
"intellij.platform.util.base",
"intellij.platform.util.base.multiplatform",
"intellij.platform.util.zip",
))
)
mavenArtifacts.validateForMavenCentralPublication = { module ->
JewelMavenArtifacts.isPublishedJewelModule(module)
}
@@ -120,47 +114,19 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai
additionalVmOptions = persistentListOf("-Dllm.show.ai.promotion.window.on.start=false")
}
override fun getProductContentDescriptor(): ProductModulesContentSpec {
return getIntelliJCommunityProductContentDescriptor(includeCommunityExtensions = true)
override val moduleSetsProviders: List<ModuleSetProvider>
get() = listOf(CommunityModuleSets)
override val baseFileName: String
get() = "idea"
override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules {
include(intellijCommunityBaseFragment())
include(communityExtensionsFragment())
}
protected fun getIntelliJCommunityProductContentDescriptor(
includeCommunityExtensions: Boolean,
): ProductModulesContentSpec = productModules {
alias("com.intellij.modules.idea")
alias("com.intellij.modules.idea.community")
alias("com.intellij.modules.java-capable")
alias("com.intellij.modules.python-core-capable")
alias("com.intellij.modules.python-in-non-pycharm-ide-capable")
alias("com.intellij.platform.ide.provisioner")
deprecatedInclude("intellij.java.ide.resources", "META-INF/JavaIdePlugin.xml")
deprecatedInclude("intellij.idea.community.customization", "META-INF/tips-intellij-idea-community.xml")
moduleSet(CommunityModuleSets.debuggerStreams())
module("intellij.platform.coverage")
module("intellij.platform.coverage.agent")
module("intellij.xml.xmlbeans")
module("intellij.platform.ide.newUiOnboarding")
module("intellij.platform.ide.newUsersOnboarding")
module("intellij.ide.startup.importSettings")
module("intellij.platform.customization.min")
module("intellij.idea.customization.base")
module("intellij.idea.customization.backend")
module("intellij.platform.tips")
moduleSet(CommunityModuleSets.ideCommon())
moduleSet(CommunityModuleSets.rdCommon())
if (includeCommunityExtensions) {
deprecatedInclude("intellij.platform.extended.community.impl", "META-INF/community-extensions.xml", ultimateOnly = true)
}
deprecatedInclude("intellij.idea.community.customization", "META-INF/community-customization.xml")
}
override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) {
super.copyAdditionalFiles(context, targetDir)
override suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) {
super.copyAdditionalFiles(targetDir, context)
copyFileToDir(context.paths.communityHomeDir.resolve("LICENSE.txt"), targetDir)
copyFileToDir(context.paths.communityHomeDir.resolve("NOTICE.txt"), targetDir)
@@ -211,12 +177,12 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai
override fun getRootDirectoryName(appInfo: ApplicationInfoProperties, buildNumber: String): String = "idea-IC-$buildNumber"
override fun generateExecutableFilesPatterns(
context: BuildContext,
includeRuntime: Boolean,
arch: JvmArchitecture,
targetLibcImpl: LibcImpl,
context: BuildContext,
): Sequence<String> {
return super.generateExecutableFilesPatterns(context, includeRuntime, arch, targetLibcImpl)
return super.generateExecutableFilesPatterns(includeRuntime, arch, targetLibcImpl, context)
.plus(KotlinBinaries.kotlinCompilerExecutables)
.filterNot { it == "plugins/**/*.sh" }
}
@@ -242,8 +208,8 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai
}
}
override fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture): Sequence<String> {
return super.generateExecutableFilesPatterns(context, includeRuntime, arch)
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, context: BuildContext): Sequence<String> {
return super.generateExecutableFilesPatterns(includeRuntime, arch, context)
.plus(KotlinBinaries.kotlinCompilerExecutables)
.filterNot { it == "plugins/**/*.sh" }
}
@@ -257,3 +223,45 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai
override fun getOutputDirectoryName(appInfo: ApplicationInfoProperties): String = "idea-ce"
}
/**
* Base IntelliJ Community content fragment.
* This fragment is composable - subclasses can include this and optionally add community extensions.
*/
fun intellijCommunityBaseFragment(): ProductModulesContentSpec = productModules {
alias("com.intellij.modules.idea")
alias("com.intellij.modules.idea.community")
alias("com.intellij.modules.java-capable")
alias("com.intellij.modules.python-core-capable")
alias("com.intellij.modules.python-in-non-pycharm-ide-capable")
alias("com.intellij.platform.ide.provisioner")
deprecatedInclude("intellij.java.ide.resources", "META-INF/JavaIdePlugin.xml")
deprecatedInclude("intellij.idea.community.customization", "META-INF/tips-intellij-idea-community.xml")
moduleSet(CommunityModuleSets.debuggerStreams())
module("intellij.platform.coverage")
module("intellij.platform.coverage.agent")
module("intellij.xml.xmlbeans")
module("intellij.platform.ide.newUiOnboarding")
module("intellij.platform.ide.newUsersOnboarding")
module("intellij.ide.startup.importSettings")
module("intellij.platform.customization.min")
module("intellij.idea.customization.base")
module("intellij.idea.customization.backend")
module("intellij.platform.tips")
moduleSet(CommunityModuleSets.ideCommon())
moduleSet(CommunityModuleSets.rdCommon())
deprecatedInclude("intellij.idea.community.customization", "META-INF/community-customization.xml")
}
/**
* Community extensions fragment for Ultimate builds.
* This fragment is composable - subclasses can choose to include or exclude it.
*/
fun communityExtensionsFragment(): ProductModulesContentSpec = productModules {
deprecatedInclude("intellij.platform.extended.community.impl", "META-INF/community-extensions.xml", ultimateOnly = true)
}
@@ -11,15 +11,15 @@
<xi:include href="/META-INF/JavaIdePlugin.xml"/>
<xi:include href="/META-INF/tips-intellij-idea-community.xml"/>
<xi:include href="/META-INF/community-customization.xml"/>
<xi:include href="/META-INF/community-extensions.xml">
<xi:fallback/>
</xi:include>
<xi:include href="/META-INF/community-customization.xml"/>
<xi:include href="/META-INF/intellij.moduleSets.debugger.streams.xml"/>
<xi:include href="/META-INF/intellij.moduleSets.ide.common.xml"/>
<xi:include href="/META-INF/intellij.moduleSets.rd.common.xml"/>
<!-- <editor-fold desc="additional"> -->
<content namespace="jetbrains">
<!-- <editor-fold desc="additional"> -->
<module name="intellij.platform.coverage"/>
<module name="intellij.platform.coverage.agent"/>
<module name="intellij.xml.xmlbeans"/>
@@ -30,6 +30,6 @@
<module name="intellij.idea.customization.base"/>
<module name="intellij.idea.customization.backend"/>
<module name="intellij.platform.tips"/>
<!-- </editor-fold> -->
</content>
<!-- </editor-fold> -->
</idea-plugin>
+6
View File
@@ -88,6 +88,12 @@ jvm_library(
"@lib//:platform-build_scripts-jetbrains-intellij-deps-coverage-reporter",
"//libraries/ktor/client",
"//platform/plugins/parser/impl",
"//platform/build-scripts/api",
"//platform/build-scripts/product-dsl",
],
exports = [
"//platform/build-scripts/api",
"//platform/build-scripts/product-dsl",
],
runtime_deps = [
"//libraries/commons/cli",
+21
View File
@@ -0,0 +1,21 @@
### auto-generated section `build intellij.platform.buildScripts.api` start
load("@rules_jvm//:jvm.bzl", "jvm_library")
jvm_library(
name = "api",
module_name = "intellij.platform.buildScripts.api",
visibility = ["//visibility:public"],
srcs = glob(["src/**/*.kt", "src/**/*.java", "src/**/*.form"], allow_empty = True),
deps = [
"@lib//:kotlin-stdlib",
"//plugins/groovy/rt/classLoader",
"//platform/util-class-loader:util-classLoader",
"//platform/util",
"@lib//:jetbrains-annotations",
"//platform/build-scripts/downloader",
"//jps/model-api:model",
"//platform/util/zip",
"@community//build:zip",
]
)
### auto-generated section `build intellij.platform.buildScripts.api` end
@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" packagePrefix="org.jetbrains.intellij.build" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.groovy.rt.classLoader" />
<orderEntry type="module" module-name="intellij.platform.util.classLoader" />
<orderEntry type="module" module-name="intellij.platform.util" />
<orderEntry type="library" name="jetbrains-annotations" level="project" />
<orderEntry type="module" module-name="intellij.platform.buildScripts.downloader" />
<orderEntry type="module" module-name="intellij.platform.jps.model" />
<orderEntry type="module" module-name="intellij.platform.util.zip" />
<orderEntry type="module" module-name="intellij.idea.community.build.zip" />
</component>
</module>
@@ -72,4 +72,4 @@ class BuildPaths(
*/
@Deprecated("Use [artifactDir] or [tempDir] instead")
val jpsArtifacts: Path = buildOutputDir.resolve("jps-artifacts")
}
}
@@ -1,9 +1,8 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import com.intellij.openapi.application.PathManager
import com.intellij.util.lang.UrlClassLoader
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import java.nio.file.Path
import java.nio.file.Paths
@@ -60,7 +59,7 @@ internal object IdeaProjectLoaderUtil {
/**
* This method only for internal usage. Use [BuildPaths.ULTIMATE_HOME] instead.
*/
@ApiStatus.Internal
@Internal
fun guessUltimateHome(): Path {
return searchForAnyMarkerFile(listOf(ULTIMATE_REPO_MARKER_FILE))
}
@@ -68,7 +67,7 @@ internal object IdeaProjectLoaderUtil {
/**
* This method only for internal usage. Use [BuildPaths.COMMUNITY_ROOT] instead.
*/
@ApiStatus.Internal
@Internal
fun guessCommunityHome(): BuildDependenciesCommunityRoot {
val directMarker = COMMUNITY_REPO_MARKER_FILE
val inSubdirMarker = Path.of("community").resolve(COMMUNITY_REPO_MARKER_FILE)
@@ -85,7 +84,7 @@ internal object IdeaProjectLoaderUtil {
/**
* This method only for internal usage. Use [BuildPaths.MAYBE_ULTIMATE_HOME] instead.
*/
@ApiStatus.Internal
@Internal
fun maybeUltimateHome(): Path? {
return searchForOptionalMarkerFile(listOf(ULTIMATE_REPO_MARKER_FILE))
}
@@ -134,4 +133,4 @@ internal object IdeaProjectLoaderUtil {
checkNotNull(classFileURL) { "Could not get .class file location from class " + klass.getName() }
return UrlClassLoader.urlToFilePath(classFileURL.path)
}
}
}
@@ -0,0 +1,15 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import org.jetbrains.jps.model.module.JpsModule
import java.nio.file.Path
interface ModuleOutputProvider {
fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean = false): ByteArray?
fun findModule(name: String): JpsModule?
fun findRequiredModule(name: String): JpsModule
fun getModuleOutputRoots(module: JpsModule, forTests: Boolean = false): List<Path>
}
@@ -3,8 +3,6 @@ package org.jetbrains.intellij.build
import com.intellij.util.lang.ImmutableZipFile
import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.impl.ModuleOutputProvider
import org.jetbrains.intellij.build.io.ZipEntryProcessorResult
import org.jetbrains.intellij.build.io.readZipFile
import org.jetbrains.jps.model.java.JavaResourceRootType
@@ -19,6 +17,9 @@ import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
const val PLUGIN_XML_RELATIVE_PATH: String = "META-INF/plugin.xml"
val useTestSourceEnabled: Boolean = System.getProperty("idea.build.pack.test.source.enabled", "true").toBoolean()
fun getUnprocessedPluginXmlContent(module: JpsModule, context: ModuleOutputProvider): ByteArray {
return requireNotNull(findUnprocessedDescriptorContent(module = module, path = PLUGIN_XML_RELATIVE_PATH, context = context)) {
"META-INF/plugin.xml not found in ${module.name} module output"
@@ -40,7 +41,7 @@ fun findUnprocessedDescriptorContent(module: JpsModule, path: String, context: M
private val rootTypeOrder = arrayOf(JavaResourceRootType.RESOURCE, JavaSourceRootType.SOURCE, JavaResourceRootType.TEST_RESOURCE, JavaSourceRootType.TEST_SOURCE)
internal fun findFileInModuleSources(module: JpsModule, relativePath: String, onlyProductionSources: Boolean = false): Path? {
fun findFileInModuleSources(module: JpsModule, relativePath: String, onlyProductionSources: Boolean = false): Path? {
for (type in rootTypeOrder) {
for (root in module.sourceRoots) {
if (type != root.rootType || (onlyProductionSources && !(root.rootType == JavaResourceRootType.RESOURCE || root.rootType == JavaSourceRootType.SOURCE))) {
@@ -55,9 +56,9 @@ internal fun findFileInModuleSources(module: JpsModule, relativePath: String, on
return null
}
internal fun isModuleNameLikeFilename(relativePath: String): Boolean = relativePath.startsWith("intellij.") || relativePath.startsWith("fleet.")
fun isModuleNameLikeFilename(relativePath: String): Boolean = relativePath.startsWith("intellij.") || relativePath.startsWith("fleet.")
internal fun findFileInModuleLibraryDependencies(module: JpsModule, relativePath: String): ByteArray? {
fun findFileInModuleLibraryDependencies(module: JpsModule, relativePath: String): ByteArray? {
for (dependency in module.dependenciesList.dependencies) {
if (dependency is JpsLibraryDependency) {
val library = dependency.library ?: continue
@@ -71,14 +72,14 @@ internal fun findFileInModuleLibraryDependencies(module: JpsModule, relativePath
return null
}
internal fun findProductModulesFile(clientMainModuleName: String, context: CompilationContext): Path? {
fun findProductModulesFile(clientMainModuleName: String, context: ModuleOutputProvider): Path? {
return findFileInModuleSources(context.findRequiredModule(clientMainModuleName), "META-INF/$clientMainModuleName/product-modules.xml")
}
internal fun findFileInModuleDependencies(
fun findFileInModuleDependencies(
module: JpsModule,
relativePath: String,
context: CompilationContext,
context: ModuleOutputProvider,
processedModules: MutableSet<String>,
recursiveModuleExclude: String? = null,
): ByteArray? {
@@ -98,7 +99,7 @@ internal fun findFileInModuleDependencies(
private fun findFileInModuleDependenciesRecursive(
module: JpsModule,
relativePath: String,
context: CompilationContext,
context: ModuleOutputProvider,
processedModules: MutableSet<String>,
recursiveModuleExclude: String?,
): ByteArray? {
@@ -134,7 +135,7 @@ private fun findFileInModuleDependenciesRecursive(
}
@Internal
fun hasModuleOutputPath(module: JpsModule, relativePath: String, context: CompilationContext): Boolean {
fun hasModuleOutputPath(module: JpsModule, relativePath: String, context: ModuleOutputProvider): Boolean {
return context.getModuleOutputRoots(module).any { output ->
val attributes = try {
Files.readAttributes(output, BasicFileAttributes::class.java)
@@ -420,5 +420,7 @@
</orderEntry>
<orderEntry type="module" module-name="intellij.libraries.ktor.client" />
<orderEntry type="module" module-name="intellij.platform.plugins.parser.impl" />
<orderEntry type="module" module-name="intellij.platform.buildScripts.api" exported="" />
<orderEntry type="module" module-name="intellij.platform.buildScripts.productDsl" exported="" />
</component>
</module>
@@ -0,0 +1,65 @@
### auto-generated section `build intellij.platform.buildScripts.productDsl` start
load("//build:compiler-options.bzl", "create_kotlinc_options")
load("@rules_jvm//:jvm.bzl", "jvm_library")
create_kotlinc_options(
name = "custom_product-dsl",
opt_in = [
"kotlin.RequiresOptIn",
"kotlinx.serialization.ExperimentalSerializationApi",
"kotlinx.coroutines.ExperimentalCoroutinesApi",
]
)
jvm_library(
name = "product-dsl",
module_name = "intellij.platform.buildScripts.productDsl",
visibility = ["//visibility:public"],
srcs = glob(["src/**/*.kt", "src/**/*.java", "src/**/*.form"], allow_empty = True),
kotlinc_opts = ":custom_product-dsl",
deps = [
"@lib//:kotlin-stdlib",
"//platform/plugins/parser/impl",
"//platform/build-scripts/api",
"//libraries/kotlinx/serialization/core",
"//libraries/kotlinx/serialization/json",
"//jps/model-api:model",
"//jps/model-serialization",
"//platform/util",
"//platform/util/jdom",
"//libraries/jackson/jackson",
]
)
jvm_library(
name = "product-dsl_test_lib",
visibility = ["//visibility:public"],
srcs = glob(["testSrc/**/*.kt", "testSrc/**/*.java", "testSrc/**/*.form"], allow_empty = True),
kotlinc_opts = ":custom_product-dsl",
associates = [":product-dsl"],
deps = [
"@lib//:kotlin-stdlib",
"//platform/plugins/parser/impl",
"//platform/plugins/parser/impl:impl_test_lib",
"//platform/build-scripts/api",
"//libraries/kotlinx/serialization/core",
"//libraries/kotlinx/serialization/json",
"//jps/model-api:model",
"//jps/model-serialization",
"//platform/util",
"//platform/util/jdom",
"@lib//:junit5",
"@lib//:assert_j",
"//libraries/jackson/jackson",
]
)
### auto-generated section `build intellij.platform.buildScripts.productDsl` end
### auto-generated section `test intellij.platform.buildScripts.productDsl` start
load("@community//build:tests-options.bzl", "jps_test")
jps_test(
name = "product-dsl_test",
runtime_deps = [":product-dsl_test_lib"]
)
### auto-generated section `test intellij.platform.buildScripts.productDsl` end
@@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="kotlin-language" name="Kotlin">
<configuration version="5" platform="JVM 21" allPlatforms="JVM [21]" useProjectSettings="false">
<compilerSettings>
<option name="additionalArguments" value="-Xjvm-default=all -opt-in=kotlin.RequiresOptIn -opt-in=kotlinx.serialization.ExperimentalSerializationApi -opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi -XXLanguage:+AllowEagerSupertypeAccessibilityChecks" />
</compilerSettings>
<compilerArguments>
<stringArguments>
<stringArg name="jvmTarget" arg="21" />
<stringArg name="apiVersion" arg="2.2" />
<stringArg name="languageVersion" arg="2.2" />
</stringArguments>
<arrayArguments>
<arrayArg name="pluginClasspaths">
<args>$KOTLIN_BUNDLED$/lib/kotlinx-serialization-compiler-plugin.jar</args>
</arrayArg>
<arrayArg name="pluginOptions" />
</arrayArguments>
</compilerArguments>
</configuration>
</facet>
</component>
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" packagePrefix="org.jetbrains.intellij.build.productLayout" />
<sourceFolder url="file://$MODULE_DIR$/testSrc" isTestSource="true" packagePrefix="org.jetbrains.intellij.build.productLayout" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.platform.plugins.parser.impl" />
<orderEntry type="module" module-name="intellij.platform.buildScripts.api" />
<orderEntry type="module" module-name="intellij.libraries.kotlinx.serialization.core" />
<orderEntry type="module" module-name="intellij.libraries.kotlinx.serialization.json" />
<orderEntry type="module" module-name="intellij.platform.jps.model" />
<orderEntry type="module" module-name="intellij.platform.jps.model.serialization" />
<orderEntry type="module" module-name="intellij.platform.util" />
<orderEntry type="module" module-name="intellij.platform.util.jdom" />
<orderEntry type="library" scope="TEST" name="JUnit5" level="project" />
<orderEntry type="library" scope="TEST" name="assertJ" level="project" />
<orderEntry type="module" module-name="intellij.libraries.jackson" />
</component>
</module>
@@ -0,0 +1,56 @@
# Module Sets
Module sets are collections of modules that can be referenced as a single entity in product configurations.
## Overview
Module sets are now **defined in Kotlin code** in the following locations:
- **Community module sets**: `community/platform/build-scripts/product-dsl/src/CommunityModuleSets.kt`
- **Ultimate module sets**: `platform/buildScripts/src/productLayout/UltimateModuleSets.kt`
Each module set is defined as a Kotlin function returning a `ModuleSet` object. The XML files (following the pattern `intellij.moduleSets.<category>.<subcategory>.xml`) are **auto-generated** from this Kotlin code.
### Generating XML Files
To regenerate XML files from Kotlin code, run:
```bash
# For all products (community + ultimate)
UltimateModuleSets.main()
# For community products only
CommunityModuleSets.main()
```
Or use the IDE's "Generate Product Layouts" run configuration.
### Querying Module Sets
To query module set structure, relationships, and usage programmatically, use the **JSON analysis endpoint**:
```bash
UltimateModuleSets.main(args = ["--json"])
```
See the [Programmatic Content](programmatic-content.md#json-analysis-endpoint) documentation for details.
## Creating a New Module Set
See `/create-module-set` slash command for detailed instructions on creating a new module set.
## Available Module Sets
Module sets are defined in Kotlin code:
- **Community module sets**: See `community/platform/build-scripts/product-dsl/src/CommunityModuleSets.kt`
- **Ultimate module sets**: See `platform/buildScripts/src/productLayout/UltimateModuleSets.kt`
Each module set is defined as a function (e.g., `fun essential(): ModuleSet`) within these classes.
### Querying Module Sets
To discover available module sets and their composition:
1. **Browse the source code** - Open the Kotlin files listed above to see all available module sets
2. **Use the JSON endpoint** - Run `UltimateModuleSets.main(args = ["--json"])` to get comprehensive analysis
3. **Check generated XML** - The generated XML files (e.g., `intellij.moduleSets.essential.xml`) contain expanded module lists
For detailed module composition, includes/includedBy relationships, and product usage, use the [JSON analysis endpoint](#querying-module-sets).
@@ -283,6 +283,173 @@ Use `ultimateOnly = true` when:
- **`generateAllProductXmlFiles()`** (generator.kt): Batch generation for all registered products
- **`collectAndValidateAliases()`** (generator.kt): Validates module aliases for duplicates
## JSON Analysis Endpoint
The module set system provides a JSON analysis endpoint for programmatic querying and tooling integration. This endpoint is used by the Plugin Model Analyzer MCP server and other build tools.
### Usage
Run the module set main function with the `--json` flag:
```bash
# Generate complete analysis for all products and module sets
UltimateModuleSets.main(args = ["--json"])
# Community products only
CommunityModuleSets.main(args = ["--json"])
```
### Filtering Output
Use the `--json` flag with a filter to get specific sections:
```bash
# Get only products
--json='{"filter":"products"}'
# Get only module sets
--json='{"filter":"moduleSets"}'
# Include duplicate analysis
--json='{"includeDuplicates":true}'
```
### Output Structure
The JSON output contains comprehensive analysis of the module system:
#### 1. Module Distribution
Maps each module to the module sets and products that include it:
```json
{
"moduleDistribution": {
"intellij.platform.vcs.impl": {
"inModuleSets": ["vcs", "ide.common"],
"inProducts": ["WebStorm", "GoLand", "CLion", "PyCharm", ...]
}
}
}
```
**Use case:** Find where a specific module is used across the codebase.
#### 2. Module Set Hierarchy
Shows the include relationships between module sets:
```json
{
"moduleSetHierarchy": {
"ide.common": {
"includes": ["essential", "vcs"],
"includedBy": ["ide.ultimate"],
"moduleCount": 145
}
}
}
```
**Use case:** Understand module set dependencies and nesting structure.
#### 3. Module Usage Index
Comprehensive reverse lookup with source file paths:
```json
{
"moduleUsageIndex": {
"modules": {
"intellij.platform.vcs.impl": {
"moduleSets": [
{
"name": "vcs",
"location": "community",
"sourceFile": "community/platform/build-scripts/product-dsl/src/CommunityModuleSets.kt"
}
],
"products": [
{
"name": "WebStorm",
"sourceFile": "platform/buildScripts/src/productLayout/UltimateModuleSets.kt"
}
]
}
}
}
}
```
**Use case:** Trace module ownership and find where to make changes.
#### 4. Product Composition Analysis
Detailed breakdown of each product's composition:
```json
{
"productCompositionAnalysis": {
"CLion": {
"composition": {
"totalAliases": 3,
"totalModuleSets": 12,
"totalDirectModules": 45,
"totalModules": 523
},
"operations": [
{"type": "alias", "value": "com.jetbrains.modules.cidr.lang"},
{"type": "moduleSet", "value": "commercial"},
{"type": "module", "value": "intellij.clion.core"}
]
}
}
}
```
**Use case:** Analyze product composition and optimize module dependencies.
#### 5. Duplicate Analysis (Optional)
When `includeDuplicates: true` is set, detects duplicate xi:include elements:
```json
{
"duplicateAnalysis": {
"ReSharper Backend": {
"/META-INF/intellij.moduleSets.essential.xml": [
{
"directInclude": true,
"deprecatedIncludeRefs": [
"intellij.platform.resources -> /META-INF/PlatformLangPlugin.xml"
]
}
]
}
}
}
```
**Use case:** Identify redundant includes that can be removed.
### Integration with MCP Server
The Plugin Model Analyzer MCP server (`build/mcp-servers/module-analyzer`) uses this JSON endpoint to provide:
- `analyze_module_structure` - Complete module system analysis
- `get_module_info` - Query specific module details
- `find_module_paths` - Trace module to product paths
- `get_module_set_hierarchy` - Query module set relationships
- `list_products` - List products filtered by criteria
- `validate_community_products` - Ensure community/ultimate separation
### Implementation
The JSON generation is implemented in:
- `ModuleSetRunner.kt` - Orchestration and CLI parsing
- `ModuleSetJsonExport.kt` - JSON generation logic
- `ModuleSetDiscovery.kt` - Module set discovery via reflection
## Benefits
1. **Type safety**: Kotlin code with IDE support (autocomplete, refactoring)
@@ -290,6 +457,7 @@ Use `ultimateOnly = true` when:
3. **Single source of truth**: One Kotlin definition for both dev and non-dev modes
4. **Maintainability**: Easier to see what modules a product includes
5. **VCS-friendly**: Static files work without dev mode infrastructure
6. **Programmatic access**: JSON endpoint enables tooling and automation
## See Also
@@ -0,0 +1,152 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.productLayout
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
/**
* Builds content blocks and module-to-set chain mapping in a single hierarchical traversal.
* This optimized version eliminates redundant tree walking by computing all results simultaneously.
*
* @param spec The product modules specification
* @param collectModuleSetAliases Whether to collect module set aliases during traversal (for inlineModuleSets mode)
* @return Triple of (content blocks, module-to-set chain mapping, module set alias to source mapping)
*/
internal fun buildContentBlocksAndChainMapping(
spec: ProductModulesContentSpec,
collectModuleSetAliases: Boolean = false
): Triple<List<ContentBlock>, Map<String, List<String>>, Map<String, String>> {
val contentBlocks = mutableListOf<ContentBlock>()
val moduleToChain = mutableMapOf<String, List<String>>()
val moduleToSets = mutableMapOf<String, MutableList<String>>()
val aliasToSource = if (collectModuleSetAliases) mutableMapOf<String, String>() else null
val processedSets = HashSet<String>()
val contentBlockByName = mutableMapOf<String, ContentBlock>()
fun traverse(moduleSet: ModuleSet, chain: List<String>, overrides: Map<String, ModuleLoadingRule>) {
val setName = "$MODULE_SET_PREFIX${moduleSet.name}"
// Check if already processed
val alreadyProcessed = !processedSets.add(setName)
if (alreadyProcessed) {
// If already processed but now we have overrides, update the existing content block
if (overrides.isNotEmpty()) {
val existingBlock = contentBlockByName[moduleSet.name]
if (existingBlock != null) {
// Check if existing block has any loading attributes
val hasExistingOverrides = existingBlock.modules.any { it.loading != null }
if (!hasExistingOverrides) {
// Reuse the filtered module list from existing block (already filtered by first pass)
val updatedModules = mutableListOf<ModuleWithLoading>()
for (existingModule in existingBlock.modules) {
val effectiveLoading = overrides[existingModule.name] ?: existingModule.loading
updatedModules.add(ModuleWithLoading(existingModule.name, effectiveLoading))
}
// Create new content block with overrides and replace the old one
val updatedBlock = ContentBlock(moduleSet.name, updatedModules)
val oldBlockIndex = contentBlocks.indexOf(existingBlock)
if (oldBlockIndex >= 0) {
contentBlocks[oldBlockIndex] = updatedBlock
contentBlockByName[moduleSet.name] = updatedBlock
}
// Don't re-add to moduleToSets or moduleToChain - already there from first pass
}
// else: Both have overrides, keep the first one (shouldn't happen in practice)
}
}
// Already processed, don't reprocess
return
}
// Collect module set alias if requested
if (aliasToSource != null && moduleSet.alias != null) {
validateAndRecordAlias(
alias = moduleSet.alias,
source = "module set '${moduleSet.name}'",
aliasToSource = aliasToSource
)
}
val currentChain = chain + setName
// Get direct modules for this set
val directModules = getDirectModules(moduleSet, spec.excludedModules)
// Build content block and track chains/duplicates in single pass
val modulesWithLoading = mutableListOf<ModuleWithLoading>()
for (module in directModules) {
// Track for duplicate detection
moduleToSets.computeIfAbsent(module.name) { mutableListOf() }.add(moduleSet.name)
// Track chain
moduleToChain[module.name] = currentChain
// Build loading info - apply overrides from module set
val effectiveLoading = overrides[module.name] ?: module.loading
modulesWithLoading.add(ModuleWithLoading(module.name, effectiveLoading))
}
if (modulesWithLoading.isNotEmpty()) {
val block = ContentBlock(moduleSet.name, modulesWithLoading)
contentBlocks.add(block)
contentBlockByName[moduleSet.name] = block
}
// Recursively process nested sets (no override cascading - each set must be referenced directly for overrides)
for (nestedSet in moduleSet.nestedSets) {
traverse(nestedSet, currentChain, emptyMap())
}
}
// Process all top-level module sets
for (moduleSetWithOverrides in spec.moduleSets) {
traverse(moduleSetWithOverrides.moduleSet, emptyList(), moduleSetWithOverrides.loadingOverrides)
}
// Validate that all overridden modules exist as direct modules in their respective module sets
for (moduleSetWithOverrides in spec.moduleSets) {
validateModuleSetOverrides(moduleSetWithOverrides, spec)
}
// Check for duplicates and FAIL if found
validateNoDuplicateModules(moduleToSets)
// Add additional modules if any
val additionalModulesWithLoading = mutableListOf<ModuleWithLoading>()
for (module in spec.additionalModules) {
if (module.name !in spec.excludedModules) {
additionalModulesWithLoading.add(ModuleWithLoading(module.name, module.loading))
}
}
if (additionalModulesWithLoading.isNotEmpty()) {
contentBlocks.add(ContentBlock(ADDITIONAL_MODULES_BLOCK, additionalModulesWithLoading))
}
return Triple(contentBlocks, moduleToChain, aliasToSource ?: emptyMap())
}
/**
* Collects and validates product-level aliases and merges with module set aliases.
* Checks for duplicates between product-level and module set aliases.
*
* @param spec The product modules specification
* @param moduleSetAliases Aliases collected from module sets during traversal
* @return List of validated unique aliases
*/
internal fun collectAndValidateAliases(
spec: ProductModulesContentSpec,
moduleSetAliases: Map<String, String>
): List<String> {
val allAliases = moduleSetAliases.toMutableMap()
// Collect product-level aliases and check for conflicts with module set aliases
for (alias in spec.productModuleAliases) {
validateAndRecordAlias(
alias = alias,
source = "product level",
aliasToSource = allAliases
)
}
return allAliases.keys.sorted()
}
@@ -0,0 +1,288 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.productLayout
import com.intellij.openapi.util.JDOMUtil
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import org.jdom.Element
import java.nio.file.Path
import kotlin.io.path.exists
import kotlin.io.path.isRegularFile
import kotlin.io.path.readText
/**
* Detects duplicate xi:include references in product plugin.xml files.
* Analyzes both direct xi:includes and nested includes from deprecatedInclude XML files.
*/
object DuplicateIncludeDetector {
/**
* Detects duplicate xi:include references across all provided product XML files.
*
* @param productXmlFiles List of product plugin.xml file paths to analyze
* @param projectRoot Project root directory for resolving relative paths
* @return Report containing all detected duplicates
*/
fun detectDuplicates(
productXmlFiles: List<Path>,
projectRoot: Path
): DuplicateIncludesReport {
val productsWithDuplicates = mutableListOf<ProductDuplicates>()
for (productFile in productXmlFiles) {
if (!productFile.exists() || !productFile.isRegularFile()) {
continue
}
val allIncludes = mutableListOf<Pair<String, IncludeSource>>()
// 1. Parse direct xi:includes from product file
val directIncludes = parseXiIncludes(productFile)
for (include in directIncludes) {
allIncludes.add(include.href to IncludeSource(
file = productFile.toString(),
sourceType = "xi:include",
lineNumber = include.lineNumber
))
// 2. If this xi:include points to a potential deprecatedInclude XML file, parse it too
val resolvedPath = resolveIncludePath(include.href, productFile, projectRoot)
if (resolvedPath != null && resolvedPath.exists() && isDeprecatedIncludeFile(resolvedPath)) {
val nestedIncludes = parseXiIncludes(resolvedPath)
for (nested in nestedIncludes) {
allIncludes.add(nested.href to IncludeSource(
file = resolvedPath.toString(),
sourceType = "deprecatedInclude-nested",
lineNumber = nested.lineNumber
))
}
}
}
// 3. Find duplicates
val grouped = allIncludes.groupBy { it.first }
val duplicates = grouped.filter { it.value.size > 1 }.map { (href, sources) ->
DuplicateInclude(
href = href,
resolvedPath = resolveIncludePath(href, productFile, projectRoot)?.toString(),
count = sources.size,
sources = sources.map { it.second }
)
}
if (duplicates.isNotEmpty()) {
productsWithDuplicates.add(ProductDuplicates(
name = productFile.fileName.toString().removeSuffix(".xml"),
file = productFile.toString(),
duplicates = duplicates
))
}
}
return DuplicateIncludesReport(
timestamp = System.currentTimeMillis().toString(),
products = productsWithDuplicates,
summary = DuplicateSummary(
totalProducts = productXmlFiles.size,
productsWithDuplicates = productsWithDuplicates.size,
totalDuplicateFiles = productsWithDuplicates.sumOf { it.duplicates.size }
)
)
}
/**
* Parses xi:include elements from an XML file.
* Returns list of XiInclude objects with href and line number.
*/
private fun parseXiIncludes(xmlFile: Path): List<XiInclude> {
val includes = mutableListOf<XiInclude>()
try {
val content = xmlFile.readText()
val root = JDOMUtil.load(xmlFile)
// Find all xi:include elements (recursively)
val includeElements = findElementsByName(root, "include")
for (element in includeElements) {
val href = element.getAttributeValue("href")
if (!href.isNullOrEmpty()) {
// Try to find line number by searching in the file content
val lineNumber = findLineNumber(content, href)
includes.add(XiInclude(href, lineNumber))
}
}
}
catch (e: Exception) {
// Ignore parsing errors
}
return includes
}
/**
* Recursively finds all elements with the given local name (ignoring namespace).
*/
private fun findElementsByName(element: Element, localName: String): List<Element> {
val result = mutableListOf<Element>()
// Check if current element matches (ignoring namespace prefix)
if (element.name == localName || element.name.endsWith(":$localName")) {
result.add(element)
}
// Recursively search children
for (child in element.children) {
result.addAll(findElementsByName(child, localName))
}
return result
}
/**
* Attempts to find the line number where an href appears in the XML content.
*/
private fun findLineNumber(content: String, href: String): Int? {
val lines = content.lines()
for ((index, line) in lines.withIndex()) {
if (line.contains("href=\"$href\"")) {
return index + 1 // 1-based line numbers
}
}
return null
}
/**
* Resolves an xi:include href to an absolute file path.
*/
private fun resolveIncludePath(href: String, currentFile: Path, projectRoot: Path): Path? {
if (href.startsWith("/META-INF/")) {
// Absolute path - search in known resource locations
val locations = listOf(
projectRoot.resolve("community/platform/platform-resources/src$href"),
projectRoot.resolve("community/platform/platform-resources/generated$href"),
projectRoot.resolve("community/java/ide-resources/resources$href"),
projectRoot.resolve("ultimate/platform-ultimate/resources$href"),
projectRoot.resolve("licenseCommon/resources$href"),
projectRoot.resolve("licenseCommon/generated$href"),
// Also check CIDR and other products
projectRoot.resolve("CIDR/clion/main/nolang/resources$href"),
projectRoot.resolve("goland/resources$href"),
projectRoot.resolve("ruby/resources$href"),
projectRoot.resolve("WebStorm/resources$href"),
projectRoot.resolve("dbe/ide/resources$href"),
projectRoot.resolve("aqua/branding/resources$href"),
projectRoot.resolve("rider/resources$href"),
)
for (loc in locations) {
if (loc.exists()) {
return loc
}
}
return null
}
else if (href.startsWith("/")) {
// Handle paths like /something.xml
return resolveIncludePath("/META-INF$href", currentFile, projectRoot)
}
else {
// Relative path
val dir = currentFile.parent
return dir?.resolve(href)
}
}
/**
* Checks if the given path is a deprecatedInclude XML file that might contain nested xi:includes.
*/
private fun isDeprecatedIncludeFile(path: Path): Boolean {
val fileName = path.fileName.toString()
return fileName.endsWith("-customization.xml") ||
fileName == "ultimate.xml" ||
fileName == "PlatformLangPlugin.xml" ||
fileName == "JavaIdePlugin.xml" ||
fileName == "structuralsearch.xml" ||
fileName.endsWith("Plugin.xml")
}
/**
* Represents a single xi:include element.
*/
private data class XiInclude(
val href: String,
val lineNumber: Int?
)
}
/**
* Detects and prints duplicate xi:include elements in product plugin.xml files.
* This is a convenience function that takes discovered products and outputs JSON report.
*
* @param products List of discovered products with their plugin XML paths
* @param projectRoot Project root directory for resolving relative paths
*/
fun detectAndPrintDuplicateIncludes(products: List<ProductSpec>, projectRoot: Path) {
// Convert ProductSpec plugin paths to actual file paths
val productFiles = products
.mapNotNull { it.pluginXmlPath }
.map { projectRoot.resolve(it) }
.filter { it.exists() && it.isRegularFile() }
// Run detection
val report = DuplicateIncludeDetector.detectDuplicates(productFiles, projectRoot)
// Output JSON
val json = Json { prettyPrint = true }
println(json.encodeToString(report))
}
/**
* Report containing all detected duplicate includes.
*/
@Serializable
data class DuplicateIncludesReport(
val timestamp: String,
val products: List<ProductDuplicates>,
val summary: DuplicateSummary
)
/**
* Duplicates found in a single product.
*/
@Serializable
data class ProductDuplicates(
val name: String,
val file: String,
val duplicates: List<DuplicateInclude>
)
/**
* A single duplicate include with its sources.
*/
@Serializable
data class DuplicateInclude(
val href: String,
val resolvedPath: String?,
val count: Int,
val sources: List<IncludeSource>
)
/**
* Source of an include (where it was found).
*/
@Serializable
data class IncludeSource(
val file: String,
val sourceType: String, // "xi:include" or "deprecatedInclude-nested"
val lineNumber: Int?
)
/**
* Summary statistics.
*/
@Serializable
data class DuplicateSummary(
val totalProducts: Int,
val productsWithDuplicates: Int,
val totalDuplicateFiles: Int
)
@@ -0,0 +1,64 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.productLayout
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
// Constants for magic strings used throughout the generator
internal const val ADDITIONAL_MODULES_BLOCK = "additional"
internal const val JETBRAINS_NAMESPACE = "jetbrains"
internal const val GENERATOR_SUFFIX = ".main()"
internal const val MODULE_SET_PREFIX = "intellij.moduleSets."
/**
* Type-safe wrapper for module set names.
* Prevents accidental mixing of module set names with other string types.
*/
@JvmInline
value class ModuleSetName(val value: String) {
override fun toString(): String = value
}
/**
* Represents a single content block in a product plugin.xml.
* Each block corresponds to a module set or additional modules section.
*/
data class ContentBlock(
/** Source identifier for the block (e.g., "essential", "vcs", "additional") */
@JvmField val source: String,
/** List of modules with their effective loading modes */
@JvmField val modules: List<ModuleWithLoading>,
)
/**
* A module with its effective loading mode after applying overrides and exclusions.
*/
data class ModuleWithLoading(
/** Module name */
@JvmField val name: String,
/** Effective loading mode (null means default/no attribute) */
@JvmField val loading: ModuleLoadingRule?,
)
/**
* Result of building product content XML.
* Contains the generated XML string, content blocks, and module-to-set chain mapping.
*/
data class ProductContentBuildResult(
/** Generated XML content as string */
@JvmField val xml: String,
/** List of content blocks generated from the spec */
@JvmField val contentBlocks: List<ContentBlock>,
/** Mapping from module name to its module set chain as list (e.g., ["parent", "child"]) */
@JvmField val moduleToSetChainMapping: Map<String, List<String>>,
)
/**
* Result of building module set XML.
* Contains the XML string and count of direct modules (excluding nested).
*/
internal data class ModuleSetBuildResult(
@JvmField val xml: String,
@JvmField val directModuleCount: Int,
)
@@ -0,0 +1,55 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplacePutWithAssignment")
package org.jetbrains.intellij.build.productLayout
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
import kotlinx.serialization.Serializable
/**
* Wrapper for a module set with optional loading overrides.
* When overrides are present, the module set will be inlined in the product XML
* instead of being referenced via xi:include.
*/
@Serializable
data class ModuleSetWithOverrides(
@JvmField val moduleSet: ModuleSet,
@JvmField val loadingOverrides: Map<String, ModuleLoadingRule> = emptyMap(),
) {
val hasOverrides: Boolean
get() = loadingOverrides.isNotEmpty()
}
/**
* DSL builder for module loading overrides within a module set.
*/
@ProductDslMarker
class ModuleLoadingOverrideBuilder {
private val overrides = HashMap<String, ModuleLoadingRule>()
/**
* Override a module in this module set to be loaded as embedded (loading="embedded").
*/
fun overrideAsEmbedded(moduleName: String) {
overrides.put(moduleName, ModuleLoadingRule.EMBEDDED)
}
/**
* Override a module in this module set to be loaded as required (loading="required").
*/
fun overrideAsRequired(moduleName: String) {
overrides.put(moduleName, ModuleLoadingRule.REQUIRED)
}
/**
* Set custom loading rule for modules.
*/
fun loading(rule: ModuleLoadingRule, vararg moduleNames: String) {
for (name in moduleNames) {
overrides.put(name, rule)
}
}
@PublishedApi
internal fun build(): Map<String, ModuleLoadingRule> = overrides.toMap()
}
@@ -2,12 +2,23 @@
package org.jetbrains.intellij.build.productLayout
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
import kotlinx.serialization.Serializable
import org.jetbrains.intellij.build.BuildPaths
import java.lang.invoke.MethodHandles
import java.lang.invoke.MethodType
import java.nio.file.Files
import java.nio.file.Path
/**
* Represents a content module with optional loading attribute.
*
* @param name Module name
* @param loading Optional loading mode (e.g., ModuleLoadingRule.EMBEDDED)
*/
@Serializable
data class ContentModule(
@JvmField val name: String,
@JvmField val loading: ModuleLoadingRule? = null,
)
/**
* Represents a named collection of content modules.
* The name serves as metadata for debugging and XML generation (used in the 'source' attribute).
@@ -17,6 +28,7 @@ import java.nio.file.Path
* @param nestedSets List of nested module sets (for xi:include generation)
* @param alias Optional module alias for `<module value="..."/>` declaration (e.g., "com.intellij.modules.xml")
*/
@Serializable
data class ModuleSet(
@JvmField val name: String,
@JvmField val modules: List<ContentModule>,
@@ -38,6 +50,7 @@ interface ModuleSetProvider {
/**
* DSL builder for creating ModuleSets with reduced boilerplate.
*/
@ProductDslMarker
class ModuleSetBuilder {
private val modules = mutableListOf<ContentModule>()
private val nestedSets = mutableListOf<ModuleSet>()
@@ -146,15 +159,6 @@ private fun appendModuleSetContent(sb: StringBuilder, moduleSet: ModuleSet, inde
}
}
/**
* Result of building module set XML.
* Contains the XML string and count of direct modules (excluding nested).
*/
data class ModuleSetBuildResult(
val xml: String,
val directModuleCount: Int,
)
/**
* Builds the XML content for a module set.
*
@@ -202,32 +206,6 @@ internal fun buildModuleSetXml(moduleSet: ModuleSet, label: String): ModuleSetBu
return ModuleSetBuildResult(xml, directModuleCount)
}
/**
* Discovers all module set functions in the given object using reflection.
* Returns all public functions that:
* - Return ModuleSet
* - Take no parameters
* - Are not named 'main'
*
* @param obj The object to scan for module set functions (e.g., CommunityModuleSets, UltimateModuleSets)
* @return List of all discovered ModuleSets
*/
private fun discoverModuleSets(obj: Any): List<ModuleSet> {
val lookup = MethodHandles.lookup()
val clazz = obj.javaClass
val methodType = MethodType.methodType(ModuleSet::class.java)
val declaredMethods = clazz.declaredMethods
val result = ArrayList<ModuleSet>(declaredMethods.size)
for (method in declaredMethods) {
if (method.parameterCount == 0 && java.lang.reflect.Modifier.isPublic(method.modifiers) && method.returnType == ModuleSet::class.java) {
val moduleSet = lookup.findVirtual(clazz, method.name, methodType).invoke(obj) as ModuleSet
result.add(moduleSet)
}
}
return result
}
/**
* Generates all module set XMLs for the given object.
* Discovers all ModuleSet functions via reflection, generates XML files, and prints results.
@@ -0,0 +1,29 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.productLayout
import java.lang.invoke.MethodHandles
import java.lang.invoke.MethodType
/**
* Discovers all module sets from an object using reflection.
* Returns all public functions that return ModuleSet and take no parameters.
*
* This consolidates the duplicated discovery logic that was in both UltimateModuleSets and ultimateGenerator.
*/
fun discoverModuleSets(provider: Any): List<ModuleSet> {
val lookup = MethodHandles.lookup()
val clazz = provider.javaClass
val methodType = MethodType.methodType(ModuleSet::class.java)
val declaredMethods = clazz.declaredMethods
val result = ArrayList<ModuleSet>(declaredMethods.size)
for (method in declaredMethods) {
if (method.parameterCount == 0 &&
java.lang.reflect.Modifier.isPublic(method.modifiers) &&
method.returnType == ModuleSet::class.java) {
val moduleSet = lookup.findVirtual(clazz, method.name, methodType).invoke(provider) as ModuleSet
result.add(moduleSet)
}
}
return result
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,182 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.productLayout
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
/**
* Determines if a module set needs to be inlined (cannot use xi:include).
* Inlining is required when:
* - The module set has loading overrides for its direct modules
* - The module set contains nested sets that are overridden at top level
*
* @return true if the module set must be inlined, false if xi:include can be used
*/
internal fun shouldInlineModuleSet(
moduleSet: ModuleSet,
overrides: Map<String, ModuleLoadingRule>,
overriddenModuleSetNames: Set<ModuleSetName>
): Boolean {
return overrides.isNotEmpty() || containsOverriddenNestedSet(moduleSet, overriddenModuleSetNames)
}
/**
* Appends inlined module set content with direct modules and nested set processing.
* Used when xi:include cannot be used due to overrides or nested set conflicts.
*/
internal fun StringBuilder.appendInlinedModuleSet(
moduleSet: ModuleSet,
overrides: Map<String, ModuleLoadingRule>,
contentBlocks: List<ContentBlock>,
overriddenModuleSetNames: Set<ModuleSetName>
) {
val hasOverrides = overrides.isNotEmpty()
val directBlock = contentBlocks.find { it.source == moduleSet.name }
// Append direct modules if present
if (directBlock != null && directBlock.modules.isNotEmpty()) {
val label = if (hasOverrides) moduleSet.name else "${moduleSet.name} (selective inline)"
withEditorFold(this, " ", label) {
append(" <content namespace=\"$JETBRAINS_NAMESPACE\">\n")
// Add explanatory comment
if (hasOverrides) {
val overriddenModules = overrides.entries
.sortedBy { it.key }
.joinToString(", ") { "${it.key}=${it.value.name.lowercase().replace('_', '-')}" }
append(" <!-- Module set '${moduleSet.name}' with ${overrides.size} loading override(s): $overriddenModules -->\n")
}
else {
val overriddenNested = findOverriddenNestedSetNames(moduleSet, overriddenModuleSetNames)
val nestedNames = overriddenNested.joinToString(", ") { "'$it'" }
append(" <!-- Module set '${moduleSet.name}' selectively inlined - nested set(s) $nestedNames referenced separately with overrides -->\n")
}
// Append modules with effective overrides
for (module in directBlock.modules) {
val effectiveLoading = overrides[module.name] ?: module.loading
appendModuleLine(ModuleWithLoading(module.name, effectiveLoading), " ")
}
append(" </content>\n")
}
}
// Process nested sets recursively
appendNestedModuleSets(moduleSet.nestedSets, contentBlocks, overriddenModuleSetNames)
}
/**
* Appends nested module sets with appropriate handling (inline or xi:include).
*/
internal fun StringBuilder.appendNestedModuleSets(
nestedSets: List<ModuleSet>,
contentBlocks: List<ContentBlock>,
overriddenModuleSetNames: Set<ModuleSetName>
) {
for (nestedSet in nestedSets) {
val nestedSetName = ModuleSetName(nestedSet.name)
when {
nestedSetName in overriddenModuleSetNames -> {
// Skip - will be processed separately at top level with its overrides
append(" <!-- Nested set '${nestedSet.name}' is referenced at top-level with overrides, skipping xi:include here -->\n")
}
containsOverriddenNestedSet(nestedSet, overriddenModuleSetNames) -> {
// Recursively inline - contains overridden nested sets
appendModuleSetXml(nestedSet, emptyMap(), contentBlocks, overriddenModuleSetNames)
}
else -> {
// Safe to use xi:include
appendModuleSetInclude(nestedSet.name)
}
}
}
}
/**
* Appends a simple xi:include directive for a module set.
*/
internal fun StringBuilder.appendModuleSetInclude(moduleSetName: String) {
append(" <xi:include href=\"/META-INF/$MODULE_SET_PREFIX$moduleSetName.xml\"/>\n")
}
/**
* Recursively generates XML content for a module set, applying selective inlining when necessary.
*
* When a module set has overrides or contains overridden nested sets, it cannot use xi:include
* (which would lose the overrides). Instead, we inline the direct modules and generate
* xi:include directives for non-overridden nested sets.
*
* @param moduleSet The module set to generate content for
* @param overrides Loading rule overrides for direct modules
* @param contentBlocks Pre-computed content blocks containing modules with applied rules
* @param overriddenModuleSetNames Names of module sets that are overridden at top-level
*/
internal fun StringBuilder.appendModuleSetXml(
moduleSet: ModuleSet,
overrides: Map<String, ModuleLoadingRule>,
contentBlocks: List<ContentBlock>,
overriddenModuleSetNames: Set<ModuleSetName>
) {
if (shouldInlineModuleSet(moduleSet, overrides, overriddenModuleSetNames)) {
appendInlinedModuleSet(moduleSet, overrides, contentBlocks, overriddenModuleSetNames)
}
else {
appendModuleSetInclude(moduleSet.name)
}
}
/**
* Appends a single module XML element with optional loading attribute.
*/
internal fun StringBuilder.appendModuleLine(moduleWithLoading: ModuleWithLoading, indent: String = " ") {
append("$indent<module name=\"${moduleWithLoading.name}\"")
if (moduleWithLoading.loading != null) {
// convert enum to lowercase with hyphens (e.g., ON_DEMAND -> on-demand)
append(" loading=\"${moduleWithLoading.loading.name.lowercase().replace('_', '-')}\"")
}
append("/>\n")
}
/**
* Appends a content block with modules wrapped in editor fold.
*/
internal fun StringBuilder.appendContentBlock(
blockSource: String,
modules: List<ModuleWithLoading>,
indent: String = " ",
) {
withEditorFold(sb = this, indent = indent, description = blockSource) {
append("$indent<content namespace=\"$JETBRAINS_NAMESPACE\">\n")
for (module in modules) {
appendModuleLine(module, "$indent ")
}
append("$indent</content>\n")
}
}
/**
* Appends module set loading strategy comment when there are overrides.
*/
internal fun StringBuilder.appendModuleSetsStrategyComment(
spec: ProductModulesContentSpec,
overriddenModuleSetNames: Set<ModuleSetName>
) {
if (overriddenModuleSetNames.isEmpty()) return
append(" <!-- Module Set Loading Strategy:\n")
for (moduleSetWithOverrides in spec.moduleSets) {
if (moduleSetWithOverrides.hasOverrides) {
val overrideCount = moduleSetWithOverrides.loadingOverrides.size
append(" - '${moduleSetWithOverrides.moduleSet.name}': $overrideCount module(s) with loading overrides\n")
}
else if (containsOverriddenNestedSet(moduleSetWithOverrides.moduleSet, overriddenModuleSetNames)) {
val overriddenNested = findOverriddenNestedSetNames(moduleSetWithOverrides.moduleSet, overriddenModuleSetNames)
val nestedNames = overriddenNested.joinToString(", ") { "'$it'" }
append(" - '${moduleSetWithOverrides.moduleSet.name}': selectively inlined (contains overridden nested set(s): $nestedNames)\n")
}
}
append(" -->\n")
}
@@ -4,6 +4,14 @@
package org.jetbrains.intellij.build.productLayout
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
import kotlinx.serialization.Serializable
/**
* Marker annotation for the product DSL to prevent implicit receiver scope leakage.
* This ensures that methods from outer DSL scopes are not accidentally accessible in nested blocks.
*/
@DslMarker
annotation class ProductDslMarker
/**
* Represents an XML include directive that references a resource within a module.
@@ -13,6 +21,7 @@ import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
* @param ultimateOnly If true, this include is only processed in Ultimate builds (skipped in Community builds)
* @param optional If true, this include is always generated with xi:fallback and never inlined (safe for files that may not exist)
*/
@Serializable
data class DeprecatedXmlInclude(
@JvmField val moduleName: String,
@JvmField val resourcePath: String,
@@ -20,6 +29,52 @@ data class DeprecatedXmlInclude(
@JvmField val optional: Boolean = false,
)
/**
* Tracks how a product spec is composed (e.g., via include(), moduleSet(), etc.).
* This enables tracing module origins and detecting redundancies.
*/
@Serializable
data class SpecComposition(
/** Type of composition (inline spec, module set reference, or deprecated XML include) */
@JvmField val type: CompositionType,
/** Reference name (module set name or function name) if applicable */
@JvmField val reference: String? = null,
/** Breadcrumb trail showing the composition path */
@JvmField val path: List<String> = emptyList(),
/** Source location where this composition was added (file:line) */
@JvmField val sourceLocation: String? = null,
)
/**
* Type of composition operation that added content to a product spec.
*/
@Serializable
enum class CompositionType {
/** include(spec) - embedded content from another ProductModulesContentSpec */
INLINE_SPEC,
/** moduleSet(...) - reference to a module set */
MODULE_SET_REF,
/** deprecatedInclude(...) - reference to an XML file */
DEPRECATED_XML,
/** module() / embeddedModule() / requiredModule() - direct module addition */
DIRECT_MODULE,
/** alias() - module alias addition */
ALIAS,
}
/**
* Metadata about a product spec's origin for traceability.
*/
@Serializable
data class SpecMetadata(
/** Name of the product or function that created this spec */
@JvmField val name: String? = null,
/** Source file path relative to project root */
@JvmField val sourceFile: String? = null,
/** Function name that created this spec */
@JvmField val sourceFunction: String? = null,
)
/**
* Specification for programmatically defining product content modules.
* This allows products to specify module sets, individual modules, and XML includes in Kotlin code
@@ -32,6 +87,7 @@ data class DeprecatedXmlInclude(
*
* @see org.jetbrains.intellij.build.ProductProperties.getProductContentDescriptor
*/
@Serializable
class ProductModulesContentSpec(
/**
* Product module aliases for `<module value="..."/>` declarations (e.g., "com.jetbrains.gateway", "com.intellij.modules.idea").
@@ -48,10 +104,10 @@ class ProductModulesContentSpec(
@JvmField val deprecatedXmlIncludes: List<DeprecatedXmlInclude>,
/**
* Module sets to include. Each set contains a named collection of modules.
* Module sets are processed in order and can overlap (duplicates are handled).
* Module sets to include with optional loading overrides.
* When a module set has overrides, it will be inlined in XML instead of using xi:include.
*/
@JvmField val moduleSets: List<ModuleSet>,
@JvmField val moduleSets: List<ModuleSetWithOverrides>,
/**
* Additional individual modules to include beyond those from module sets.
@@ -66,23 +122,36 @@ class ProductModulesContentSpec(
@JvmField val excludedModules: Set<String>,
/**
* Loading attribute overrides for specific modules.
* Map of module name to loading mode (e.g., [ModuleLoadingRule.EMBEDDED]).
* This allows changing the loading mode of modules from module sets.
* Composition graph tracking how this spec was assembled.
* Records all include(), moduleSet(), and other composition operations.
* Used for analysis, tracing, and redundancy detection.
*/
@JvmField val moduleLoadingOverrides: Map<String, ModuleLoadingRule>,
@JvmField val compositionGraph: List<SpecComposition> = emptyList(),
/**
* Metadata about this spec's origin (product name, source file, function).
* Useful for debugging and tracing where a spec was created.
*/
@JvmField val metadata: SpecMetadata? = null,
)
/**
* DSL builder for creating ProductModulesContentSpec with reduced boilerplate.
*/
@ProductDslMarker
class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
private val productModuleAliases = mutableListOf<String>()
private val xmlIncludes = mutableListOf<DeprecatedXmlInclude>()
private val moduleSets = mutableListOf<ModuleSet>()
private val moduleSets = mutableListOf<ModuleSetWithOverrides>()
private val additionalModules = mutableListOf<ContentModule>()
private val excludedModules = mutableSetOf<String>()
private val loadingOverrides = mutableMapOf<String, ModuleLoadingRule>()
// Composition tracking
private val compositionGraph = mutableListOf<SpecComposition>()
private val pathStack = mutableListOf<String>() // Current path for nested compositions
// Metadata for this spec
internal var metadata: SpecMetadata? = null
/**
* Add a product module alias for `<module value="..."/>` declaration.
@@ -92,6 +161,47 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
*/
fun alias(value: String) {
productModuleAliases.add(value)
compositionGraph.add(SpecComposition(
type = CompositionType.ALIAS,
reference = value,
path = pathStack.toList(),
sourceLocation = null // TODO: capture if needed
))
}
/**
* Include another ProductModulesContentSpec, merging all its contents into this builder.
* This enables composition of product spec fragments for reuse across products.
*
* Example:
* ```
* override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules {
* include(commonCapabilityAliases()) // include spec fragment with common aliases
* include(platformCommonIncludes()) // include spec fragment with deprecatedIncludes
* moduleSet(commercialIdeBase()) // include module set
* }
* ```
*
* @param spec The ProductModulesContentSpec to merge into this builder
*/
fun include(spec: ProductModulesContentSpec) {
// Record composition before flattening (for analysis)
compositionGraph.add(SpecComposition(
type = CompositionType.INLINE_SPEC,
reference = spec.metadata?.name ?: spec.metadata?.sourceFunction,
path = pathStack.toList(),
sourceLocation = spec.metadata?.sourceFile
))
// Flatten content (existing behavior for backward compatibility)
productModuleAliases.addAll(spec.productModuleAliases)
xmlIncludes.addAll(spec.deprecatedXmlIncludes)
moduleSets.addAll(spec.moduleSets)
additionalModules.addAll(spec.additionalModules)
excludedModules.addAll(spec.excludedModules)
// Also preserve the nested spec's composition graph for deep analysis
compositionGraph.addAll(spec.compositionGraph)
}
/**
@@ -116,13 +226,48 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
*/
fun deprecatedInclude(moduleName: String, resourcePath: String, ultimateOnly: Boolean = false, optional: Boolean = false) {
xmlIncludes.add(DeprecatedXmlInclude(moduleName, resourcePath, ultimateOnly, optional))
compositionGraph.add(SpecComposition(
type = CompositionType.DEPRECATED_XML,
reference = "$moduleName:$resourcePath",
path = pathStack.toList(),
sourceLocation = null
))
}
/**
* Add a module set.
* Add a module set without loading overrides.
*/
fun moduleSet(set: ModuleSet) {
moduleSets.add(set)
addModuleSet(set = set, overrides = emptyMap())
}
/**
* Add a module set with loading overrides for specific modules.
* When overrides are provided, the module set will be inlined in the product XML
* instead of being referenced via xi:include.
*
* Example:
* ```
* moduleSet(UltimateModuleSets.commercialIdeBase()) {
* overrideAsEmbedded("intellij.rd.platform")
* overrideAsEmbedded("intellij.rd.ui")
* overrideAsRequired("intellij.some.module")
* }
* ```
*/
inline fun moduleSet(set: ModuleSet, block: ModuleLoadingOverrideBuilder.() -> Unit) {
addModuleSet(set, ModuleLoadingOverrideBuilder().apply(block).build())
}
@PublishedApi
internal fun addModuleSet(set: ModuleSet, overrides: Map<String, ModuleLoadingRule>) {
moduleSets.add(ModuleSetWithOverrides(set, overrides))
compositionGraph.add(SpecComposition(
type = CompositionType.MODULE_SET_REF,
reference = set.name,
path = pathStack.toList(),
sourceLocation = null
))
}
/**
@@ -130,6 +275,12 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
*/
fun module(name: String, loading: ModuleLoadingRule? = null) {
additionalModules.add(ContentModule(name, loading))
compositionGraph.add(SpecComposition(
type = CompositionType.DIRECT_MODULE,
reference = name,
path = pathStack.toList(),
sourceLocation = null
))
}
/**
@@ -137,6 +288,12 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
*/
fun embeddedModule(name: String) {
additionalModules.add(ContentModule(name, ModuleLoadingRule.EMBEDDED))
compositionGraph.add(SpecComposition(
type = CompositionType.DIRECT_MODULE,
reference = name,
path = pathStack.toList(),
sourceLocation = null
))
}
/**
@@ -144,6 +301,12 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
*/
fun requiredModule(name: String) {
additionalModules.add(ContentModule(name, ModuleLoadingRule.REQUIRED))
compositionGraph.add(SpecComposition(
type = CompositionType.DIRECT_MODULE,
reference = name,
path = pathStack.toList(),
sourceLocation = null
))
}
/**
@@ -153,13 +316,6 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
excludedModules.add(moduleName)
}
/**
* Override the loading mode for a specific module.
*/
fun override(moduleName: String, loading: ModuleLoadingRule) {
loadingOverrides.put(moduleName, loading)
}
@PublishedApi
internal fun build(): ProductModulesContentSpec {
return ProductModulesContentSpec(
@@ -168,7 +324,8 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
moduleSets = java.util.List.copyOf(moduleSets),
additionalModules = java.util.List.copyOf(additionalModules),
excludedModules = java.util.Set.copyOf(excludedModules),
moduleLoadingOverrides = java.util.Map.copyOf(loadingOverrides),
compositionGraph = java.util.List.copyOf(compositionGraph),
metadata = metadata,
)
}
}
@@ -191,9 +348,8 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() {
* // Individual modules
* embeddedModule("com.example.additional")
*
* // Exclusions and overrides
* // Exclusions
* exclude("unwanted.module")
* override("some.module", ModuleLoadingRule.OPTIONAL)
* }
* }
* ```
@@ -0,0 +1,118 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.productLayout
import com.intellij.openapi.util.JDOMUtil
import org.jetbrains.intellij.build.ModuleOutputProvider
import org.jetbrains.intellij.build.findFileInModuleLibraryDependencies
import org.jetbrains.intellij.build.findFileInModuleSources
import org.jetbrains.intellij.build.isModuleNameLikeFilename
/**
* Appends XML header comments.
*/
internal fun StringBuilder.appendXmlHeader(generatorCommand: String, productPropertiesClass: String) {
append(" <!-- DO NOT EDIT: This file is auto-generated from Kotlin code -->\n")
append(" <!-- To regenerate, run 'Generate Product Layouts' or directly $generatorCommand -->\n")
append(" <!-- Source: $productPropertiesClass -->\n")
}
/**
* Appends the opening <idea-plugin> tag with optional xi:include namespace.
*/
internal fun StringBuilder.appendOpeningTag(
spec: ProductModulesContentSpec,
inlineXmlIncludes: Boolean,
inlineModuleSets: Boolean
) {
// Determine if xi:include namespace is needed
val hasXmlIncludes = !inlineXmlIncludes && spec.deprecatedXmlIncludes.isNotEmpty()
val hasModuleSetIncludes = !inlineModuleSets && spec.moduleSets.isNotEmpty()
val needsXiNamespace = hasXmlIncludes || hasModuleSetIncludes
if (needsXiNamespace) {
append("<idea-plugin xmlns:xi=\"http://www.w3.org/2001/XInclude\">\n")
}
else {
append("<idea-plugin>\n")
}
// Add id and name as child tags if PlatformLangPlugin.xml is not included
val includesPlatformLang = spec.deprecatedXmlIncludes.any {
it.resourcePath == "META-INF/PlatformLangPlugin.xml" ||
it.resourcePath == "META-INF/JavaIdePlugin.xml" ||
it.resourcePath == "META-INF/pycharm-core.xml"
}
if (!includesPlatformLang) {
append(" <id>com.intellij</id>\n")
append(" <name>IDEA CORE</name>\n")
}
}
/**
* Generates xi:include directives or inline content for deprecated XML includes.
*/
internal fun generateXIncludes(
spec: ProductModulesContentSpec,
moduleOutputProvider: ModuleOutputProvider,
inlineXmlIncludes: Boolean,
sb: StringBuilder,
isUltimateBuild: Boolean,
) {
for (include in spec.deprecatedXmlIncludes) {
// When inlining: skip ultimate-only xi-includes in Community builds
if (inlineXmlIncludes && include.ultimateOnly && !isUltimateBuild) {
continue
}
// Find the module and file
val module = moduleOutputProvider.findModule(include.moduleName)
val resourcePath = include.resourcePath
if (module == null) {
if (include.ultimateOnly) {
error("Ultimate-only module '${include.moduleName}' not found in Ultimate build - this is a configuration error (referenced in xi:include for '$resourcePath')")
}
error("Module '${include.moduleName}' not found (referenced in xi:include for '$resourcePath')")
}
val data = findFileInModuleSources(module, resourcePath)?.let { JDOMUtil.load(it) }
?: findFileInModuleLibraryDependencies(module = module, relativePath = resourcePath)?.let { JDOMUtil.load(it) }
?: error("Resource '$resourcePath' not found in module '${module.name}' sources or libraries (referenced in xi:include)")
if (inlineXmlIncludes && !include.optional) {
withEditorFold(sb, " ", "Inlined from ${include.moduleName}/$resourcePath") {
// Inline the actual XML content
for (element in data.children) {
sb.append(JDOMUtil.write(element).prependIndent(" "))
sb.append("\n")
}
}
sb.append("\n")
}
else {
// Generate xi:include with absolute path (resources are in /META-INF/... in jars)
// Wrap ultimate-only and optional xi-includes with xi:fallback for graceful handling
if (include.ultimateOnly || include.optional) {
sb.append(""" <xi:include href="${resourcePathToXIncludePath(resourcePath)}">""")
sb.append("\n")
sb.append(""" <xi:fallback/>""")
sb.append("\n")
sb.append(""" </xi:include>""")
sb.append("\n")
}
else {
sb.append(""" <xi:include href="${resourcePathToXIncludePath(resourcePath)}"/>""")
sb.append("\n")
}
}
}
}
/**
* Converts a resource path to an xi:include href path.
*/
internal fun resourcePathToXIncludePath(resourcePath: String): String {
return if (isModuleNameLikeFilename(resourcePath)) resourcePath else "/$resourcePath"
}
@@ -0,0 +1,201 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.productLayout
/**
* Formats a validation error message with consistent structure.
*
* @param title Error title (will be prefixed with ❌)
* @param details List of detail lines to include in the message body
* @param hint Optional hint text (will be prefixed with 💡 Hint:)
* @return Formatted error message string
*/
internal fun formatValidationError(
title: String,
details: List<String>,
hint: String? = null
): String = buildString {
appendLine("❌ $title")
appendLine()
details.forEach { appendLine(it) }
if (hint != null) {
appendLine()
appendLine("💡 Hint: $hint")
}
}
/**
* Validates that all overridden modules exist as direct modules in their respective module sets.
* Throws an error if invalid overrides are found.
*
* @param moduleSetWithOverrides The module set with overrides to validate
* @param spec The product modules specification (for excluded modules)
*/
internal fun validateModuleSetOverrides(
moduleSetWithOverrides: ModuleSetWithOverrides,
spec: ProductModulesContentSpec
) {
if (moduleSetWithOverrides.loadingOverrides.isEmpty()) return
val directModules = getDirectModules(moduleSetWithOverrides.moduleSet, spec.excludedModules).map { it.name }.toSet()
val invalidOverrides = moduleSetWithOverrides.loadingOverrides.keys.filter { it !in directModules }
if (invalidOverrides.isNotEmpty()) {
val details = buildList {
add("The following ${invalidOverrides.size} module(s) are not direct modules of this set:")
invalidOverrides.sorted().forEach { add(" ✗ $it") }
add("")
add("Note: You cannot override nested set modules.")
add("")
add("Available direct modules in '${moduleSetWithOverrides.moduleSet.name}':")
directModules.sorted().take(10).forEach { add(" ✓ $it") }
if (directModules.size > 10) {
add(" ... and ${directModules.size - 10} more")
}
}
val hint = """You can only override direct modules, not modules from nested sets.
To override modules from nested sets, reference the nested set directly:
moduleSet(YourNestedSet()) {
overrideAsEmbedded("module.name")
}"""
error(formatValidationError(
"Invalid loading overrides for module set '${moduleSetWithOverrides.moduleSet.name}'",
details,
hint
))
}
}
/**
* Validates that no module appears in multiple module sets.
* Throws an error if duplicates are found.
*
* @param moduleToSets Map from module name to list of module set names containing it
*/
internal fun validateNoDuplicateModules(moduleToSets: Map<String, MutableList<String>>) {
val duplicates = moduleToSets.filterValues { it.size > 1 }
if (duplicates.isNotEmpty()) {
val details = buildList {
for ((moduleName, sets) in duplicates.toSortedMap()) {
add(" ✗ Module '$moduleName' appears in:")
sets.sorted().forEach { add(" - $it") }
add(" → Suggested fix: Remove from: ${sets.sorted().drop(1).joinToString(", ")}")
add("")
}
add("📋 Each module must belong to exactly one module set.")
add(" Fix the module set definitions in:")
add(" • community/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/CommunityModuleSets.kt")
add(" • platform/buildScripts/src/productLayout/UltimateModuleSets.kt")
}
error(formatValidationError(
"ERROR: Duplicate modules found across ${duplicates.size} module set(s)",
details,
hint = null
))
}
}
/**
* Validates that a module alias is unique and records it.
* Throws an error if the alias is already defined.
*
* @param alias The alias to validate
* @param source The source defining this alias (e.g., "product level", "module set 'foo'")
* @param aliasToSource Map tracking where each alias is defined
*/
internal fun validateAndRecordAlias(
alias: String,
source: String,
aliasToSource: MutableMap<String, String>
) {
val existing = aliasToSource.put(alias, source)
if (existing != null) {
val hint = if (source == "product level") {
"Module aliases must be unique across the entire product. Remove the duplicate alias declaration from your product specification."
} else {
"""Module aliases must be unique across all module sets.
Either:
• Remove the alias from $source
• Rename the alias to something unique
• Remove the alias from $existing"""
}
error(formatValidationError(
"Duplicate module alias detected: '$alias'",
listOf(
" Already defined in: $existing",
" Attempted to redefine at: $source"
),
hint
))
}
}
/**
* Validates that products don't reference redundant module sets.
* A module set is redundant if it's already nested inside another module set the product uses
* and the product doesn't apply any overrides to it.
*
* This validation ensures product specifications are correct and maintainable.
* Redundant module set references can lead to confusion and maintenance issues.
*
* @param allModuleSets All available module sets
* @param productSpecs List of products with their specifications to validate
* @throws IllegalStateException if any product has redundant module set references
*/
fun validateNoRedundantModuleSets(
allModuleSets: List<ModuleSet>,
productSpecs: List<Pair<String, ProductModulesContentSpec?>>
) {
// Build map of module set name -> nested set names
val moduleSetToNested = allModuleSets.associate { moduleSet ->
moduleSet.name to moduleSet.nestedSets.map { it.name }.toSet()
}
val errors = mutableListOf<String>()
for ((productName, contentSpec) in productSpecs) {
if (contentSpec == null || contentSpec.moduleSets.isEmpty()) continue
// Get module set names the product uses
val usedSets = contentSpec.moduleSets.map { it.moduleSet.name }
// Check each module set for redundancy
for (moduleSetWithOverrides in contentSpec.moduleSets) {
val setName = moduleSetWithOverrides.moduleSet.name
// Skip if this set has overrides (overrides make it non-redundant)
if (moduleSetWithOverrides.loadingOverrides.isNotEmpty()) {
continue
}
// Check if this set is nested in any other set the product uses
for (otherSetName in usedSets) {
if (otherSetName == setName) continue
val nestedInOther = moduleSetToNested[otherSetName]
if (nestedInOther != null && setName in nestedInOther) {
errors.add(" ✗ Product '$productName': module set '$setName' is redundant (already nested in '$otherSetName')")
}
}
}
}
if (errors.isNotEmpty()) {
val hint = """Remove redundant module sets from product's getProductContentDescriptor() method.
Example fix:
override fun getProductContentDescriptor() = productModules {
// moduleSet(ssh()) // ← REMOVE (already in ide.ultimate)
// moduleSet(rd.common()) // ← REMOVE (already in ide.ultimate)
moduleSet(ideUltimate()) // ← KEEP (includes ssh and rd.common)
}"""
error(formatValidationError(
"Product specification errors: Redundant module set references detected",
errors,
hint
))
}
}
@@ -27,6 +27,7 @@ internal fun visitAllModuleSets(sets: List<ModuleSet>, visitor: (ModuleSet) -> U
/**
* Recursively collects all aliases from a module set and its nested sets.
* Collects the single `alias` field (for module set's own capability).
*
* @param moduleSet The module set to collect aliases from
* @param result Accumulator for collecting aliases (avoids intermediate allocations)
@@ -67,6 +68,31 @@ internal inline fun withEditorFold(sb: StringBuilder, indent: String, descriptio
sb.append("$indent<!-- </editor-fold> -->\n")
}
/**
* Recursively collects all module names from a module set and its nested sets.
*
* @param moduleSet The module set to collect modules from
* @param excludedModules Set of module names to exclude
* @return Set of all module names found in the module set hierarchy
*/
internal fun collectAllModuleNames(moduleSet: ModuleSet, excludedModules: Set<String> = emptySet()): Set<String> {
val result = HashSet<String>()
fun collect(set: ModuleSet) {
for (module in set.modules) {
if (module.name !in excludedModules) {
result.add(module.name)
}
}
for (nestedSet in set.nestedSets) {
collect(nestedSet)
}
}
collect(moduleSet)
return result
}
/**
* Gets direct modules from a module set (excluding modules from nested sets and excluded modules).
*
@@ -101,4 +127,39 @@ internal fun getDirectModules(moduleSet: ModuleSet, excludedModules: Set<String>
}
}
return directModules
}
/**
* Checks if a module set contains (directly or transitively) any nested set whose name is in the given set.
* Used to detect when a parent module set contains overridden nested sets.
*/
internal fun containsOverriddenNestedSet(moduleSet: ModuleSet, overriddenNames: Set<ModuleSetName>): Boolean {
// Check direct nested sets
for (nestedSet in moduleSet.nestedSets) {
if (ModuleSetName(nestedSet.name) in overriddenNames) {
return true
}
// Check recursively
if (containsOverriddenNestedSet(nestedSet, overriddenNames)) {
return true
}
}
return false
}
/**
* Finds all nested set names (directly or transitively) that are in the given set of names.
* Used for generating descriptive comments about which nested sets are overridden.
*/
internal fun findOverriddenNestedSetNames(moduleSet: ModuleSet, overriddenNames: Set<ModuleSetName>): List<ModuleSetName> {
val result = mutableListOf<ModuleSetName>()
for (nestedSet in moduleSet.nestedSets) {
val nestedSetName = ModuleSetName(nestedSet.name)
if (nestedSetName in overriddenNames) {
result.add(nestedSetName)
}
// Check recursively
result.addAll(findOverriddenNestedSetNames(nestedSet, overriddenNames))
}
return result
}
@@ -0,0 +1,192 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.productLayout
import org.jetbrains.intellij.build.ModuleOutputProvider
import java.nio.file.Files
import java.nio.file.Path
/**
* Main entry point for generating XML files for both module sets and products.
* This file orchestrates the generation process by calling specialized components for:
* - Building content blocks (ContentBlockBuilder.kt)
* - Validating specifications (ValidationUtils.kt)
* - Rendering XML content (ModuleSetXmlRenderer.kt, ProductXmlRenderer.kt)
*/
/**
* Generates an XML file for a module set.
* Used to maintain backward compatibility with XML-based module set loading.
*
* @param moduleSet The module set to generate XML for
* @param outputDir The directory where the XML file will be written
* @param label Description label ("community" or "ultimate") for header generation
* @return Result containing file status and statistics
*/
fun generateModuleSetXml(moduleSet: ModuleSet, outputDir: Path, label: String): ModuleSetFileResult {
val fileName = "${MODULE_SET_PREFIX}${moduleSet.name}.xml"
val outputPath = outputDir.resolve(fileName)
val buildResult = buildModuleSetXml(moduleSet, label)
// determine change status
val status = when {
!Files.exists(outputPath) -> FileChangeStatus.CREATED
Files.readString(outputPath) == buildResult.xml -> FileChangeStatus.UNCHANGED
else -> FileChangeStatus.MODIFIED
}
// Only write if changed
if (status != FileChangeStatus.UNCHANGED) {
Files.writeString(outputPath, buildResult.xml)
}
return ModuleSetFileResult(fileName, status, buildResult.directModuleCount)
}
/**
* Generates complete product plugin.xml file from programmatic specification.
*
* @param isUltimateBuild Whether this is an Ultimate build (vs. Community build)
* @return Result containing file status and statistics
*/
fun generateProductXml(
pluginXmlPath: Path,
spec: ProductModulesContentSpec,
productName: String,
productPropertiesClass: String,
moduleOutputProvider: ModuleOutputProvider,
projectRoot: Path,
isUltimateBuild: Boolean,
): ProductFileResult {
// Determine which generator to recommend based on plugin.xml file location
// Community products are under community/ directory, Ultimate products are not
val generatorCommand = (if (pluginXmlPath.toString().contains("/community/")) "CommunityModuleSets" else "UltimateModuleSets") + GENERATOR_SUFFIX
// Build complete plugin.xml file
val buildResult = buildProductContentXml(
spec = spec,
moduleOutputProvider = moduleOutputProvider,
inlineXmlIncludes = false,
inlineModuleSets = false,
productPropertiesClass = productPropertiesClass,
generatorCommand = generatorCommand,
isUltimateBuild = isUltimateBuild
)
// Compare with existing file if it exists
val originalContent = Files.readString(pluginXmlPath)
val status = if (originalContent == buildResult.xml) {
FileChangeStatus.UNCHANGED
}
else {
FileChangeStatus.MODIFIED
}
// Only write if changed
if (status != FileChangeStatus.UNCHANGED) {
Files.writeString(pluginXmlPath, buildResult.xml)
}
// Calculate statistics using the contentBlocks from generation
val totalModules = buildResult.contentBlocks.sumOf { it.modules.size }
val relativePath = projectRoot.relativize(pluginXmlPath).toString()
return ProductFileResult(
productName = productName,
relativePath = relativePath,
status = status,
includeCount = spec.deprecatedXmlIncludes.size,
contentBlockCount = buildResult.contentBlocks.size,
totalModules = totalModules
)
}
/**
* Builds XML content for programmatic product modules.
* Generates module alias, xi:include directives (or inlined content), and `<content>` blocks for each module set.
*/
fun buildProductContentXml(
spec: ProductModulesContentSpec,
moduleOutputProvider: ModuleOutputProvider,
inlineXmlIncludes: Boolean,
inlineModuleSets: Boolean,
productPropertiesClass: String,
generatorCommand: String,
isUltimateBuild: Boolean,
): ProductContentBuildResult {
// Build content blocks, chain mapping, and collect module set aliases in single traversal
val (contentBlocks, moduleToSetChainMapping, moduleSetAliases) =
buildContentBlocksAndChainMapping(spec, collectModuleSetAliases = inlineModuleSets)
val xml = buildString {
appendXmlHeader(generatorCommand, productPropertiesClass)
appendOpeningTag(spec, inlineXmlIncludes, inlineModuleSets)
// Collect and validate product-level aliases, checking for conflicts with module set aliases
val validatedAliases = collectAndValidateAliases(spec, moduleSetAliases)
val aliasXml = buildModuleAliasesXml(validatedAliases)
if (aliasXml.isNotEmpty()) {
append(aliasXml)
append("\n")
}
// Generate xi:include directives or inline content
if (spec.deprecatedXmlIncludes.isNotEmpty()) {
generateXIncludes(spec = spec, moduleOutputProvider = moduleOutputProvider, inlineXmlIncludes = inlineXmlIncludes, sb = this, isUltimateBuild = isUltimateBuild)
}
// Generate module sets as xi:includes or inline content blocks
if (spec.moduleSets.isNotEmpty()) {
if (inlineModuleSets) {
// Generate single content block with all module sets inlined
append(" <content namespace=\"$JETBRAINS_NAMESPACE\">\n")
for ((index, block) in contentBlocks.withIndex()) {
if (block.source == ADDITIONAL_MODULES_BLOCK) continue // Skip additional modules, handle separately
withEditorFold(this, " ", block.source) {
for (module in block.modules) {
appendModuleLine(module, " ")
}
}
// Add blank line between sections for readability (except after last block)
if (index < contentBlocks.size - 1) {
append("\n")
}
}
append(" </content>\n")
}
else {
// Build set of module set names that are referenced at top-level WITH overrides
// These cannot be brought in via `xi:include` from parent sets (would lose overrides)
val overriddenModuleSetNames = spec.moduleSets
.filter { it.hasOverrides }
.map { ModuleSetName(it.moduleSet.name) }
.toSet()
appendModuleSetsStrategyComment(spec, overriddenModuleSetNames)
// Generate content for each top-level module set
for (moduleSetWithOverrides in spec.moduleSets) {
appendModuleSetXml(
moduleSetWithOverrides.moduleSet,
moduleSetWithOverrides.loadingOverrides,
contentBlocks,
overriddenModuleSetNames
)
}
}
}
// Handle additional modules separately (they don't have XML files, inline them)
val additionalBlock = contentBlocks.firstOrNull { it.source == ADDITIONAL_MODULES_BLOCK }
if (additionalBlock != null) {
appendContentBlock(additionalBlock.source, additionalBlock.modules)
}
// Closing tag
append("</idea-plugin>\n")
}
return ProductContentBuildResult(xml = xml, contentBlocks = contentBlocks, moduleToSetChainMapping = moduleToSetChainMapping)
}
@@ -0,0 +1,161 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.productLayout
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
/**
* Tests for generator.kt helper functions.
*/
class ProductDslGeneratorTest {
// Test fixtures
private fun createSimpleModuleSet(name: String, vararg moduleNames: String): ModuleSet {
return ModuleSet(
name = name,
modules = moduleNames.map { ContentModule(it) }
)
}
private fun createNestedModuleSet(
name: String,
moduleNames: List<String>,
nestedSets: List<ModuleSet>,
): ModuleSet {
return ModuleSet(
name = name,
modules = moduleNames.map { ContentModule(it) },
nestedSets = nestedSets
)
}
// Tests for containsOverriddenNestedSet()
@Test
fun `containsOverriddenNestedSet detects direct nested override`() {
val overriddenSet = createSimpleModuleSet("overridden", "mod.a")
val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(overriddenSet))
val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("overridden")))
assertThat(result).isTrue()
}
@Test
fun `containsOverriddenNestedSet detects deeply nested override`() {
val deeplyNested = createSimpleModuleSet("deep", "mod.a")
val middleNested = createNestedModuleSet("middle", listOf("mod.b"), listOf(deeplyNested))
val parentSet = createNestedModuleSet("parent", listOf("mod.c"), listOf(middleNested))
val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("deep")))
assertThat(result).isTrue()
}
@Test
fun `containsOverriddenNestedSet returns false when no overrides`() {
val nestedSet = createSimpleModuleSet("nested", "mod.a")
val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet))
val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("someOtherSet")))
assertThat(result).isFalse()
}
@Test
fun `containsOverriddenNestedSet returns false for empty override set`() {
val nestedSet = createSimpleModuleSet("nested", "mod.a")
val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet))
val result = containsOverriddenNestedSet(parentSet, emptySet())
assertThat(result).isFalse()
}
@Test
fun `containsOverriddenNestedSet returns false for module set with no nested sets`() {
val parentSet = createSimpleModuleSet("parent", "mod.a", "mod.b")
val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("someSet")))
assertThat(result).isFalse()
}
// Tests for findOverriddenNestedSetNames()
@Test
fun `findOverriddenNestedSetNames finds direct overridden set`() {
val overridden1 = createSimpleModuleSet("overridden1", "mod.a")
val overridden2 = createSimpleModuleSet("overridden2", "mod.b")
val notOverridden = createSimpleModuleSet("notOverridden", "mod.c")
val parentSet = createNestedModuleSet(
"parent",
listOf("mod.d"),
listOf(overridden1, notOverridden, overridden2)
)
val result = findOverriddenNestedSetNames(parentSet, setOf(ModuleSetName("overridden1"), ModuleSetName("overridden2")))
assertThat(result).containsExactlyInAnyOrder(ModuleSetName("overridden1"), ModuleSetName("overridden2"))
}
@Test
fun `findOverriddenNestedSetNames finds all overridden sets recursively`() {
val deepOverridden = createSimpleModuleSet("deepOverridden", "mod.a")
val deepNormal = createSimpleModuleSet("deepNormal", "mod.b")
val middleOverridden = createNestedModuleSet(
"middleOverridden",
listOf("mod.c"),
listOf(deepOverridden, deepNormal)
)
val middleNormal = createSimpleModuleSet("middleNormal", "mod.d")
val parentSet = createNestedModuleSet(
"parent",
listOf("mod.e"),
listOf(middleOverridden, middleNormal)
)
val result = findOverriddenNestedSetNames(
parentSet,
setOf(ModuleSetName("middleOverridden"), ModuleSetName("deepOverridden"))
)
assertThat(result).containsExactlyInAnyOrder(ModuleSetName("middleOverridden"), ModuleSetName("deepOverridden"))
}
@Test
fun `findOverriddenNestedSetNames returns empty for no overrides`() {
val nestedSet = createSimpleModuleSet("nested", "mod.a")
val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet))
val result = findOverriddenNestedSetNames(parentSet, setOf(ModuleSetName("someOtherSet")))
assertThat(result).isEmpty()
}
@Test
fun `findOverriddenNestedSetNames returns empty for empty override set`() {
val nestedSet = createSimpleModuleSet("nested", "mod.a")
val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet))
val result = findOverriddenNestedSetNames(parentSet, emptySet())
assertThat(result).isEmpty()
}
@Test
fun `findOverriddenNestedSetNames preserves order of discovery`() {
val nested1 = createSimpleModuleSet("nested1", "mod.a")
val nested2 = createSimpleModuleSet("nested2", "mod.b")
val nested3 = createSimpleModuleSet("nested3", "mod.c")
val parentSet = createNestedModuleSet(
"parent",
listOf("mod.d"),
listOf(nested1, nested2, nested3)
)
val result = findOverriddenNestedSetNames(parentSet, setOf(ModuleSetName("nested1"), ModuleSetName("nested2"), ModuleSetName("nested3")))
// Should be in order of traversal
assertThat(result).containsExactly(ModuleSetName("nested1"), ModuleSetName("nested2"), ModuleSetName("nested3"))
}
}
@@ -0,0 +1,453 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.productLayout
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.jetbrains.intellij.build.ModuleOutputProvider
import org.jetbrains.jps.model.module.JpsModule
import org.junit.jupiter.api.Test
import java.nio.file.Path
class ProductModulesContentSpecTest {
@Test
fun `test valid overrides for existing modules`() {
// Create a simple module set with some modules
val moduleSet = ModuleSet(
name = "testSet",
modules = listOf(
ContentModule("module.a"),
ContentModule("module.b"),
ContentModule("module.c")
)
)
val spec = productModules {
moduleSet(moduleSet) {
overrideAsEmbedded("module.a")
overrideAsEmbedded("module.b")
}
}
// This should not throw - all overridden modules exist
val result = buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
assertThat(result.xml).contains("module.a")
assertThat(result.xml).contains("loading=\"embedded\"")
}
@Test
fun `test invalid overrides for non-existent modules`() {
val moduleSet = ModuleSet(
name = "testSet",
modules = listOf(
ContentModule("module.a"),
ContentModule("module.b")
)
)
val spec = productModules {
moduleSet(moduleSet) {
overrideAsEmbedded("module.a")
overrideAsEmbedded("module.nonexistent")
}
}
// This should throw because module.nonexistent doesn't exist
assertThatThrownBy {
buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
}
.isInstanceOf(IllegalStateException::class.java)
.hasMessageContaining("Invalid loading overrides for module set 'testSet'")
.hasMessageContaining("module.nonexistent")
}
@Test
fun `test cannot override modules from nested sets`() {
val nestedSet = ModuleSet(
name = "nested",
modules = listOf(
ContentModule("nested.module.a"),
ContentModule("nested.module.b")
)
)
val parentSet = ModuleSet(
name = "parent",
modules = listOf(
ContentModule("parent.module.a")
),
nestedSets = listOf(nestedSet)
)
val spec = productModules {
moduleSet(parentSet) {
// Trying to override nested module should fail
overrideAsEmbedded("parent.module.a")
overrideAsEmbedded("nested.module.a")
}
}
// This should throw because nested.module.a is not a direct module of parent
assertThatThrownBy {
buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
}
.isInstanceOf(IllegalStateException::class.java)
.hasMessageContaining("Invalid loading overrides for module set 'parent'")
.hasMessageContaining("cannot override nested set modules")
.hasMessageContaining("nested.module.a")
}
@Test
fun `test empty overrides work fine`() {
val moduleSet = ModuleSet(
name = "testSet",
modules = listOf(
ContentModule("module.a")
)
)
val spec = productModules {
moduleSet(moduleSet)
}
// This should not throw
val result = buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
assertThat(result.xml).contains("module.a")
}
@Test
fun `test multiple invalid overrides are all reported`() {
val moduleSet = ModuleSet(
name = "testSet",
modules = listOf(
ContentModule("module.a")
)
)
val spec = productModules {
moduleSet(moduleSet) {
overrideAsEmbedded("module.nonexistent1")
overrideAsEmbedded("module.nonexistent2")
overrideAsEmbedded("module.a")
}
}
// Should report both nonexistent modules
assertThatThrownBy {
buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
}
.isInstanceOf(IllegalStateException::class.java)
.hasMessageContaining("module.nonexistent1")
.hasMessageContaining("module.nonexistent2")
}
@Test
fun `test overrides respect excluded modules`() {
val moduleSet = ModuleSet(
name = "testSet",
modules = listOf(
ContentModule("module.a"),
ContentModule("module.b")
)
)
val spec = productModules {
moduleSet(moduleSet) {
overrideAsEmbedded("module.b") // module.b is excluded, so this override is invalid
}
exclude("module.b")
}
// This should throw because module.b is excluded
assertThatThrownBy {
buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
}
.isInstanceOf(IllegalStateException::class.java)
.hasMessageContaining("Invalid loading overrides")
.hasMessageContaining("module.b")
}
@Test
fun `test selective inlining with overrides generates correct XML`() {
val nestedSet = ModuleSet(
name = "nested",
modules = listOf(
ContentModule("nested.module.a"),
ContentModule("nested.module.b")
)
)
val parentSet = ModuleSet(
name = "parent",
modules = listOf(
ContentModule("parent.module.a"),
ContentModule("parent.module.b")
),
nestedSets = listOf(nestedSet)
)
val spec = productModules {
moduleSet(parentSet) {
overrideAsEmbedded("parent.module.a")
}
}
// Test with inlineModuleSets = false (selective inlining mode)
val result = buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = false,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
// Should inline parent modules with loading attributes
assertThat(result.xml).contains("parent.module.a")
assertThat(result.xml).contains("loading=\"embedded\"")
assertThat(result.xml).contains("parent.module.b")
// Should generate xi:include for nested set
assertThat(result.xml).contains("<xi:include href=\"/META-INF/intellij.moduleSets.nested.xml\"/>")
}
@Test
fun `test loading attributes are correctly applied in inlined mode`() {
val moduleSet = ModuleSet(
name = "testSet",
modules = listOf(
ContentModule("module.a"),
ContentModule("module.b"),
ContentModule("module.c")
)
)
val spec = productModules {
moduleSet(moduleSet) {
overrideAsEmbedded("module.a")
overrideAsEmbedded("module.b")
}
}
// Test with inlineModuleSets = true (full inlining mode)
val result = buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
// Verify embedded modules have loading attribute
assertThat(result.xml).containsPattern("<module name=\"module\\.a\" loading=\"embedded\"/>")
assertThat(result.xml).containsPattern("<module name=\"module\\.b\" loading=\"embedded\"/>")
// Verify non-overridden module does not have loading attribute
assertThat(result.xml).containsPattern("<module name=\"module\\.c\"/>")
assertThat(result.xml).doesNotContain("module.c\" loading")
}
@Test
fun `test nested set override prevents duplicate module entries`() {
// This test covers the real-world Rider scenario:
// Parent set (commercialIdeBase) contains nested set (rdCommon)
// Product also references rdCommon directly with overrides
// Expected: rdCommon modules appear ONCE with overrides, not twice
val deeplyNestedSet = ModuleSet(
name = "rdCommon",
modules = listOf(
ContentModule("rd.module.a"),
ContentModule("rd.module.b"),
ContentModule("rd.module.c")
)
)
val middleSet = ModuleSet(
name = "ideUltimate",
modules = listOf(
ContentModule("ide.module.a")
),
nestedSets = listOf(deeplyNestedSet)
)
val parentSet = ModuleSet(
name = "commercialIdeBase",
modules = listOf(
ContentModule("commercial.module.a")
),
nestedSets = listOf(middleSet)
)
val spec = productModules {
// Include parent (which contains rdCommon nested deeply)
moduleSet(parentSet)
// Also reference rdCommon directly with overrides
moduleSet(deeplyNestedSet) {
overrideAsEmbedded("rd.module.a")
overrideAsEmbedded("rd.module.b")
}
}
val result = buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = false,
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
// Verify rdCommon modules appear exactly ONCE with loading attributes
// Use regex to count only actual module elements, not comments
val rdModuleACount = Regex("""<module name="rd\.module\.a"""").findAll(result.xml).count()
val rdModuleBCount = Regex("""<module name="rd\.module\.b"""").findAll(result.xml).count()
assertThat(rdModuleACount).isEqualTo(1)
assertThat(rdModuleBCount).isEqualTo(1)
// Verify the modules have the embedded loading attribute
assertThat(result.xml).contains("<module name=\"rd.module.a\" loading=\"embedded\"/>")
assertThat(result.xml).contains("<module name=\"rd.module.b\" loading=\"embedded\"/>")
// Verify parent and middle sets were selectively inlined (not xi:included)
assertThat(result.xml).contains("commercial.module.a")
assertThat(result.xml).contains("ide.module.a")
// Verify no xi:include for sets containing overridden nested sets
assertThat(result.xml).doesNotContain("<xi:include href=\"/META-INF/intellij.moduleSets.commercialIdeBase.xml\"/>")
assertThat(result.xml).doesNotContain("<xi:include href=\"/META-INF/intellij.moduleSets.ideUltimate.xml\"/>")
}
@Test
fun `test overrides preserved when module set nested and directly referenced with full inlining`() {
// This test covers the Rider ClassNotFoundException bug:
// When inlineModuleSets = true (full inlining mode), overrides are lost if:
// - A module set is nested inside another set
// - The same module set is also directly referenced with overrides
// Bug: Only the first encounter's overrides are kept due to processedSets deduplication
val rdCommon = ModuleSet(
name = "rdCommon",
modules = listOf(
ContentModule("intellij.rd.platform"),
ContentModule("intellij.rd.ui")
)
)
val commercialIdeBase = ModuleSet(
name = "commercialIdeBase",
modules = listOf(
ContentModule("commercial.module")
),
nestedSets = listOf(rdCommon)
)
val spec = productModules {
moduleSet(commercialIdeBase) // Includes rdCommon nested (no overrides)
moduleSet(rdCommon) { // Also reference rdCommon with overrides
overrideAsEmbedded("intellij.rd.platform")
overrideAsEmbedded("intellij.rd.ui")
}
}
// Test with inlineModuleSets = true (full inlining mode) - this is where the bug occurs
val result = buildProductContentXml(
spec = spec,
moduleOutputProvider = MockModuleOutputProvider(),
inlineXmlIncludes = false,
inlineModuleSets = true, // Full inlining mode
productPropertiesClass = "TestProperties",
generatorCommand = "test",
isUltimateBuild = false
)
// Verify rd modules have loading="embedded" attribute
assertThat(result.xml).contains("<module name=\"intellij.rd.platform\" loading=\"embedded\"/>")
assertThat(result.xml).contains("<module name=\"intellij.rd.ui\" loading=\"embedded\"/>")
// Verify they don't appear without loading attribute (bug would cause this)
assertThat(result.xml).doesNotContainPattern("<module name=\"intellij\\.rd\\.platform\"\\s*/>")
assertThat(result.xml).doesNotContainPattern("<module name=\"intellij\\.rd\\.ui\"\\s*/>")
// Verify modules appear exactly once
val rdPlatformCount = Regex("""<module name="intellij\.rd\.platform"""").findAll(result.xml).count()
val rdUiCount = Regex("""<module name="intellij\.rd\.ui"""").findAll(result.xml).count()
assertThat(rdPlatformCount).describedAs("rd.platform should appear exactly once").isEqualTo(1)
assertThat(rdUiCount).describedAs("rd.ui should appear exactly once").isEqualTo(1)
}
}
/**
* Mock ModuleOutputProvider for testing that doesn't load modules.
*/
private class MockModuleOutputProvider : ModuleOutputProvider {
override fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean): ByteArray? {
TODO("Not yet implemented")
}
override fun findModule(name: String): JpsModule? {
TODO("Not yet implemented")
}
override fun findRequiredModule(name: String): JpsModule {
TODO("Not yet implemented")
}
override fun getModuleOutputRoots(module: JpsModule, forTests: Boolean): List<Path> {
TODO("Not yet implemented")
}
}
@@ -5,7 +5,6 @@ import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.intellij.build.dependencies.DependenciesProperties
import org.jetbrains.intellij.build.impl.BundledRuntime
import org.jetbrains.intellij.build.impl.CompilationTasksImpl
import org.jetbrains.intellij.build.impl.ModuleOutputProvider
import org.jetbrains.intellij.build.moduleBased.OriginalModuleRepository
import org.jetbrains.jps.model.JpsModel
import org.jetbrains.jps.model.JpsProject
@@ -0,0 +1,142 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import com.intellij.openapi.application.PathManager
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.json.Json
import org.jetbrains.intellij.build.productLayout.DuplicateIncludeDetector
import java.nio.file.Path
import kotlin.io.path.exists
import kotlin.io.path.isDirectory
import kotlin.io.path.isRegularFile
import kotlin.io.path.listDirectoryEntries
import kotlin.io.path.readText
import kotlin.io.path.walk
/**
* Command-line tool to detect duplicate xi:include elements in product plugin.xml files.
*
* Usage:
* bazel run //community/platform/build-scripts:DuplicateIncludeAnalyzer
*
* Output:
* JSON report showing products with duplicate includes, where each duplicate comes from,
* and summary statistics.
*/
object DuplicateIncludeAnalyzer {
@JvmStatic
fun main(args: Array<String>) {
runBlocking(Dispatchers.Default) {
val projectRoot = Path.of(PathManager.getHomePathFor(DuplicateIncludeAnalyzer::class.java)!!)
// Discover all product plugin.xml files
val productFiles = discoverProductFiles(projectRoot)
// Run detection
val report = DuplicateIncludeDetector.detectDuplicates(productFiles, projectRoot)
// Output JSON
val json = Json { prettyPrint = true }
println(json.encodeToString(report))
}
}
/**
* Discovers all product plugin.xml files in the project.
* Searches in known product directories and looks for files ending with Plugin.xml or named plugin.xml.
*/
private fun discoverProductFiles(projectRoot: Path): List<Path> {
val productFiles = mutableListOf<Path>()
// Known product directories to search
val productDirs = listOf(
projectRoot.resolve("community"),
projectRoot.resolve("ultimate"),
projectRoot.resolve("CIDR"),
projectRoot.resolve("goland"),
projectRoot.resolve("ruby"),
projectRoot.resolve("WebStorm"),
projectRoot.resolve("dbe"),
projectRoot.resolve("aqua"),
projectRoot.resolve("rider"),
projectRoot.resolve("python"),
projectRoot.resolve("plugins"),
)
for (dir in productDirs) {
if (!dir.exists() || !dir.isDirectory()) {
continue
}
// Search for Plugin.xml files in resources/META-INF directories
dir.walk()
.filter { it.isRegularFile() }
.filter { it.parent?.fileName?.toString() == "META-INF" }
.filter {
val name = it.fileName.toString()
name.endsWith("Plugin.xml") || name == "plugin.xml"
}
.filter { isProductFile(it) }
.forEach { productFiles.add(it) }
}
return productFiles.distinct()
}
/**
* Checks if an XML file is a product descriptor (not a plugin descriptor).
* Products don't have their own <id> tag or have <id>com.intellij</id>.
* Also filters out test files and non-product descriptors.
*/
private fun isProductFile(file: Path): Boolean {
// Skip test resources
if (file.toString().contains("/testResources/") ||
file.toString().contains("/testSrc/") ||
file.toString().contains("/test/")) {
return false
}
// Skip toolbox
if (file.toString().contains("/toolbox/")) {
return false
}
val fileName = file.fileName.toString()
// Skip module descriptor files (have dots in the name like intellij.platform.jewel.detektPlugin.xml)
if (fileName != "plugin.xml" && fileName.contains(".") && !fileName.matches(Regex("^[A-Z][a-zA-Z]*Plugin\\.xml$"))) {
return false
}
try {
val content = file.readText()
// Check for <id> tag that's NOT com.intellij
val idMatch = Regex("""<id>([^<]+)</id>""").find(content)
if (idMatch != null && idMatch.groupValues[1] != "com.intellij") {
// This is a plugin with its own ID, not a product
return false
}
// Check if it has ApplicationInfo.xml nearby (strong indicator of a product)
val resourceRoot = file.parent?.parent // Go up from META-INF to resources
if (resourceRoot != null) {
val ideaDir = resourceRoot.resolve("idea")
if (ideaDir.exists() && ideaDir.isDirectory()) {
val hasAppInfo = ideaDir.listDirectoryEntries()
.any { it.fileName.toString().endsWith("ApplicationInfo.xml") }
if (hasAppInfo) {
return true
}
}
}
// If no <id> tag or has com.intellij, likely a product
return true
}
catch (e: Exception) {
return false
}
}
}
@@ -5,11 +5,8 @@ package org.jetbrains.intellij.build
import com.intellij.util.xml.dom.XmlElement
import com.intellij.util.xml.dom.readXmlAsModel
import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.impl.ModuleItem
import org.jetbrains.intellij.build.impl.PluginLayout
import org.jetbrains.jps.model.java.JavaResourceRootType
import org.jetbrains.jps.model.java.JavaSourceRootType
import org.jetbrains.jps.model.java.JpsJavaClasspathKind
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsDependencyElement
@@ -17,11 +14,8 @@ import org.jetbrains.jps.model.module.JpsLibraryDependency
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.model.module.JpsModuleDependency
import org.jetbrains.jps.model.module.JpsModuleReference
import java.nio.file.Path
import java.util.concurrent.ConcurrentHashMap
internal val useTestSourceEnabled: Boolean = System.getProperty("idea.build.pack.test.source.enabled", "true").toBoolean()
// production-only - JpsJavaClasspathKind.PRODUCTION_RUNTIME
internal class JarPackagerDependencyHelper(private val context: CompilationContext) {
private val javaExtensionService = JpsJavaExtensionService.getInstance()
@@ -74,7 +68,7 @@ internal class JarPackagerDependencyHelper(private val context: CompilationConte
return moduleName.endsWith("._test")
}
suspend fun getPluginIdByModule(pluginModule: JpsModule): String {
fun getPluginIdByModule(pluginModule: JpsModule): String {
// it is ok to read the plugin descriptor with unresolved x-include as the ID should be specified at the root
val root = readXmlAsModel(getUnprocessedPluginXmlContent(module = pluginModule, context = context))
val element = root.getChild("id") ?: root.getChild("name") ?: throw IllegalStateException("Cannot find attribute id or name (module=$pluginModule)")
@@ -33,7 +33,7 @@ open class LinuxDistributionCustomizer {
*/
var extraExecutables: PersistentList<String> = persistentListOf()
open fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture, targetLibcImpl: LibcImpl): Sequence<String> {
open fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, targetLibcImpl: LibcImpl, context: BuildContext): Sequence<String> {
val basePatterns = sequenceOf(
"bin/*.sh",
"plugins/**/*.sh",
@@ -77,13 +77,14 @@ open class LinuxDistributionCustomizer {
/**
* Name of the root directory inside the .tar.gz archive.
*/
open fun getRootDirectoryName(appInfo: ApplicationInfoProperties, buildNumber: String): String =
"${appInfo.fullProductName}-${if (appInfo.isEAP) buildNumber else appInfo.fullVersion}"
open fun getRootDirectoryName(appInfo: ApplicationInfoProperties, buildNumber: String): String {
return "${appInfo.fullProductName}-${if (appInfo.isEAP) buildNumber else appInfo.fullVersion}"
}
/**
* Override this method to copy additional files to the Linux distribution of the product.
*/
open suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path, arch: JvmArchitecture) {
open suspend fun copyAdditionalFiles(targetDir: Path, arch: JvmArchitecture, context: BuildContext) {
RepairUtilityBuilder.bundle(context, OsFamily.LINUX, arch, targetDir)
}
}
@@ -6,7 +6,7 @@ import kotlinx.collections.immutable.persistentListOf
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.intellij.build.impl.support.RepairUtilityBuilder
import java.nio.file.Path
import java.util.*
import java.util.UUID
import java.util.function.Predicate
open class MacDistributionCustomizer {
@@ -142,7 +142,7 @@ open class MacDistributionCustomizer {
RepairUtilityBuilder.bundle(context, OsFamily.MACOS, arch, targetDir)
}
open fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture): Sequence<String> {
open fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, context: BuildContext): Sequence<String> {
val basePatterns = sequenceOf(
"bin/*.sh",
"plugins/**/*.sh",
@@ -330,7 +330,7 @@ abstract class ProductProperties {
/**
* Override this method to copy additional files to distributions of all operating systems.
*/
open suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) { }
open suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) { }
/**
* Override this method if the product has several editions to ensure that their artifacts won't be mixed up.
@@ -408,7 +408,7 @@ abstract class ProductProperties {
* Copies additional localization resources to the plugin-generated localization resources directory.
*/
@ApiStatus.Internal
open suspend fun copyAdditionalLocalizationResourcesToPlugin(context: BuildContext, lang: String, targetDir: Path) {}
open suspend fun copyAdditionalLocalizationResourcesToPlugin(lang: String, targetDir: Path, context: BuildContext) {}
/**
* Build steps which are always skipped for this product.
@@ -75,7 +75,7 @@ open class WindowsDistributionCustomizer {
/**
* Override this method to copy additional files to the Windows distribution of the product.
*/
open suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path, arch: JvmArchitecture) {
open suspend fun copyAdditionalFiles(targetDir: Path, arch: JvmArchitecture, context: BuildContext) {
RepairUtilityBuilder.bundle(context, OsFamily.WINDOWS, arch, targetDir)
}
@@ -3,7 +3,6 @@ package org.jetbrains.intellij.build
import com.intellij.openapi.util.JDOMUtil
import com.intellij.util.xml.dom.readXmlAsModel
import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.impl.BUILT_IN_HELP_MODULE_NAME
import org.jetbrains.intellij.build.impl.JarPackager
import org.jetbrains.intellij.build.impl.ModuleItem
@@ -11,6 +11,7 @@ import org.jetbrains.annotations.VisibleForTesting
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.PLATFORM_LOADER_JAR
import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.UTIL_8_JAR
import org.jetbrains.intellij.build.UTIL_JAR
import org.jetbrains.intellij.build.impl.DescriptorCacheContainer
@@ -27,11 +27,11 @@ import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.FrontendModuleFilter
import org.jetbrains.intellij.build.JarPackagerDependencyHelper
import org.jetbrains.intellij.build.ModuleOutputProvider
import org.jetbrains.intellij.build.findFileInModuleDependencies
import org.jetbrains.intellij.build.findUnprocessedDescriptorContent
import org.jetbrains.intellij.build.impl.BuildContextImpl
import org.jetbrains.intellij.build.impl.DescriptorCacheContainer
import org.jetbrains.intellij.build.impl.ModuleOutputProvider
import org.jetbrains.intellij.build.impl.PluginLayout
import org.jetbrains.intellij.build.impl.ScopedCachedDescriptorContainer
import org.jetbrains.intellij.build.impl.XIncludeElementResolver
@@ -41,8 +41,6 @@ import org.jetbrains.intellij.build.impl.toLoadPath
import java.io.IOException
import java.nio.file.Files
internal const val PLUGIN_XML_RELATIVE_PATH = "META-INF/plugin.xml"
/**
* Defines a search scope for resolving XInclude references in plugin descriptors.
*
@@ -29,6 +29,7 @@ import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LibcImpl
import org.jetbrains.intellij.build.LinuxDistributionCustomizer
import org.jetbrains.intellij.build.MacDistributionCustomizer
import org.jetbrains.intellij.build.ModuleOutputProvider
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.ProductProperties
import org.jetbrains.intellij.build.ProprietaryBuildTools
@@ -44,7 +45,6 @@ import org.jetbrains.intellij.build.getDevModeOrTestBuildDateInSeconds
import org.jetbrains.intellij.build.impl.BuildContextImpl
import org.jetbrains.intellij.build.impl.CompilationContextImpl
import org.jetbrains.intellij.build.impl.ModuleOutputPatcher
import org.jetbrains.intellij.build.impl.ModuleOutputProvider
import org.jetbrains.intellij.build.impl.PLUGIN_CLASSPATH
import org.jetbrains.intellij.build.impl.PlatformLayout
import org.jetbrains.intellij.build.impl.asArchived
@@ -209,7 +209,7 @@ private suspend fun layoutShared(context: BuildContext) {
Files.createDirectories(to.parent)
Files.copy(from, to, StandardCopyOption.REPLACE_EXISTING)
}
context.productProperties.copyAdditionalFiles(context, context.paths.distAllDir)
context.productProperties.copyAdditionalFiles(context.paths.distAllDir, context)
}
}
checkClassFiles(root = context.paths.distAllDir, isDistAll = true, context)
@@ -107,7 +107,7 @@ class CompilationContextImpl private constructor(
val global: JpsGlobal
get() = model.global
private val moduleOutputProvider = ModuleOutputProvider.jps(project.modules)
private val moduleOutputProvider = jpsModuleOutputProvider(project.modules)
override var classesOutputDirectory: Path
get() = Path.of(JpsPathUtil.urlToPath(JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(project).outputUrl))
@@ -0,0 +1,43 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.impl
import org.jetbrains.intellij.build.ModuleOutputProvider
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
import java.nio.file.Files
import java.nio.file.NoSuchFileException
import java.nio.file.Path
internal fun jpsModuleOutputProvider(modules: List<JpsModule>): ModuleOutputProvider {
return object : ModuleOutputProvider {
private val nameToModule = modules.associateByTo(HashMap(modules.size)) { it.name }
override fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean): ByteArray? {
val outputRoots = getModuleOutputRoots(module, forTests)
val outputDir = outputRoots.singleOrNull() ?: error("More than one output root for module '${module.name}': ${outputRoots.joinToString()}")
val file = outputDir.resolve(relativePath)
try {
return Files.readAllBytes(file)
}
catch (_: NoSuchFileException) {
return null
}
}
override fun findModule(name: String): JpsModule? = nameToModule.get(name.removeSuffix("._test"))
override fun findRequiredModule(name: String): JpsModule {
return checkNotNull(findModule(name)) {
"Cannot find required module '$name' in the project"
}
}
override fun getModuleOutputRoots(module: JpsModule, forTests: Boolean): List<Path> {
val url = JpsJavaExtensionService.getInstance().getOutputUrl(/* module = */ module, /* forTests = */ forTests)
requireNotNull(url) {
"Output directory for ${module.name} isn't set"
}
return listOf(Path.of(JpsPathUtil.urlToPath(url)))
}
}
}
@@ -91,12 +91,12 @@ class LinuxDistributionBuilder(
createFrontendContextForLaunchers(context)?.let { clientContext ->
writeLinuxVmOptions(distBinDir, clientContext)
generateLauncherScript(
distBinDir, arch, ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, clientContext, targetLibcImpl
distBinDir, arch, ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, targetLibcImpl, clientContext
)
}
generateReadme(targetPath)
generateVersionMarker(targetPath, context)
customizer.copyAdditionalFiles(context, targetPath, arch)
customizer.copyAdditionalFiles(targetPath, arch, context)
}
}
}
@@ -137,12 +137,12 @@ class LinuxDistributionBuilder(
"linux_tar_gz_${arch.name}"
) { _ ->
val suffix = suffix(arch, targetLibcImpl)
buildTarGz(arch, runtimeDir, osAndArchSpecificDistPath, suffix)
buildTarGz(arch = arch, runtimeDir = runtimeDir, unixDistPath = osAndArchSpecificDistPath, suffix = suffix)
}
if (targetLibcImpl != LinuxLibcImpl.MUSL) {
launch(Dispatchers.IO + CoroutineName("build Snap package")) {
buildSnapPackage(runtimeDir, osAndArchSpecificDistPath, arch, targetLibcImpl)
buildSnapPackage(runtimeDir = runtimeDir, unixDistPath = osAndArchSpecificDistPath, arch = arch, targetLibcImpl = targetLibcImpl)
}
}
@@ -179,8 +179,9 @@ class LinuxDistributionBuilder(
)
}
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence<String> =
customizer.generateExecutableFilesPatterns(context, includeRuntime, arch, libc)
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence<String> {
return customizer.generateExecutableFilesPatterns(includeRuntime = includeRuntime, arch = arch, targetLibcImpl = libc, context = context)
}
private val rootDirectoryName: String
get() = customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)
@@ -202,7 +203,7 @@ class LinuxDistributionBuilder(
}
val productJsonDir = context.paths.tempDir.resolve("linux.dist.product-info.json${suffix}")
val productJsonFile = writeProductJsonFile(productJsonDir, arch, withRuntime = runtimeDir != null)
val productJsonFile = writeProductJsonFile(targetDir = productJsonDir, arch = arch, withRuntime = runtimeDir != null)
dirs.add(productJsonDir)
spanBuilder("build Linux tar.gz")
@@ -216,7 +217,13 @@ class LinuxDistributionBuilder(
context.notifyArtifactBuilt(tarProductInfoJsonPath)
context.notifyArtifactBuilt(tarPath)
checkExecutablePermissions(tarPath, rootDirectoryName, includeRuntime = runtimeDir != null, arch, this@LinuxDistributionBuilder.targetLibcImpl)
checkExecutablePermissions(
distribution = tarPath,
root = rootDirectoryName,
includeRuntime = runtimeDir != null,
arch = arch,
libc = this@LinuxDistributionBuilder.targetLibcImpl,
)
}
tarPath
}
@@ -227,11 +234,6 @@ class LinuxDistributionBuilder(
"${appInfo.majorVersion}.${appInfo.minorVersion}${if (versionSuffix.isEmpty()) "" else "-${versionSuffix}"}"
}
private fun getSnapArchName(arch: JvmArchitecture) = when (arch) {
JvmArchitecture.x64 -> "amd64"
JvmArchitecture.aarch64 -> "arm64"
}
private fun getSnapArtifactName(snapName: String, arch: JvmArchitecture): String = "${snapName}_${snapVersion}_${getSnapArchName(arch)}.snap"
private suspend fun buildSnapPackage(runtimeDir: Path, unixDistPath: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl) {
@@ -244,7 +246,7 @@ class LinuxDistributionBuilder(
}
buildSnapPackage(snapName, runtimeDir, unixDistPath, arch, targetLibcImpl)
customizer.snapLegacyAliases.forEach {
buildSnapPackage(it, runtimeDir, unixDistPath, arch, targetLibcImpl)
buildSnapPackage(snapName = it, runtimeDir = runtimeDir, unixDistPath = unixDistPath, arch = arch, targetLibcImpl = targetLibcImpl)
}
}
@@ -279,9 +281,7 @@ class LinuxDistributionBuilder(
)
)
copyFile(iconPngPath, snapDir.resolve("$snapName.png"))
val snapcraftTemplate = context.paths.communityHomeDir.resolve(
"platform/build-scripts/resources/linux/snap/snapcraft-template.yaml"
)
val snapcraftTemplate = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/snap/snapcraft-template.yaml")
val snapcraftConfig = snapDir.resolve("snapcraft.yaml")
substituteTemplatePlaceholders(
inputFile = snapcraftTemplate,
@@ -306,7 +306,7 @@ class LinuxDistributionBuilder(
val productJsonDir = context.paths.tempDir.resolve("linux.dist.snap.$snapName.product-info.json.$architecture")
val productJsonFile = writeProductJsonFile(productJsonDir, arch)
val installationDirectories = listOf(context.paths.distAllDir, unixDistPath, runtimeDir)
validateProductJson(jsonText = productJsonFile.readText(), installationDirectories, installationArchives = emptyList(), context)
validateProductJson(jsonText = productJsonFile.readText(), installationDirectories = installationDirectories, installationArchives = emptyList(), context = context)
val resultDir = snapDir.resolve("result")
Files.createDirectories(resultDir)
@@ -385,101 +385,112 @@ class LinuxDistributionBuilder(
writeProductInfoJson(file, json, context)
return file
}
}
private fun generateVersionMarker(unixDistPath: Path, context: BuildContext) {
val targetDir = unixDistPath.resolve("lib")
Files.createDirectories(targetDir)
Files.writeString(targetDir.resolve("build-marker-" + context.fullBuildNumber), context.fullBuildNumber)
}
private fun generateVersionMarker(unixDistPath: Path, context: BuildContext) {
val targetDir = unixDistPath.resolve("lib")
Files.createDirectories(targetDir)
Files.writeString(targetDir.resolve("build-marker-${context.fullBuildNumber}"), context.fullBuildNumber)
}
private fun generateScripts(distBinDir: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl, context: BuildContext) {
Files.createDirectories(distBinDir)
private fun generateScripts(distBinDir: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl, context: BuildContext) {
Files.createDirectories(distBinDir)
val sourceScriptDir = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts")
Files.newDirectoryStream(sourceScriptDir).use {
for (file in it) {
val fileName = file.fileName.toString()
if (fileName != EXECUTABLE_TEMPLATE_NAME) {
copyScript(file, distBinDir.resolve(fileName), additionalTemplateValues = emptyList(), context)
}
val sourceScriptDir = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts")
Files.newDirectoryStream(sourceScriptDir).use {
for (file in it) {
val fileName = file.fileName.toString()
if (fileName != EXECUTABLE_TEMPLATE_NAME) {
copyScript(sourceFile = file, targetFile = distBinDir.resolve(fileName), additionalTemplateValues = emptyList(), context = context)
}
}
copyInspectScript(context, distBinDir)
generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = emptyList(), context, targetLibcImpl)
}
private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture, context: BuildContext) {
val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch)
copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName))
copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html"))
}
copyInspectScript(context, distBinDir)
private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List<String>, context: BuildContext, targetLibcImpl: LinuxLibcImpl) {
val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions")
generateLauncherScript(distBinDir = distBinDir, arch = arch, nonCustomizableJvmArgs = emptyList(), targetLibcImpl = targetLibcImpl, context = context)
}
val defaultXmxParameter = try {
Files.readAllLines(vmOptionsPath).firstOrNull { it.startsWith("-Xmx") }
?: throw IllegalStateException("-Xmx was not found in '$vmOptionsPath'")
}
catch (e: NoSuchFileException) {
throw IllegalStateException("File '$vmOptionsPath' should be already generated at this point", e)
}
private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture, context: BuildContext) {
val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch)
copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName))
copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html"))
}
val classPathJars = context.bootClassPathJarNames
var classPath = $$"CLASS_PATH=\"$IDE_HOME/lib/$${classPathJars[0]}\""
for (i in 1 until classPathJars.size) {
classPath += $$"\nCLASS_PATH=\"$CLASS_PATH:$IDE_HOME/lib/$${classPathJars[i]}\""
}
private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List<String>, targetLibcImpl: LinuxLibcImpl, context: BuildContext) {
val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions")
val additionalJvmArguments = mutableListOf<String>()
// https://youtrack.jetbrains.com/issue/IDEA-304440
// "-Djdk.lang.Process.launchMechanism=vfork"
if (targetLibcImpl == LinuxLibcImpl.MUSL) {
additionalJvmArguments.add("-Djdk.lang.Process.launchMechanism=vfork")
}
additionalJvmArguments.addAll(context.getAdditionalJvmArguments(OsFamily.LINUX, arch, isScript = true) + nonCustomizableJvmArgs)
val defaultXmxParameter = try {
Files.readAllLines(vmOptionsPath).firstOrNull { it.startsWith("-Xmx") }
?: throw IllegalStateException("-Xmx was not found in '$vmOptionsPath'")
}
catch (e: NoSuchFileException) {
throw IllegalStateException("File '$vmOptionsPath' should be already generated at this point", e)
}
val additionalTemplateValues = listOf(
Pair("vm_options", context.productProperties.baseFileName),
Pair("system_selector", context.systemSelector),
Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")),
Pair("ide_default_xmx", defaultXmxParameter.trim()),
Pair("class_path", classPath),
Pair("main_class_name", context.ideMainClassName),
)
val classPathJars = context.bootClassPathJarNames
var classPath = $$"CLASS_PATH=\"$IDE_HOME/lib/$${classPathJars[0]}\""
for (i in 1 until classPathJars.size) {
classPath += $$"\nCLASS_PATH=\"$CLASS_PATH:$IDE_HOME/lib/$${classPathJars[i]}\""
}
val template = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts/${EXECUTABLE_TEMPLATE_NAME}")
copyScript(template, distBinDir.resolve("${context.productProperties.baseFileName}.sh"), additionalTemplateValues, context)
val additionalJvmArguments = mutableListOf<String>()
// https://youtrack.jetbrains.com/issue/IDEA-304440
// "-Djdk.lang.Process.launchMechanism=vfork"
if (targetLibcImpl == LinuxLibcImpl.MUSL) {
additionalJvmArguments.add("-Djdk.lang.Process.launchMechanism=vfork")
}
additionalJvmArguments.addAll(context.getAdditionalJvmArguments(os = OsFamily.LINUX, arch = arch, isScript = true) + nonCustomizableJvmArgs)
val additionalTemplateValues = listOf(
Pair("vm_options", context.productProperties.baseFileName),
Pair("system_selector", context.systemSelector),
Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")),
Pair("ide_default_xmx", defaultXmxParameter.trim()),
Pair("class_path", classPath),
Pair("main_class_name", context.ideMainClassName),
)
private fun copyScript(sourceFile: Path, targetFile: Path, additionalTemplateValues: List<Pair<String, String>>, context: BuildContext) {
// Until CR (\r) will be removed from the repository checkout, we need to filter it out from Unix-style scripts
// https://youtrack.jetbrains.com/issue/IJI-526/Force-git-to-use-LF-line-endings-in-working-copy-of-via-gitattri
substituteTemplatePlaceholders(
inputFile = sourceFile,
outputFile = targetFile,
placeholder = "__",
values = listOf(
Pair("product_full", context.applicationInfo.fullProductName),
Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)),
Pair("product_vendor", context.applicationInfo.shortCompanyName),
Pair("product_code", context.applicationInfo.productCode),
Pair("script_name", "${context.productProperties.baseFileName}.sh"),
) + additionalTemplateValues,
mustUseAllPlaceholders = false,
convertToUnixLineEndings = true,
)
}
val template = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts/${EXECUTABLE_TEMPLATE_NAME}")
copyScript(
sourceFile = template,
targetFile = distBinDir.resolve("${context.productProperties.baseFileName}.sh"),
additionalTemplateValues = additionalTemplateValues,
context = context,
)
}
private fun copyScript(sourceFile: Path, targetFile: Path, additionalTemplateValues: List<Pair<String, String>>, context: BuildContext) {
// Until CR (\r) will be removed from the repository checkout, we need to filter it out from Unix-style scripts
// https://youtrack.jetbrains.com/issue/IJI-526/Force-git-to-use-LF-line-endings-in-working-copy-of-via-gitattri
substituteTemplatePlaceholders(
inputFile = sourceFile,
outputFile = targetFile,
placeholder = "__",
values = listOf(
Pair("product_full", context.applicationInfo.fullProductName),
Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)),
Pair("product_vendor", context.applicationInfo.shortCompanyName),
Pair("product_code", context.applicationInfo.productCode),
Pair("script_name", "${context.productProperties.baseFileName}.sh"),
) + additionalTemplateValues,
mustUseAllPlaceholders = false,
convertToUnixLineEndings = true,
)
}
private fun writeLinuxVmOptions(distBinDir: Path, context: BuildContext): Path {
val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions")
val vmOptions = generateVmOptions(context).asSequence() + sequenceOf("-Dsun.tools.attach.tmp.only=true", "-Dawt.lock.fair=true")
writeVmOptions(file = vmOptionsPath, vmOptions = vmOptions, separator = "\n")
return vmOptionsPath
}
private fun writeLinuxVmOptions(distBinDir: Path, context: BuildContext): Path {
val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions")
val vmOptions = VmOptionsGenerator.generate(context).asSequence() + sequenceOf("-Dsun.tools.attach.tmp.only=true", "-Dawt.lock.fair=true")
VmOptionsGenerator.writeVmOptions(vmOptionsPath, vmOptions, separator = "\n")
return vmOptionsPath
}
private fun suffix(arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl): String {
return suffix(arch) + if (targetLibcImpl == LinuxLibcImpl.MUSL) "-musl" else ""
}
private fun suffix(arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl): String =
suffix(arch) + if (targetLibcImpl == LinuxLibcImpl.MUSL) "-musl" else ""
private fun getSnapArchName(arch: JvmArchitecture) = when (arch) {
JvmArchitecture.x64 -> "amd64"
JvmArchitecture.aarch64 -> "arm64"
}
@@ -118,7 +118,7 @@ class MacDistributionBuilder(
override suspend fun copyFilesForOsDistribution(targetPath: Path, arch: JvmArchitecture) {
withContext(Dispatchers.IO) {
doCopyExtraFiles(targetPath, arch, copyDistFiles = true)
doCopyExtraFiles(macDistDir = targetPath, arch = arch, copyDistFiles = true)
}
}
@@ -327,7 +327,7 @@ class MacDistributionBuilder(
}
override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence<String> =
customizer.generateExecutableFilesPatterns(context, includeRuntime, arch)
customizer.generateExecutableFilesPatterns(includeRuntime, arch, context)
private suspend fun buildForArch(
arch: JvmArchitecture,
@@ -490,15 +490,15 @@ class MacDistributionBuilder(
}
}
validateProductJson(targetFile, pathInArchive = "${zipRoot}/Resources", macDistributionBuilder.context)
validateProductJson(archiveFile = targetFile, pathInArchive = "${zipRoot}/Resources", context = macDistributionBuilder.context)
}
}
private fun writeMacOsVmOptions(distBinDir: Path, context: BuildContext): Path {
val executable = context.productProperties.baseFileName
val vmOptions = VmOptionsGenerator.generate(context).asSequence() + sequenceOf("-Dapple.awt.application.appearance=system")
val vmOptions = generateVmOptions(context).asSequence() + sequenceOf("-Dapple.awt.application.appearance=system")
val vmOptionsPath = distBinDir.resolve("${executable}.vmoptions")
VmOptionsGenerator.writeVmOptions(vmOptionsPath, vmOptions, separator = "\n")
writeVmOptions(vmOptionsPath, vmOptions, separator = "\n")
return vmOptionsPath
}
@@ -567,8 +567,9 @@ class MacDistributionBuilder(
}
}
private fun getMacZipRoot(customizer: MacDistributionCustomizer, context: BuildContext): String =
"${customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)}/Contents"
private fun getMacZipRoot(customizer: MacDistributionCustomizer, context: BuildContext): String {
return "${customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)}/Contents"
}
private val publishSitArchive: Boolean
get() = !context.isStepSkipped(BuildOptions.MAC_SIT_PUBLICATION_STEP)
@@ -636,76 +637,75 @@ class MacDistributionBuilder(
NioFiles.deleteRecursively(tempDir)
}
}
}
private fun prepareDmgBuildScripts(tempDir: Path, staple: Boolean, customizer: MacDistributionCustomizer, context: BuildContext): Path {
NioFiles.deleteRecursively(tempDir)
Files.createDirectories(tempDir)
val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png")
Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath), dmgImageCopy)
val scriptsDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts")
Files.copy(scriptsDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
NioFiles.setExecutable(tempDir.resolve("makedmg.sh"))
Files.copy(scriptsDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
Files.copy(scriptsDir.resolve("staple.sh"), tempDir.resolve("staple.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
val entrypoint = tempDir.resolve("build.sh")
Files.writeString(
entrypoint,
Files.readString(scriptsDir.resolve("build-template.sh"))
.resolveTemplateVar("staple", "$staple")
.resolveTemplateVar("appName", context.fullBuildNumber)
.resolveTemplateVar("contentSigned", "${context.isMacCodeSignEnabled}")
.resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}")
)
NioFiles.setExecutable(entrypoint)
return entrypoint
private fun prepareDmgBuildScripts(tempDir: Path, staple: Boolean, customizer: MacDistributionCustomizer, context: BuildContext): Path {
NioFiles.deleteRecursively(tempDir)
Files.createDirectories(tempDir)
val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png")
Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath), dmgImageCopy)
val scriptsDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts")
Files.copy(scriptsDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
NioFiles.setExecutable(tempDir.resolve("makedmg.sh"))
Files.copy(scriptsDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
Files.copy(scriptsDir.resolve("staple.sh"), tempDir.resolve("staple.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
val entrypoint = tempDir.resolve("build.sh")
Files.writeString(
entrypoint,
Files.readString(scriptsDir.resolve("build-template.sh"))
.resolveTemplateVar("staple", "$staple")
.resolveTemplateVar("appName", context.fullBuildNumber)
.resolveTemplateVar("contentSigned", "${context.isMacCodeSignEnabled}")
.resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}")
)
NioFiles.setExecutable(entrypoint)
return entrypoint
}
private fun String.resolveTemplateVar(variable: String, value: String): String {
val reference = "%$variable%"
check(contains(reference)) { "No $reference is found in:\n'$this'" }
return replace(reference, value)
}
private fun publishDmgBuildScripts(entrypoint: Path, tempDir: Path, context: BuildContext) {
val artifactDir = context.paths.artifactDir.resolve("macos-dmg-build")
artifactDir.createDirectories()
synchronized("$artifactDir".intern()) {
tempDir.listDirectoryEntries().forEach {
Files.copy(it, artifactDir.resolve(it.name), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
}
val message = """
To build .dmg(s):
1. transfer .sit(s) to macOS host;
2. transfer ${artifactDir.name}/ content to the same folder;
3. execute ${entrypoint.name} from Terminal.
.dmg(s) will be built in the same folder.
""".trimIndent()
artifactDir.resolve("README.txt").writeText(message)
context.messages.info(message)
context.notifyArtifactBuilt(artifactDir)
}
}
private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, arch: JvmArchitecture, context: BuildContext) {
if (context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) {
return
}
private fun String.resolveTemplateVar(variable: String, value: String): String {
val reference = "%$variable%"
check(contains(reference)) { "No $reference is found in:\n'$this'" }
return replace(reference, value)
val tempSit = Files.createTempDirectory(context.paths.tempDir, "sit-")
try {
spanBuilder("extracting ${sitFile.name}").use(Dispatchers.IO) {
Decompressor.Zip(sitFile)
.withZipExtensions()
.extract(tempSit)
}
RepairUtilityBuilder.generateManifest(context, tempSit.resolve(sitRoot), OsFamily.MACOS, arch)
}
private fun publishDmgBuildScripts(entrypoint: Path, tempDir: Path, context: BuildContext) {
val artifactDir = context.paths.artifactDir.resolve("macos-dmg-build")
artifactDir.createDirectories()
synchronized("$artifactDir".intern()) {
tempDir.listDirectoryEntries().forEach {
Files.copy(it, artifactDir.resolve(it.name), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES)
}
val message = """
To build .dmg(s):
1. transfer .sit(s) to macOS host;
2. transfer ${artifactDir.name}/ content to the same folder;
3. execute ${entrypoint.name} from Terminal.
.dmg(s) will be built in the same folder.
""".trimIndent()
artifactDir.resolve("README.txt").writeText(message)
context.messages.info(message)
context.notifyArtifactBuilt(artifactDir)
}
}
private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, arch: JvmArchitecture, context: BuildContext) {
if (context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) {
return
}
val tempSit = Files.createTempDirectory(context.paths.tempDir, "sit-")
try {
spanBuilder("extracting ${sitFile.name}").use(Dispatchers.IO) {
Decompressor.Zip(sitFile)
.withZipExtensions()
.extract(tempSit)
}
RepairUtilityBuilder.generateManifest(context, tempSit.resolve(sitRoot), OsFamily.MACOS, arch)
}
finally {
withContext(Dispatchers.IO + NonCancellable) {
@OptIn(ExperimentalPathApi::class)
tempSit.deleteRecursively()
}
finally {
withContext(Dispatchers.IO + NonCancellable) {
@OptIn(ExperimentalPathApi::class)
tempSit.deleteRecursively()
}
}
}
@@ -1,56 +0,0 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.impl
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
import java.nio.file.Files
import java.nio.file.NoSuchFileException
import java.nio.file.Path
interface ModuleOutputProvider {
companion object {
fun jps(modules: List<JpsModule>): ModuleOutputProvider {
return object : ModuleOutputProvider {
private val nameToModule = modules.associateByTo(HashMap(modules.size)) { it.name }
override fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean): ByteArray? {
val outputRoots = getModuleOutputRoots(module, forTests)
val outputDir = outputRoots.singleOrNull() ?: error("More than one output root for module '${module.name}': ${outputRoots.joinToString()}")
val file = outputDir.resolve(relativePath)
try {
return Files.readAllBytes(file)
}
catch (_: NoSuchFileException) {
return null
}
}
override fun findModule(name: String): JpsModule? = nameToModule.get(name.removeSuffix("._test"))
override fun findRequiredModule(name: String): JpsModule {
return checkNotNull(findModule(name)) {
"Cannot find required module '$name' in the project"
}
}
override fun getModuleOutputRoots(module: JpsModule, forTests: Boolean): List<Path> {
val url = JpsJavaExtensionService.getInstance().getOutputUrl(/* module = */ module, /* forTests = */ forTests)
requireNotNull(url) {
"Output directory for ${module.name} isn't set"
}
return listOf(Path.of(JpsPathUtil.urlToPath(url)))
}
}
}
}
fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean = false): ByteArray?
fun findModule(name: String): JpsModule?
fun findRequiredModule(name: String): JpsModule
fun getModuleOutputRoots(module: JpsModule, forTests: Boolean = false): List<Path>
}
@@ -9,8 +9,8 @@ import org.jdom.Element
import org.jetbrains.annotations.TestOnly
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.CompatibleBuildRange
import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.classPath.DescriptorSearchScope
import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.classPath.XIncludeElementResolverImpl
import org.jetbrains.intellij.build.classPath.embedContentModule
import org.jetbrains.intellij.build.getUnprocessedPluginXmlContent
@@ -633,7 +633,7 @@ internal class TestingTasksImpl(context: CompilationContext, private val options
val customMemoryOptions = options.jvmMemoryOptions?.trim()?.split(Regex("\\s+"))?.takeIf { it.isNotEmpty() }
jvmArgs.addAll(
index = 0,
elements = VmOptionsGenerator.generate(
elements = generateVmOptions(
isEAP = true,
customVmMemoryOptions = if (customMemoryOptions == null) mapOf("-Xms" to "750m", "-Xmx" to "1024m") else emptyMap(),
additionalVmOptions = customMemoryOptions ?: emptyList(),
@@ -9,92 +9,92 @@ import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.name
object VmOptionsGenerator {
private const val DEFAULT_MIN_HEAP = "128m"
private const val DEFAULT_MAX_HEAP = "2048m"
/** duplicates RepositoryHelper.CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY */
private const val CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY = "intellij.plugins.custom.built.in.repository.url"
private val COMMON_VM_OPTIONS: List<String> = listOf(
"-XX:JbrShrinkingGcMaxHeapFreeRatio=40", // IJPL-181469. Used in a couple with AppIdleMemoryCleaner.runGc()
"-XX:ReservedCodeCacheSize=512m",
"-XX:+HeapDumpOnOutOfMemoryError",
"-XX:-OmitStackTraceInFastThrow",
"-XX:CICompilerCount=2",
"-XX:+IgnoreUnrecognizedVMOptions", // allowing the JVM to start even with outdated options stuck in user configs
"-XX:+UnlockDiagnosticVMOptions",
"-XX:TieredOldPercentage=100000",
"-ea",
"-Dsun.io.useCanonCaches=false",
"-Dsun.java2d.metal=true",
"-Djbr.catch.SIGABRT=true",
"-Djdk.http.auth.tunneling.disabledSchemes=\"\"",
"-Djdk.attach.allowAttachSelf=true",
"-Djdk.module.illegalAccess.silent=true",
"-Djdk.nio.maxCachedBufferSize=2097152",
"-Djava.util.zip.use.nio.for.zip.file.access=true", // IJPL-149160
"-Dkotlinx.coroutines.debug=off",
)
private const val DEFAULT_MIN_HEAP = "128m"
private const val DEFAULT_MAX_HEAP = "2048m"
/** duplicates RepositoryHelper.CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY */
private const val CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY = "intellij.plugins.custom.built.in.repository.url"
private val COMMON_VM_OPTIONS: List<String> = listOf(
"-XX:JbrShrinkingGcMaxHeapFreeRatio=40", // IJPL-181469. Used in a couple with AppIdleMemoryCleaner.runGc()
"-XX:ReservedCodeCacheSize=512m",
"-XX:+HeapDumpOnOutOfMemoryError",
"-XX:-OmitStackTraceInFastThrow",
"-XX:CICompilerCount=2",
"-XX:+IgnoreUnrecognizedVMOptions", // allowing the JVM to start even with outdated options stuck in user configs
"-XX:+UnlockDiagnosticVMOptions",
"-XX:TieredOldPercentage=100000",
"-ea",
"-Dsun.io.useCanonCaches=false",
"-Dsun.java2d.metal=true",
"-Djbr.catch.SIGABRT=true",
"-Djdk.http.auth.tunneling.disabledSchemes=\"\"",
"-Djdk.attach.allowAttachSelf=true",
"-Djdk.module.illegalAccess.silent=true",
"-Djdk.nio.maxCachedBufferSize=2097152",
"-Djava.util.zip.use.nio.for.zip.file.access=true", // IJPL-149160
"-Dkotlinx.coroutines.debug=off",
)
fun generate(context: BuildContext): List<String> = generate(
context.applicationInfo.isEAP,
context.productProperties.customJvmMemoryOptions,
context.productProperties.additionalVmOptions.let {
fun generateVmOptions(context: BuildContext): List<String> {
return generateVmOptions(
isEAP = context.applicationInfo.isEAP,
customVmMemoryOptions = context.productProperties.customJvmMemoryOptions,
additionalVmOptions = context.productProperties.additionalVmOptions.let {
val url = computeCustomPluginRepositoryUrl(context)
if (url == null) it else it + "-D${CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY}=${url}"
},
context.productProperties.platformPrefix,
platformPrefix = context.productProperties.platformPrefix,
)
}
internal fun generateVmOptions(
isEAP: Boolean,
customVmMemoryOptions: Map<String, String>,
additionalVmOptions: List<String>,
platformPrefix: String?,
): List<String> {
val result = ArrayList<String>()
val memory = LinkedHashMap<String, String>(customVmMemoryOptions)
memory.putIfAbsent("-Xms", DEFAULT_MIN_HEAP)
memory.putIfAbsent("-Xmx", DEFAULT_MAX_HEAP)
for ((k, v) in memory) {
result.add(k + v)
}
internal fun generate(
isEAP: Boolean,
customVmMemoryOptions: Map<String, String>,
additionalVmOptions: List<String>,
platformPrefix: String?,
): List<String> {
val result = ArrayList<String>()
val memory = LinkedHashMap<String, String>(customVmMemoryOptions)
memory.putIfAbsent("-Xms", DEFAULT_MIN_HEAP)
memory.putIfAbsent("-Xmx", DEFAULT_MAX_HEAP)
for ((k, v) in memory) {
result += k + v
}
result.addAll(COMMON_VM_OPTIONS)
result += COMMON_VM_OPTIONS
if (isMultiRoutingFileSystemEnabledForProduct(platformPrefix)) {
result.addAll(MULTI_ROUTING_FILE_SYSTEM_VMOPTIONS)
}
if (isMultiRoutingFileSystemEnabledForProduct(platformPrefix)) {
result.addAll(MULTI_ROUTING_FILE_SYSTEM_VMOPTIONS)
}
result.addAll(additionalVmOptions)
result += additionalVmOptions
if (isEAP) {
var index = result.indexOf("-ea")
if (index < 0) index = result.indexOfFirst { it.startsWith("-D") }
if (index < 0) index = result.size
result.add(index, "-XX:MaxJavaStackTraceDepth=10000") // must be consistent with `ConfigImportHelper#updateVMOptions`
}
if (isEAP) {
var index = result.indexOf("-ea")
if (index < 0) index = result.indexOfFirst { it.startsWith("-D") }
if (index < 0) index = result.size
result.add(index, "-XX:MaxJavaStackTraceDepth=10000") // must be consistent with `ConfigImportHelper#updateVMOptions`
}
return result
}
return result
}
private fun computeCustomPluginRepositoryUrl(context: BuildContext): String? {
val artifactsServer = context.proprietaryBuildTools.artifactsServer
if (artifactsServer != null && context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins) {
val builtinPluginsRepoUrl = artifactsServer.urlToArtifact(context, "${context.nonBundledPlugins.name}/plugins.xml")
if (builtinPluginsRepoUrl != null) {
if (builtinPluginsRepoUrl.startsWith("http:")) {
context.messages.logErrorAndThrow("Insecure artifact server: ${builtinPluginsRepoUrl}")
}
return builtinPluginsRepoUrl
private fun computeCustomPluginRepositoryUrl(context: BuildContext): String? {
val artifactsServer = context.proprietaryBuildTools.artifactsServer
if (artifactsServer != null && context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins) {
val builtinPluginsRepoUrl = artifactsServer.urlToArtifact(context, "${context.nonBundledPlugins.name}/plugins.xml")
if (builtinPluginsRepoUrl != null) {
if (builtinPluginsRepoUrl.startsWith("http:")) {
context.messages.logErrorAndThrow("Insecure artifact server: ${builtinPluginsRepoUrl}")
}
return builtinPluginsRepoUrl
}
return null
}
return null
}
internal fun writeVmOptions(file: Path, vmOptions: Sequence<String>, separator: String) {
Files.writeString(file, vmOptions.joinToString(separator, postfix = separator), StandardCharsets.US_ASCII)
}
internal fun writeVmOptions(file: Path, vmOptions: Sequence<String>, separator: String) {
Files.writeString(file, vmOptions.joinToString(separator, postfix = separator), StandardCharsets.US_ASCII)
}
@@ -84,31 +84,31 @@ internal class WindowsDistributionBuilder(
copyDir(sourceBinDir.resolve(arch.dirName), distBinDir)
copyDir(sourceBinDir, distBinDir, dirFilter = { it == sourceBinDir })
copyDir(sourceDir = sourceBinDir, targetDir = distBinDir, dirFilter = { it == sourceBinDir })
copyFileToDir(NativeBinaryDownloader.getRestarter(context, OsFamily.WINDOWS, arch), distBinDir)
generateBuildTxt(context, targetPath)
copyDistFiles(context, targetPath, OsFamily.WINDOWS, arch, WindowsLibcImpl.DEFAULT)
copyDistFiles(context = context, newDir = targetPath, os = OsFamily.WINDOWS, arch = arch, libcImpl = WindowsLibcImpl.DEFAULT)
Files.writeString(distBinDir.resolve(PROPERTIES_FILE_NAME), StringUtilRt.convertLineSeparators(ideaProperties!!, "\r\n"))
Files.copy(computeIcoPath(context), distBinDir.resolve("${context.productProperties.baseFileName}.ico"), StandardCopyOption.REPLACE_EXISTING)
if (customizer.includeBatchLaunchers) {
generateScripts(distBinDir, arch)
generateScripts(distBinDir, arch, context)
}
writeVmOptions(distBinDir)
buildWinLauncher(targetPath, arch, context, copyLicense = true)
buildWinLauncher(winDistPath = targetPath, arch = arch, copyLicense = true, context = context)
createFrontendContextForLaunchers(context)?.let { clientContext ->
writeWindowsVmOptions(distBinDir, clientContext)
buildWinLauncher(targetPath, arch, clientContext, copyLicense = false)
buildWinLauncher(winDistPath = targetPath, arch = arch, copyLicense = false, context = clientContext)
}
customizer.copyAdditionalFiles(context, targetPath, arch)
customizer.copyAdditionalFiles(targetPath, arch, context)
}
context.executeStep(spanBuilder("sign windows"), BuildOptions.WIN_SIGN_STEP) {
@@ -184,129 +184,14 @@ internal class WindowsDistributionBuilder(
Span.current().addEvent("comparing .zip and .exe is not supported on ${SystemInfoRt.OS_NAME}")
}
else {
checkThatExeInstallerAndZipWithJbrAreTheSame(zipWithJbrPath, exePath, arch, context.paths.tempDir)
checkThatExeInstallerAndZipWithJbrAreTheSame(zipPath = zipWithJbrPath, exePath = exePath, arch = arch, tempDir = context.paths.tempDir, context = context)
}
}
}
override suspend fun writeProductInfoFile(targetDir: Path, arch: JvmArchitecture): Path = writeProductJsonFile(targetDir, arch, context)
private fun generateScripts(distBinDir: Path, arch: JvmArchitecture) {
val fullName = context.applicationInfo.fullProductName
val baseName = context.productProperties.baseFileName
val scriptName = "${baseName}.bat"
val vmOptionsFileName = "${baseName}64.exe"
val classPathJars = context.bootClassPathJarNames
var classPath = "SET \"CLASS_PATH=%IDE_HOME%\\lib\\${classPathJars[0]}\""
for (i in 1 until classPathJars.size) {
classPath += "\nSET \"CLASS_PATH=%CLASS_PATH%;%IDE_HOME%\\lib\\${classPathJars[i]}\""
}
val additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch, isScript = true)
val winScripts = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/win/scripts")
val actualScriptNames = Files.newDirectoryStream(winScripts).use { dirStream -> dirStream.map { it.fileName.toString() }.sorted() }
val expectedScriptNames = listOf("executable-template.bat", "format.bat", "inspect.bat", "ltedit.bat")
check(actualScriptNames == expectedScriptNames) {
"Expected script names '${expectedScriptNames.joinToString(separator = " ")}', " +
"but got '${actualScriptNames.joinToString(separator = " ")}' " +
"in ${winScripts}. Please review ${WindowsDistributionBuilder::class.java.name} and update accordingly"
}
substituteTemplatePlaceholders(
inputFile = winScripts.resolve("executable-template.bat"),
outputFile = distBinDir.resolve(scriptName),
placeholder = "@@",
values = listOf(
Pair("product_full", fullName),
Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)),
Pair("product_vendor", context.applicationInfo.shortCompanyName),
Pair("vm_options", vmOptionsFileName),
Pair("system_selector", context.systemSelector),
Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")),
Pair("class_path", classPath),
Pair("base_name", baseName),
Pair("main_class_name", context.ideMainClassName),
)
)
val inspectScript = context.productProperties.inspectCommandName
for (fileName in listOf("format.bat", "inspect.bat", "ltedit.bat")) {
substituteTemplatePlaceholders(
inputFile = winScripts.resolve(fileName),
outputFile = distBinDir.resolve(fileName),
placeholder = "@@",
values = listOf(
Pair("product_full", fullName),
Pair("script_name", scriptName),
)
)
}
if (inspectScript != "inspect") {
val targetPath = distBinDir.resolve("${inspectScript}.bat")
Files.move(distBinDir.resolve("inspect.bat"), targetPath)
context.patchInspectScript(targetPath)
}
FileSet(distBinDir)
.include("*.bat")
.enumerate()
.forEach { file ->
transformFile(file) { target ->
Files.writeString(target, toDosLineEndings(Files.readString(file)))
}
}
}
override fun writeVmOptions(distBinDir: Path) : Path =
writeWindowsVmOptions(distBinDir, context)
private suspend fun createBuildWinZipTask(
runtimeDir: Path?,
zipNameSuffix: String,
winDistPath: Path,
arch: JvmArchitecture,
customizer: WindowsDistributionCustomizer,
context: BuildContext
): Path {
val baseName = context.productProperties.getBaseArtifactName(context)
val targetFile = context.paths.artifactDir.resolve("${baseName}${zipNameSuffix}.zip")
val targetFileProductInfoJson = targetFile.resolveProductInfoJsonSibling()
spanBuilder("build Windows ${zipNameSuffix}.zip distribution")
.setAttribute("targetFile", targetFile.toString())
.setAttribute("arch", arch.dirName)
.use {
val dirs = mutableListOf(context.paths.distAllDir, winDistPath)
if (runtimeDir != null) {
dirs.add(runtimeDir)
}
val productJsonDir = context.paths.tempDir.resolve("win.dist.product-info.json.zip${zipNameSuffix}")
val productJsonFile = writeProductJsonFile(targetDir = productJsonDir, arch = arch, context = context, withRuntime = runtimeDir != null)
dirs.add(productJsonDir)
copyFile(productJsonFile, targetFileProductInfoJson)
val zipPrefix = customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)
val dirMap = dirs.associateWithTo(LinkedHashMap(dirs.size)) { zipPrefix }
if (context.options.compressZipFiles) {
zipWithCompression(targetFile, dirMap)
}
else {
zip(targetFile, dirMap, AddDirEntriesMode.NONE)
}
validateProductJson(targetFile, zipPrefix, context)
context.notifyArtifactBuilt(targetFile)
context.notifyArtifactBuilt(targetFileProductInfoJson)
targetFile
}
return targetFile
}
override fun writeVmOptions(distBinDir: Path) : Path = writeWindowsVmOptions(distBinDir, context)
override fun distributionFilesBuilt(arch: JvmArchitecture): List<Path> {
val archSuffix = suffix(arch)
@@ -322,182 +207,302 @@ internal class WindowsDistributionBuilder(
}
override fun isRuntimeBundled(file: Path): Boolean = !file.name.contains(customizer.zipArchiveWithoutBundledJreSuffix)
}
private suspend fun buildWinLauncher(winDistPath: Path, arch: JvmArchitecture, context: BuildContext, copyLicense: Boolean) {
spanBuilder("build Windows executable").use {
val communityHome = context.paths.communityHomeDir
val appInfo = context.applicationInfo
val executableBaseName = "${context.productProperties.baseFileName}64"
val launcherPropertiesPath = context.paths.tempDir.resolve("launcher-${arch.dirName}.properties")
val icoFile = computeIcoPath(context)
private fun generateScripts(distBinDir: Path, arch: JvmArchitecture, context: BuildContext) {
val fullName = context.applicationInfo.fullProductName
val baseName = context.productProperties.baseFileName
val scriptName = "${baseName}.bat"
val vmOptionsFileName = "${baseName}64.exe"
val productVersion = context.buildNumber.replace(".SNAPSHOT", ".0") + ".0".repeat(3 - context.buildNumber.count { it == '.' })
val launcherProperties = listOf(
"CompanyName" to appInfo.companyName,
"LegalCopyright" to "Copyright 2000-${LocalDate.now().year} ${appInfo.companyName}",
"FileDescription" to appInfo.productNameWithEdition,
"ProductName" to appInfo.productNameWithEdition,
"ProductVersion" to "$productVersion-${appInfo.productCode}", // "242.1234.56.0-IU"
val classPathJars = context.bootClassPathJarNames
var classPath = "SET \"CLASS_PATH=%IDE_HOME%\\lib\\${classPathJars[0]}\""
for (i in 1 until classPathJars.size) {
classPath += "\nSET \"CLASS_PATH=%CLASS_PATH%;%IDE_HOME%\\lib\\${classPathJars[i]}\""
}
val additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch, isScript = true)
val winScripts = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/win/scripts")
val actualScriptNames = Files.newDirectoryStream(winScripts).use { dirStream -> dirStream.map { it.fileName.toString() }.sorted() }
val expectedScriptNames = listOf("executable-template.bat", "format.bat", "inspect.bat", "ltedit.bat")
check(actualScriptNames == expectedScriptNames) {
"Expected script names '${expectedScriptNames.joinToString(separator = " ")}', " +
"but got '${actualScriptNames.joinToString(separator = " ")}' " +
"in ${winScripts}. Please review ${WindowsDistributionBuilder::class.java.name} and update accordingly"
}
substituteTemplatePlaceholders(
inputFile = winScripts.resolve("executable-template.bat"),
outputFile = distBinDir.resolve(scriptName),
placeholder = "@@",
values = listOf(
Pair("product_full", fullName),
Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)),
Pair("product_vendor", context.applicationInfo.shortCompanyName),
Pair("vm_options", vmOptionsFileName),
Pair("system_selector", context.systemSelector),
Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")),
Pair("class_path", classPath),
Pair("base_name", baseName),
Pair("main_class_name", context.ideMainClassName),
)
)
val inspectScript = context.productProperties.inspectCommandName
for (fileName in listOf("format.bat", "inspect.bat", "ltedit.bat")) {
substituteTemplatePlaceholders(
inputFile = winScripts.resolve(fileName),
outputFile = distBinDir.resolve(fileName),
placeholder = "@@",
values = listOf(
Pair("product_full", fullName),
Pair("script_name", scriptName),
)
Files.writeString(launcherPropertiesPath, launcherProperties.joinToString(separator = System.lineSeparator()) { (k, v) -> "${k}=${v}" })
)
}
val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.WINDOWS, arch)
val outputPath = winDistPath.resolve("bin/${executableBaseName}.exe")
if (inspectScript != "inspect") {
val targetPath = distBinDir.resolve("${inspectScript}.bat")
Files.move(distBinDir.resolve("inspect.bat"), targetPath)
context.patchInspectScript(targetPath)
}
if (copyLicense) {
copyFile(licensePath, winDistPath.resolve("license/launcher-third-party-libraries.html"))
FileSet(distBinDir)
.include("*.bat")
.enumerate()
.forEach { file ->
transformFile(file) { target ->
Files.writeString(target, toDosLineEndings(Files.readString(file)))
}
}
}
private suspend fun createBuildWinZipTask(
runtimeDir: Path?,
zipNameSuffix: String,
winDistPath: Path,
arch: JvmArchitecture,
customizer: WindowsDistributionCustomizer,
context: BuildContext
): Path {
val baseName = context.productProperties.getBaseArtifactName(context)
val targetFile = context.paths.artifactDir.resolve("${baseName}${zipNameSuffix}.zip")
val targetFileProductInfoJson = targetFile.resolveProductInfoJsonSibling()
spanBuilder("build Windows ${zipNameSuffix}.zip distribution")
.setAttribute("targetFile", targetFile.toString())
.setAttribute("arch", arch.dirName)
.use {
val dirs = mutableListOf(context.paths.distAllDir, winDistPath)
if (runtimeDir != null) {
dirs.add(runtimeDir)
}
val generatorModule = context.findRequiredModule("intellij.tools.launcherGenerator")
runJava(
mainClass = "com.pme.launcher.LauncherGeneratorMain",
args = listOf(
execPath.absolutePathString(),
"${communityHome}/native/XPlatLauncher/resources/windows/resource.h",
launcherPropertiesPath.absolutePathString(),
icoFile.absolutePathString(),
outputPath.absolutePathString(),
),
jvmArgs = listOf("-Djava.awt.headless=true"),
context.getModuleRuntimeClasspath(generatorModule, forTests = false),
context.stableJavaExecutable
)
val productJsonDir = context.paths.tempDir.resolve("win.dist.product-info.json.zip${zipNameSuffix}")
val productJsonFile = writeProductJsonFile(targetDir = productJsonDir, arch = arch, context = context, withRuntime = runtimeDir != null)
dirs.add(productJsonDir)
copyFile(productJsonFile, targetFileProductInfoJson)
val zipPrefix = customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)
val dirMap = dirs.associateWithTo(LinkedHashMap(dirs.size)) { zipPrefix }
if (context.options.compressZipFiles) {
zipWithCompression(targetFile, dirMap)
}
else {
zip(targetFile, dirMap, AddDirEntriesMode.NONE)
}
validateProductJson(targetFile, zipPrefix, context)
context.notifyArtifactBuilt(targetFile)
context.notifyArtifactBuilt(targetFileProductInfoJson)
targetFile
}
}
private fun computeIcoPath(context: BuildContext): Path {
val customizer = context.windowsDistributionCustomizer!!
val icoPath = (if (context.applicationInfo.isEAP) customizer.icoPathForEAP else null) ?: customizer.icoPath
requireNotNull(icoPath) { "`WindowsDistributionCustomizer#icoPath` must be set" }
return Path.of(icoPath)
}
return targetFile
}
private suspend fun checkThatExeInstallerAndZipWithJbrAreTheSame(zipPath: Path, exePath: Path, arch: JvmArchitecture, tempDir: Path) = CompareDistributionsSemaphore.withPermit {
fun compareStreams(stream1: InputStream, stream2: InputStream): Boolean {
val b1 = ByteArray(DEFAULT_BUFFER_SIZE)
val b2 = ByteArray(DEFAULT_BUFFER_SIZE)
stream1.use { s1 ->
stream2.use { s2 ->
while (true) {
val l1 = s1.readNBytes(b1, 0, b1.size)
val l2 = s2.readNBytes(b2, 0, b2.size)
if (l1 != l2) return false
if (l1 <= 0) return true
if (!Arrays.equals(b1, 0, l1, b2, 0, l2)) return false
}
private suspend fun buildWinLauncher(winDistPath: Path, arch: JvmArchitecture, copyLicense: Boolean, context: BuildContext) {
spanBuilder("build Windows executable").use {
val communityHome = context.paths.communityHomeDir
val appInfo = context.applicationInfo
val executableBaseName = "${context.productProperties.baseFileName}64"
val launcherPropertiesPath = context.paths.tempDir.resolve("launcher-${arch.dirName}.properties")
val icoFile = computeIcoPath(context)
val productVersion = context.buildNumber.replace(".SNAPSHOT", ".0") + ".0".repeat(3 - context.buildNumber.count { it == '.' })
val launcherProperties = listOf(
"CompanyName" to appInfo.companyName,
"LegalCopyright" to "Copyright 2000-${LocalDate.now().year} ${appInfo.companyName}",
"FileDescription" to appInfo.productNameWithEdition,
"ProductName" to appInfo.productNameWithEdition,
"ProductVersion" to "$productVersion-${appInfo.productCode}", // "242.1234.56.0-IU"
)
Files.writeString(launcherPropertiesPath, launcherProperties.joinToString(separator = System.lineSeparator()) { (k, v) -> "${k}=${v}" })
val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.WINDOWS, arch)
val outputPath = winDistPath.resolve("bin/${executableBaseName}.exe")
if (copyLicense) {
copyFile(licensePath, winDistPath.resolve("license/launcher-third-party-libraries.html"))
}
val generatorModule = context.findRequiredModule("intellij.tools.launcherGenerator")
runJava(
mainClass = "com.pme.launcher.LauncherGeneratorMain",
args = listOf(
execPath.absolutePathString(),
"${communityHome}/native/XPlatLauncher/resources/windows/resource.h",
launcherPropertiesPath.absolutePathString(),
icoFile.absolutePathString(),
outputPath.absolutePathString(),
),
jvmArgs = listOf("-Djava.awt.headless=true"),
context.getModuleRuntimeClasspath(generatorModule, forTests = false),
context.stableJavaExecutable
)
}
}
private suspend fun checkThatExeInstallerAndZipWithJbrAreTheSame(
zipPath: Path,
exePath: Path,
arch: JvmArchitecture,
tempDir: Path,
context: BuildContext,
) = CompareDistributionsSemaphore.withPermit {
fun compareStreams(stream1: InputStream, stream2: InputStream): Boolean {
val b1 = ByteArray(DEFAULT_BUFFER_SIZE)
val b2 = ByteArray(DEFAULT_BUFFER_SIZE)
stream1.use { s1 ->
stream2.use { s2 ->
while (true) {
val l1 = s1.readNBytes(b1, 0, b1.size)
val l2 = s2.readNBytes(b2, 0, b2.size)
if (l1 != l2) return false
if (l1 <= 0) return true
if (!Arrays.equals(b1, 0, l1, b2, 0, l2)) return false
}
}
}
}
val tempExe = withContext(Dispatchers.IO) { Files.createTempDirectory(tempDir, "exe-${arch.dirName}") }
try {
withContext(Dispatchers.IO) {
spanBuilder("compare zip and exe contents")
.setAttribute("zipPath", zipPath.toString())
.setAttribute("exePath", exePath.toString())
.use {
runProcess(args = listOf("7z", "x", "-bd", exePath.toString()), workingDir = tempExe)
// deleting NSIS-related files that appear after manual unpacking of .exe installer and do not belong to its contents
@Suppress("SpellCheckingInspection")
NioFiles.deleteRecursively(tempExe.resolve($$"$PLUGINSDIR"))
Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe.nsis"))
Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe"))
val tempExe = withContext(Dispatchers.IO) { Files.createTempDirectory(tempDir, "exe-${arch.dirName}") }
try {
withContext(Dispatchers.IO) {
spanBuilder("compare zip and exe contents")
.setAttribute("zipPath", zipPath.toString())
.setAttribute("exePath", exePath.toString())
.use {
runProcess(args = listOf("7z", "x", "-bd", exePath.toString()), workingDir = tempExe)
// deleting NSIS-related files that appear after manual unpacking of .exe installer and do not belong to its contents
@Suppress("SpellCheckingInspection")
NioFiles.deleteRecursively(tempExe.resolve($$"$PLUGINSDIR"))
Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe.nsis"))
Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe"))
val extraInZip = ArrayList<String>()
val differ = ArrayList<String>()
ZipFile.Builder().setSeekableByteChannel(Files.newByteChannel(zipPath)).get().use { zipFile ->
zipFile.entries.asSequence()
.filter { !it.isDirectory }.toList()
.mapConcurrent(Runtime.getRuntime().availableProcessors().coerceAtLeast(4)) { entry ->
val entryPath = Path.of(entry.name)
val fileInExe = tempExe.resolve(entryPath)
if (!fileInExe.exists()) {
extraInZip.add(entryPath.toString())
val extraInZip = ArrayList<String>()
val differ = ArrayList<String>()
ZipFile.Builder().setSeekableByteChannel(Files.newByteChannel(zipPath)).get().use { zipFile ->
zipFile.entries.asSequence()
.filter { !it.isDirectory }.toList()
.mapConcurrent(Runtime.getRuntime().availableProcessors().coerceAtLeast(4)) { entry ->
val entryPath = Path.of(entry.name)
val fileInExe = tempExe.resolve(entryPath)
if (!fileInExe.exists()) {
extraInZip.add(entryPath.toString())
}
else {
if (fileInExe.fileSize() != entry.size) {
differ.add(entryPath.toString())
}
else {
if (fileInExe.fileSize() != entry.size) {
else if (entry.size < 2 * FileUtilRt.MEGABYTE) {
if (!fileInExe.readBytes().contentEquals(zipFile.getInputStream(entry).readAllBytes())) {
differ.add(entryPath.toString())
}
else if (entry.size < 2 * FileUtilRt.MEGABYTE) {
if (!fileInExe.readBytes().contentEquals(zipFile.getInputStream(entry).readAllBytes())) {
differ.add(entryPath.toString())
}
}
else if (!compareStreams(fileInExe.inputStream().buffered(FileUtilRt.MEGABYTE), zipFile.getInputStream(entry).buffered(FileUtilRt.MEGABYTE))) {
differ.add(entryPath.toString())
}
NioFiles.deleteRecursively(fileInExe)
}
else if (!compareStreams(fileInExe.inputStream().buffered(FileUtilRt.MEGABYTE), zipFile.getInputStream(entry).buffered(FileUtilRt.MEGABYTE))) {
differ.add(entryPath.toString())
}
NioFiles.deleteRecursively(fileInExe)
}
}
val extraInExe = Files.walk(tempExe)
.filter { Files.isRegularFile(it) }
.map { tempExe.relativize(it).toString() }
.toList()
if (extraInExe.isNotEmpty() || extraInZip.isNotEmpty() || differ.isNotEmpty()) {
error(buildString {
if (extraInZip.isNotEmpty()) {
append("Files present only in ZIP:\n")
extraInZip.forEach { append(" ").append(it).append('\n') }
}
if (extraInExe.isNotEmpty()) {
append("Files present only in EXE:\n")
extraInExe.forEach { append(" ").append(it).append('\n') }
}
if (differ.isNotEmpty()) {
append("Files with different content:\n")
differ.forEach { append(" ").append(it).append('\n') }
}
})
}
}
}
}
if (!context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) {
RepairUtilityBuilder.generateManifest(context, tempExe, OsFamily.WINDOWS, arch)
}
val extraInExe = Files.walk(tempExe)
.filter { Files.isRegularFile(it) }
.map { tempExe.relativize(it).toString() }
.toList()
if (extraInExe.isNotEmpty() || extraInZip.isNotEmpty() || differ.isNotEmpty()) {
error(buildString {
if (extraInZip.isNotEmpty()) {
append("Files present only in ZIP:\n")
extraInZip.forEach { append(" ").append(it).append('\n') }
}
if (extraInExe.isNotEmpty()) {
append("Files present only in EXE:\n")
extraInExe.forEach { append(" ").append(it).append('\n') }
}
if (differ.isNotEmpty()) {
append("Files with different content:\n")
differ.forEach { append(" ").append(it).append('\n') }
}
})
}
}
}
finally {
withContext(Dispatchers.IO + NonCancellable) {
NioFiles.deleteRecursively(tempExe)
}
if (!context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) {
RepairUtilityBuilder.generateManifest(context, tempExe, OsFamily.WINDOWS, arch)
}
}
private fun writeWindowsVmOptions(distBinDir: Path, context: BuildContext): Path {
val vmOptionsFile = distBinDir.resolve("${context.productProperties.baseFileName}64.exe.vmoptions")
val vmOptions = VmOptionsGenerator.generate(context).asSequence()
VmOptionsGenerator.writeVmOptions(vmOptionsFile, vmOptions, separator = "\r\n")
return vmOptionsFile
}
private suspend fun writeProductJsonFile(targetDir: Path, arch: JvmArchitecture, context: BuildContext, withRuntime: Boolean = true): Path {
val embeddedFrontendLaunchData = generateEmbeddedFrontendLaunchData(arch, OsFamily.WINDOWS, context) {
"bin/${it.productProperties.baseFileName}64.exe.vmoptions"
finally {
withContext(Dispatchers.IO + NonCancellable) {
NioFiles.deleteRecursively(tempExe)
}
val qodanaCustomLaunchData = generateQodanaLaunchData(context, arch, OsFamily.WINDOWS)
val json = generateProductInfoJson(
relativePathToBin = "bin",
builtinModules = context.builtinModule,
launch = listOf(
ProductInfoLaunchData.create(
os = OsFamily.WINDOWS.osName,
arch = arch.dirName,
launcherPath = "bin/${context.productProperties.baseFileName}64.exe",
javaExecutablePath = if (withRuntime) "jbr/bin/java.exe" else null,
vmOptionsFilePath = "bin/${context.productProperties.baseFileName}64.exe.vmoptions",
bootClassPathJarNames = context.bootClassPathJarNames,
additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch),
mainClass = context.ideMainClassName,
customCommands = listOfNotNull(embeddedFrontendLaunchData, qodanaCustomLaunchData),
)
),
context)
val file = targetDir.resolve(PRODUCT_INFO_FILE_NAME)
writeProductInfoJson(file, json, context)
return file
}
private fun toDosLineEndings(x: String): String = x.replace("\r", "").replace("\n", "\r\n")
}
private fun computeIcoPath(context: BuildContext): Path {
val customizer = context.windowsDistributionCustomizer!!
val icoPath = (if (context.applicationInfo.isEAP) customizer.icoPathForEAP else null) ?: customizer.icoPath
requireNotNull(icoPath) { "`WindowsDistributionCustomizer#icoPath` must be set" }
return Path.of(icoPath)
}
private fun writeWindowsVmOptions(distBinDir: Path, context: BuildContext): Path {
val vmOptionsFile = distBinDir.resolve("${context.productProperties.baseFileName}64.exe.vmoptions")
val vmOptions = generateVmOptions(context).asSequence()
writeVmOptions(file = vmOptionsFile, vmOptions = vmOptions, separator = "\r\n")
return vmOptionsFile
}
private suspend fun writeProductJsonFile(targetDir: Path, arch: JvmArchitecture, context: BuildContext, withRuntime: Boolean = true): Path {
val embeddedFrontendLaunchData = generateEmbeddedFrontendLaunchData(arch, OsFamily.WINDOWS, context) {
"bin/${it.productProperties.baseFileName}64.exe.vmoptions"
}
val qodanaCustomLaunchData = generateQodanaLaunchData(context, arch, OsFamily.WINDOWS)
val json = generateProductInfoJson(
relativePathToBin = "bin",
builtinModules = context.builtinModule,
launch = listOf(
ProductInfoLaunchData.create(
os = OsFamily.WINDOWS.osName,
arch = arch.dirName,
launcherPath = "bin/${context.productProperties.baseFileName}64.exe",
javaExecutablePath = if (withRuntime) "jbr/bin/java.exe" else null,
vmOptionsFilePath = "bin/${context.productProperties.baseFileName}64.exe.vmoptions",
bootClassPathJarNames = context.bootClassPathJarNames,
additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch),
mainClass = context.ideMainClassName,
customCommands = listOfNotNull(embeddedFrontendLaunchData, qodanaCustomLaunchData),
)
),
context)
val file = targetDir.resolve(PRODUCT_INFO_FILE_NAME)
writeProductInfoJson(file, json, context)
return file
}
private fun toDosLineEndings(x: String): String = x.replace("\r", "").replace("\n", "\r\n")
@@ -7,8 +7,8 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.MAVEN_REPO
import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.classPath.DescriptorSearchScope
import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.classPath.PluginBuildDescriptor
import org.jetbrains.intellij.build.classPath.XIncludeElementResolverImpl
import org.jetbrains.intellij.build.getUnprocessedPluginXmlContent
@@ -22,8 +22,8 @@ import kotlinx.coroutines.launch
import org.apache.commons.compress.archivers.zip.Zip64Mode
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.SearchableOptionSetDescriptor
import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.classPath.PluginBuildDescriptor
import org.jetbrains.intellij.build.executeStep
import org.jetbrains.intellij.build.getUnprocessedPluginXmlContent
@@ -10,8 +10,8 @@ import org.jdom.Element
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.ContentModuleFilter
import org.jetbrains.intellij.build.FrontendModuleFilter
import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.classPath.DescriptorSearchScope
import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH
import org.jetbrains.intellij.build.classPath.XIncludeElementResolverImpl
import org.jetbrains.intellij.build.classPath.resolveAndEmbedContentModuleDescriptor
import org.jetbrains.intellij.build.impl.PlatformJarNames.PRODUCT_BACKEND_JAR
@@ -2,7 +2,6 @@
package org.jetbrains.intellij.build.productLayout
import com.intellij.openapi.application.PathManager
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
import org.jetbrains.intellij.build.BuildPaths
import java.nio.file.Path
@@ -460,15 +459,5 @@ object CommunityModuleSets : ModuleSetProvider {
// Reason: Rider uses custom module loading mode due to early backend startup requirements.
// Products that need rd.common include it explicitly in their product files.
}
}
/**
* Represents a content module with optional loading attribute.
*
* @param name Module name
* @param loading Optional loading mode (e.g., ModuleLoadingRule.EMBEDDED)
*/
data class ContentModule(
@JvmField val name: String,
@JvmField val loading: ModuleLoadingRule? = null,
)
}
@@ -0,0 +1,95 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.productLayout
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.json.Json
import java.nio.file.Path
/**
* Parses JSON argument from command line in the format --json or --json='{"filter":"...","value":"..."}'.
* Returns null for full JSON output, or JsonFilter for filtered output.
*
* @param arg The command line argument (e.g., "--json" or "--json={...}")
* @return JsonFilter if filter is specified, null for full JSON output
*/
fun parseJsonArgument(arg: String): JsonFilter? {
return if (arg.contains("=")) {
val filterJson = arg.substringAfter("=")
try {
Json.decodeFromString<JsonFilter>(filterJson)
}
catch (e: Exception) {
System.err.println("Failed to parse JSON filter: $filterJson")
System.err.println("Error: ${e.message}")
null
}
}
else {
null // Full JSON output
}
}
/**
* Generic main runner for module set generation and analysis.
* Supports two modes:
* 1. JSON mode (--json): Outputs comprehensive analysis as JSON to stdout
* 2. Default mode: Generates XML files for module sets and products
*
* This is the generic orchestration logic that was previously duplicated in UltimateModuleSets.kt.
* Now module set providers (community, ultimate) can call this function with their specific data.
*
* @param args Command line arguments
* @param communityModuleSets Module sets from community
* @param ultimateModuleSets Module sets from ultimate (or empty for community-only)
* @param communitySourceFile Source file path for community module sets
* @param ultimateSourceFile Source file path for ultimate module sets (or null for community-only)
* @param projectRoot Project root path
* @param generateXmlImpl Lambda to generate XML files (default mode implementation)
*/
fun runModuleSetMain(
args: Array<String>,
communityModuleSets: List<ModuleSet>,
ultimateModuleSets: List<ModuleSet>,
communitySourceFile: String,
ultimateSourceFile: String?,
projectRoot: Path,
generateXmlImpl: suspend () -> Unit
) {
// Parse --json arg with optional filter
val jsonArg = args.firstOrNull { it.startsWith("--json") }
when {
jsonArg != null -> {
runBlocking(Dispatchers.Default) {
// Prepare all module sets with metadata
val communityModuleSetsWithMeta = communityModuleSets.map {
ModuleSetMetadata(it, "community", communitySourceFile)
}
val ultimateModuleSetsWithMeta = if (ultimateSourceFile != null) {
ultimateModuleSets.map {
ModuleSetMetadata(it, "ultimate", ultimateSourceFile)
}
} else {
emptyList()
}
val allModuleSets = communityModuleSetsWithMeta + ultimateModuleSetsWithMeta
// Discover all products (reuse existing logic from productXmlFileGenerator)
val products = discoverAllProductsForJson(projectRoot)
// Parse filter from --json='{"filter":"..."}' format
val filter = parseJsonArgument(jsonArg)
// Stream JSON to stdout
streamModuleAnalysisJson(allModuleSets, products, projectRoot, filter)
}
}
else -> {
// Default mode: Generate XML files
runBlocking(Dispatchers.Default) {
generateXmlImpl()
}
}
}
}
@@ -0,0 +1,106 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.productLayout
/**
* Product spec helper functions for reusable product content fragments.
* These functions return ProductModulesContentSpec instances that can be composed using include().
*
* Use these helpers to reduce duplication across product specifications while keeping
* ProductModulesContentSpec immutable and declarative (suitable for future YAML representation).
*/
/**
* Common capability aliases shared across most ultimate products.
* Bundles frequently repeated capability declarations to reduce duplication.
*
* Includes:
* - Run targets support
* - Microservices capabilities
* - ML inline completion
* - IDE provisioner
* - Marketplace integration
*
* Usage:
* ```
* override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules {
* include(commonCapabilityAliases())
* // ...
* }
* ```
*/
fun commonCapabilityAliases(): ProductModulesContentSpec = productModules {
alias("com.intellij.modules.run.targets")
alias("com.intellij.modules.microservices")
alias("com.intellij.ml.inline.completion")
alias("com.intellij.platform.ide.provisioner")
alias("com.intellij.marketplace")
}
/**
* Python capability aliases for non-PyCharm products.
* Enables Python support in multi-language IDEs like GoLand, DataGrip, CLion, RustRover, etc.
*
* Includes:
* - Python core capabilities
* - Python in mini-IDE capabilities
* - Python in non-PyCharm IDE capabilities
*
* Usage:
* ```
* override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules {
* include(pythonMiniIdeCapabilities())
* // ...
* }
* ```
*/
fun pythonMiniIdeCapabilities(): ProductModulesContentSpec = productModules {
alias("com.intellij.modules.python-core-capable")
alias("com.intellij.modules.python-in-mini-ide-capable")
alias("com.intellij.modules.python-in-non-pycharm-ide-capable")
}
/**
* Common platform includes repeated across most ultimate products.
* Bundles deprecatedInclude statements for legacy XML resource inclusion.
*
* Includes:
* - Platform lang plugin resources
* - Structural search resources
* - Remote servers implementation
* - Ultimate edition resources
*
* Usage:
* ```
* override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules {
* include(platformCommonIncludes())
* // ...
* }
* ```
*/
fun platformCommonIncludes(): ProductModulesContentSpec = productModules {
deprecatedInclude("intellij.platform.resources", "META-INF/PlatformLangPlugin.xml")
deprecatedInclude("intellij.platform.structuralSearch", "META-INF/structuralsearch.xml")
deprecatedInclude("intellij.platform.remoteServers.impl", "intellij.platform.remoteServers.impl.xml")
deprecatedInclude("intellij.platform.commercial", "META-INF/ultimate.xml")
}
/**
* Extensions for native development IDEs (CLion, GoLand, RustRover).
* Combines process elevation support with native debugger capability.
*
* Includes:
* - Process elevation module set (for operations requiring elevated privileges)
* - Native debugger plugin capability alias
*
* Usage:
* ```
* override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules {
* include(nativeDevExtensions())
* // ...
* }
* ```
*/
fun nativeDevExtensions(): ProductModulesContentSpec = productModules {
moduleSet(CommunityModuleSets.elevation())
alias("com.intellij.modules.nativeDebug-plugin-capable")
}
@@ -1,498 +0,0 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build.productLayout
import com.intellij.openapi.util.JDOMUtil
import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule
import kotlinx.serialization.json.Json
import org.jetbrains.intellij.build.BuildPaths
import org.jetbrains.intellij.build.dev.createProductProperties
import org.jetbrains.intellij.build.findFileInModuleLibraryDependencies
import org.jetbrains.intellij.build.findFileInModuleSources
import org.jetbrains.intellij.build.impl.ModuleOutputProvider
import org.jetbrains.intellij.build.isModuleNameLikeFilename
import org.jetbrains.jps.model.serialization.JpsMavenSettings
import org.jetbrains.jps.model.serialization.JpsSerializationManager
import java.nio.file.Files
import java.nio.file.Path
/**
* Generates an XML file for a module set.
* Used to maintain backward compatibility with XML-based module set loading.
*
* @param moduleSet The module set to generate XML for
* @param outputDir The directory where the XML file will be written
* @param label Description label ("community" or "ultimate") for header generation
* @return Result containing file status and statistics
*/
fun generateModuleSetXml(moduleSet: ModuleSet, outputDir: Path, label: String): ModuleSetFileResult {
val fileName = "intellij.moduleSets.${moduleSet.name}.xml"
val outputPath = outputDir.resolve(fileName)
val buildResult = buildModuleSetXml(moduleSet, label)
// determine change status
val status = if (Files.exists(outputPath)) {
val existingContent = Files.readString(outputPath)
if (existingContent == buildResult.xml) FileChangeStatus.UNCHANGED else FileChangeStatus.MODIFIED
}
else {
FileChangeStatus.CREATED
}
// Only write if changed
if (status != FileChangeStatus.UNCHANGED) {
Files.writeString(outputPath, buildResult.xml)
}
return ModuleSetFileResult(fileName, status, buildResult.directModuleCount)
}
/**
* Represents a single content block in a product plugin.xml.
* Each block corresponds to a module set or additional modules section.
*/
data class ContentBlock(
/** Source identifier for the block (e.g., "essential", "vcs", "additional") */
val source: String,
/** List of modules with their effective loading modes */
val modules: List<ModuleWithLoading>,
)
/**
* A module with its effective loading mode after applying overrides and exclusions.
*/
data class ModuleWithLoading(
/** Module name */
val name: String,
/** Effective loading mode (null means default/no attribute) */
val loading: ModuleLoadingRule?,
)
/**
* Result of building product content XML.
* Contains the generated XML string, content blocks, and module-to-set chain mapping.
*/
data class ProductContentBuildResult(
/** Generated XML content as string */
val xml: String,
/** List of content blocks generated from the spec */
val contentBlocks: List<ContentBlock>,
/** Mapping from module name to its module set chain as list (e.g., ["parent", "child"]) */
val moduleToSetChainMapping: Map<String, List<String>>,
)
/**
* Generates complete product plugin.xml file from programmatic specification.
*
* @param isUltimateBuild Whether this is an Ultimate build (vs. Community build)
* @return Result containing file status and statistics
*/
fun generateProductXml(
pluginXmlPath: Path,
spec: ProductModulesContentSpec,
productName: String,
productPropertiesClass: String,
moduleOutputProvider: ModuleOutputProvider,
projectRoot: Path,
isUltimateBuild: Boolean,
): ProductFileResult {
// Determine which generator to recommend based on plugin.xml file location
// Community products are under community/ directory, Ultimate products are not
val generatorCommand = (if (pluginXmlPath.toString().contains("/community/")) "CommunityModuleSets" else "UltimateModuleSets") + ".main()"
// Build complete plugin.xml file
val buildResult = buildProductContentXml(
spec = spec,
moduleOutputProvider = moduleOutputProvider,
inlineXmlIncludes = false,
inlineModuleSets = false,
productPropertiesClass = productPropertiesClass,
generatorCommand = generatorCommand,
isUltimateBuild = isUltimateBuild
)
// Compare with existing file if it exists
val originalContent = Files.readString(pluginXmlPath)
val status = if (originalContent == buildResult.xml) {
FileChangeStatus.UNCHANGED
}
else {
FileChangeStatus.MODIFIED
}
// Only write if changed
if (status != FileChangeStatus.UNCHANGED) {
Files.writeString(pluginXmlPath, buildResult.xml)
}
// Calculate statistics using the contentBlocks from generation
val totalModules = buildResult.contentBlocks.sumOf { it.modules.size }
val relativePath = projectRoot.relativize(pluginXmlPath).toString()
return ProductFileResult(
productName = productName,
relativePath = relativePath,
status = status,
includeCount = spec.deprecatedXmlIncludes.size,
contentBlockCount = buildResult.contentBlocks.size,
totalModules = totalModules
)
}
/**
* Generates product XMLs for all products using programmatic content.
* Discovers products from dev-build.json and generates complete plugin.xml files.
*
* @param projectRoot The project root path
* @return Result containing generation statistics or null if dev-build.json doesn't exist
*/
suspend fun generateAllProductXmlFiles(projectRoot: Path): ProductGenerationResult {
val jsonContent = Files.readString(projectRoot.resolve(PRODUCT_REGISTRY_PATH))
val productToConfiguration = Json.decodeFromString<ProductConfigurationRegistry>(jsonContent).products
val project = JpsSerializationManager.getInstance().loadProject(projectRoot.toString(), mapOf("MAVEN_REPOSITORY" to JpsMavenSettings.getMavenRepositoryPath()), false)
val moduleOutputProvider = ModuleOutputProvider.jps(project.modules)
// Detect if this is an Ultimate build (projectRoot != communityRoot)
val isUltimateBuild = projectRoot != BuildPaths.COMMUNITY_ROOT.communityRoot
val productResults = mutableListOf<ProductFileResult>()
for ((productName, productConfig) in productToConfiguration) {
// Skip products without pluginXmlPath configured
val pluginXmlRelativePath = productConfig.pluginXmlPath ?: continue
val productProperties = createProductProperties(
productConfiguration = productConfig,
moduleOutputProvider = moduleOutputProvider,
projectDir = projectRoot,
platformPrefix = null
)
val spec = productProperties.getProductContentDescriptor() ?: continue
val pluginXmlPath = projectRoot.resolve(pluginXmlRelativePath)
val result = generateProductXml(
pluginXmlPath = pluginXmlPath,
spec = spec,
productName = productName,
moduleOutputProvider = moduleOutputProvider,
productPropertiesClass = productProperties::class.java.name,
projectRoot = projectRoot,
isUltimateBuild = isUltimateBuild,
)
productResults.add(result)
}
return ProductGenerationResult(productResults)
}
/**
* Generates content blocks and module-to-set chain mapping in a single hierarchical traversal.
* This optimized version eliminates redundant tree walking by computing both results simultaneously.
*
* @param spec The product modules specification
* @return Pair of (content blocks, module-to-set chain mapping)
*/
private fun generateContentBlocksWithChainMapping(
spec: ProductModulesContentSpec,
): Pair<List<ContentBlock>, Map<String, List<String>>> {
val contentBlocks = mutableListOf<ContentBlock>()
val moduleToChain = mutableMapOf<String, List<String>>()
val moduleToSets = mutableMapOf<String, MutableList<String>>()
val processedSets = HashSet<String>()
fun traverse(moduleSet: ModuleSet, chain: List<String>) {
val setName = "intellij.moduleSets.${moduleSet.name}"
val currentChain = chain + setName
// Skip if already processed
if (!processedSets.add(setName)) {
return
}
// Get direct modules for this set
val directModules = getDirectModules(moduleSet, spec.excludedModules)
// Build content block and track chains/duplicates in single pass
val modulesWithLoading = mutableListOf<ModuleWithLoading>()
for (module in directModules) {
// Track for duplicate detection
moduleToSets.computeIfAbsent(module.name) { mutableListOf() }.add(moduleSet.name)
// Track chain
moduleToChain[module.name] = currentChain
// Build loading info
val effectiveLoading = spec.moduleLoadingOverrides[module.name] ?: module.loading
modulesWithLoading.add(ModuleWithLoading(module.name, effectiveLoading))
}
if (modulesWithLoading.isNotEmpty()) {
contentBlocks.add(ContentBlock(moduleSet.name, modulesWithLoading))
}
// Recursively process nested sets
for (nestedSet in moduleSet.nestedSets) {
traverse(nestedSet, currentChain)
}
}
// Process all top-level module sets
for (moduleSet in spec.moduleSets) {
traverse(moduleSet, emptyList())
}
// Check for duplicates and FAIL if found
val duplicates = moduleToSets.filter { it.value.size > 1 }
if (duplicates.isNotEmpty()) {
val errorMessage = buildString {
appendLine("ERROR: Duplicate modules found across module sets:")
for ((moduleName, sets) in duplicates.toSortedMap()) {
appendLine(" - Module '$moduleName' appears in: ${sets.sorted().joinToString(", ")}")
}
appendLine()
appendLine("Each module must belong to exactly one module set.")
appendLine("Fix the module set definitions in CommunityModuleSets.kt or UltimateModuleSets.kt")
}
error(errorMessage)
}
// Add additional modules if any
val additionalModulesWithLoading = mutableListOf<ModuleWithLoading>()
for (module in spec.additionalModules) {
if (module.name !in spec.excludedModules) {
val effectiveLoading = spec.moduleLoadingOverrides[module.name] ?: module.loading
additionalModulesWithLoading.add(ModuleWithLoading(module.name, effectiveLoading))
}
}
if (additionalModulesWithLoading.isNotEmpty()) {
contentBlocks.add(ContentBlock("additional", additionalModulesWithLoading))
}
return Pair(contentBlocks, moduleToChain)
}
/**
* Appends a single module XML element with optional loading attribute.
*/
private fun StringBuilder.appendModuleLine(moduleWithLoading: ModuleWithLoading, indent: String = " ") {
append("$indent<module name=\"${moduleWithLoading.name}\"")
if (moduleWithLoading.loading != null) {
// convert enum to lowercase with hyphens (e.g., ON_DEMAND -> on-demand)
append(" loading=\"${moduleWithLoading.loading.name.lowercase().replace('_', '-')}\"")
}
append("/>\n")
}
/**
* Appends a content block with modules wrapped in editor fold.
*/
private fun StringBuilder.appendContentBlock(
blockSource: String,
modules: List<ModuleWithLoading>,
indent: String = " ",
) {
append("$indent<content namespace=\"jetbrains\">\n")
withEditorFold(this, "$indent ", blockSource) {
for (module in modules) {
appendModuleLine(module, "$indent ")
}
}
append("$indent</content>\n")
}
/**
* Collects and validates module aliases from the spec.
* Checks for duplicates and fails if any are found.
*
* @param spec The product modules specification
* @param inlineModuleSets Whether module sets are being inlined
* @return List of validated unique aliases
*/
private fun collectAndValidateAliases(spec: ProductModulesContentSpec, inlineModuleSets: Boolean): List<String> {
val aliasToSource = HashMap<String, String>()
// Collect product-level aliases
for (alias in spec.productModuleAliases) {
val existing = aliasToSource.put(alias, "product level")
if (existing != null) {
error("Duplicate alias '$alias' found at product level (already defined in: $existing)")
}
}
// When inlining module sets, also collect their aliases
if (inlineModuleSets) {
visitAllModuleSets(spec.moduleSets) { moduleSet ->
if (moduleSet.alias != null) {
val existing = aliasToSource.put(moduleSet.alias, "module set '${moduleSet.name}'")
if (existing != null) {
error("Duplicate alias '${moduleSet.alias}' in module set '${moduleSet.name}' (already defined in: $existing)")
}
}
}
}
return aliasToSource.keys.sorted()
}
/**
* Builds XML content for programmatic product modules.
* Generates module alias, xi:include directives (or inlined content), and `<content>` blocks for each module set.
*/
internal fun buildProductContentXml(
spec: ProductModulesContentSpec,
moduleOutputProvider: ModuleOutputProvider,
inlineXmlIncludes: Boolean,
inlineModuleSets: Boolean,
productPropertiesClass: String,
generatorCommand: String,
isUltimateBuild: Boolean,
): ProductContentBuildResult {
// Generate content blocks and module-to-set chain mapping in single pass
val (contentBlocks, moduleToSetChainMapping) = generateContentBlocksWithChainMapping(spec)
val xml = buildString {
// Header comments
append(" <!-- DO NOT EDIT: This file is auto-generated from Kotlin code -->\n")
append(" <!-- To regenerate, run 'Generate Product Layouts' or directly $generatorCommand -->\n")
append(" <!-- Source: $productPropertiesClass -->\n")
// Opening tag with optional XInclude namespace
val needsXiNamespace = (!inlineXmlIncludes && spec.deprecatedXmlIncludes.isNotEmpty()) ||
(!inlineModuleSets && spec.moduleSets.isNotEmpty())
// Check if PlatformLangPlugin.xml is included - if NOT, we need explicit id/name tags
// Products that include PlatformLangPlugin.xml inherit id/name from it
// Products without it (like Git Client) need explicit child tags
val includesPlatformLang = spec.deprecatedXmlIncludes.any {
it.resourcePath == "META-INF/PlatformLangPlugin.xml" ||
it.resourcePath == "META-INF/JavaIdePlugin.xml" ||
it.resourcePath == "META-INF/pycharm-core.xml"
}
if (needsXiNamespace) {
append("<idea-plugin xmlns:xi=\"http://www.w3.org/2001/XInclude\">\n")
}
else {
append("<idea-plugin>\n")
}
// Add id and name as child tags if PlatformLangPlugin.xml is not included
if (!includesPlatformLang) {
append(" <id>com.intellij</id>\n")
append(" <name>IDEA CORE</name>\n")
}
// Collect and validate aliases in a single pass
val validatedAliases = collectAndValidateAliases(spec, inlineModuleSets)
val aliasXml = buildModuleAliasesXml(validatedAliases)
if (aliasXml.isNotEmpty()) {
append(aliasXml)
append("\n")
}
// Generate xi:include directives or inline content
if (spec.deprecatedXmlIncludes.isNotEmpty()) {
generateXIncludes(spec = spec, moduleOutputProvider = moduleOutputProvider, inlineXmlIncludes = inlineXmlIncludes, sb = this, isUltimateBuild = isUltimateBuild)
}
// Generate module sets as xi:includes or inline content blocks
if (spec.moduleSets.isNotEmpty()) {
if (inlineModuleSets) {
// Generate single content block with all module sets inlined
append(" <content namespace=\"jetbrains\">\n")
for ((index, block) in contentBlocks.withIndex()) {
if (block.source == "additional") continue // Skip additional modules, handle separately
withEditorFold(this, " ", block.source) {
for (module in block.modules) {
appendModuleLine(module, " ")
}
}
// Add blank line between sections for readability (except after last block)
if (index < contentBlocks.size - 1) {
append("\n")
}
}
append(" </content>\n")
}
else {
// Generate xi:include directives for top-level module sets only (nested sets are resolved via parent includes)
for (moduleSet in spec.moduleSets) {
append(" <xi:include href=\"/META-INF/intellij.moduleSets.${moduleSet.name}.xml\"/>\n")
}
}
}
// Handle additional modules separately (they don't have XML files, inline them)
val additionalBlock = contentBlocks.firstOrNull { it.source == "additional" }
if (additionalBlock != null) {
appendContentBlock(additionalBlock.source, additionalBlock.modules)
}
// Closing tag
append("</idea-plugin>\n")
}
return ProductContentBuildResult(xml = xml, contentBlocks = contentBlocks, moduleToSetChainMapping = moduleToSetChainMapping)
}
private fun generateXIncludes(
spec: ProductModulesContentSpec,
moduleOutputProvider: ModuleOutputProvider,
inlineXmlIncludes: Boolean,
sb: StringBuilder,
isUltimateBuild: Boolean,
) {
for (include in spec.deprecatedXmlIncludes) {
// When inlining: skip ultimate-only xi-includes in Community builds
if (inlineXmlIncludes && include.ultimateOnly && !isUltimateBuild) {
continue
}
// Find the module and file
val module = moduleOutputProvider.findModule(include.moduleName)
val resourcePath = include.resourcePath
if (module == null) {
if (include.ultimateOnly) {
error("Ultimate-only module '${include.moduleName}' not found in Ultimate build - this is a configuration error (referenced in xi:include for '$resourcePath')")
}
error("Module '${include.moduleName}' not found (referenced in xi:include for '$resourcePath')")
}
val data = findFileInModuleSources(module, resourcePath)?.let { JDOMUtil.load(it) }
?: findFileInModuleLibraryDependencies(module = module, relativePath = resourcePath)?.let { JDOMUtil.load(it) }
?: error("Resource '$resourcePath' not found in module '${module.name}' sources or libraries (referenced in xi:include)")
if (inlineXmlIncludes && !include.optional) {
withEditorFold(sb, " ", "Inlined from ${include.moduleName}/$resourcePath") {
// Inline the actual XML content
for (element in data.children) {
sb.append(JDOMUtil.write(element).prependIndent(" "))
sb.append("\n")
}
}
sb.append("\n")
}
else {
// Generate xi:include with absolute path (resources are in /META-INF/... in jars)
// Wrap ultimate-only and optional xi-includes with xi:fallback for graceful handling
if (include.ultimateOnly || include.optional) {
sb.append(""" <xi:include href="${resourcePathToXIncludePath(resourcePath)}">""")
sb.append("\n")
sb.append(""" <xi:fallback/>""")
sb.append("\n")
sb.append(""" </xi:include>""")
sb.append("\n")
}
else {
sb.append(""" <xi:include href="${resourcePathToXIncludePath(resourcePath)}"/>""")
sb.append("\n")
}
}
}
}
private fun resourcePathToXIncludePath(resourcePath: String): String {
return if (isModuleNameLikeFilename(resourcePath)) resourcePath else "/$resourcePath"
}
@@ -1,132 +0,0 @@
# Module Sets
Module sets are collections of modules that can be referenced as a single entity in product configurations.
All module set files follow the naming pattern: `intellij.moduleSets.<category>.<subcategory>.xml`
## Creating a New Module Set
See `/create-module-set` slash command for detailed instructions on creating a new module set.
## IDE Module Sets
### [intellij.moduleSets.ide.common.xml](intellij.moduleSets.ide.common.xml)
A set of product modules for regular IDE. For example, for WebStorm, but not for Fleet backend.
This set includes [intellij.moduleSets.essential](#intellijmodulesetsessentialxml) and [intellij.moduleSets.vcs](#intellijmodulesetsvcsxml) sets.
### [intellij.moduleSets.essential.xml](intellij.moduleSets.essential.xml)
A set of product modules that are essential for any IDE based on IJ Platform.
Includes [intellij.moduleSets.libraries](#intellijmodulesetslibrariesxml).
Included in the [ide.common](#intellijmodulesetsidecommonxml) set.
### [intellij.moduleSets.ide.ultimate.xml](intellij.moduleSets.ide.ultimate.xml)
A set of modules common to all ultimate IDEs.
Includes observability features (coverage, profiling), IDE infrastructure (new UI onboarding, import settings, DAP, tips, registry cloud), and Language Server Protocol support.
Includes the [commercial](#intellijmodulesetscommercialxml) module set.
This set is used by WebStorm, GoLand, RustRover, RubyMine, PhpStorm, CLion, DataGrip, Rider, PyCharm Pro, Aqua, and Ultimate.
### [intellij.moduleSets.commercial.xml](intellij.moduleSets.commercial.xml)
A set of commercial platform modules required by all JetBrains commercial IDEs.
Includes the commercial platform module and licensing functionality.
This set is included in the [ide.ultimate](#intellijmodulesetsideultimatexml) set.
### [intellij.moduleSets.ide.trial.xml](intellij.moduleSets.ide.trial.xml)
A set of trial and monetization modules for commercial IDEs without a free tier.
Includes trial promotion and trace consent modules.
This set is used by WebStorm, GoLand, RustRover, RubyMine, PhpStorm, CLion, and DataGrip.
## VCS Module Sets
### [intellij.moduleSets.vcs.xml](intellij.moduleSets.vcs.xml)
A set of product modules for regular IDE with VCS support.
This is a separate set because, for instance, `intellij.platform.smRunner.vcs` should not be included in Rider,
yet we still want to avoid duplicating the list of VCS modules.
This set is included in the [ide.common](#intellijmodulesetsidecommonxml) set.
### [intellij.moduleSets.vcs.shared.xml](intellij.moduleSets.vcs.shared.xml)
A set of VCS modules shared between different product variants.
### [intellij.moduleSets.vcs.frontend.xml](intellij.moduleSets.vcs.frontend.xml)
A set of VCS modules specific to frontend/client implementations.
## Library Module Sets
### [intellij.moduleSets.libraries.xml](intellij.moduleSets.libraries.xml)
A set that aggregates all library module sets. This is the main entry point for including all platform libraries.
Includes [libraries.core](#intellijmodulesetslibrariescorexml), [libraries.ktor](#intellijmodulesetslibrariesktrxml), [libraries.misc](#intellijmodulesetslibrariesmiscxml), and [libraries.temporaryBundled](#intellijmodulesetslibrariestemporarybundledxml).
### [intellij.moduleSets.libraries.core.xml](intellij.moduleSets.libraries.core.xml)
A set of library modules that are embedded into Core and bundled to all IDEs based on IJ Platform.
All library modules in this file must have `loading="embedded"`.
### [intellij.moduleSets.libraries.misc.xml](intellij.moduleSets.libraries.misc.xml)
A set of library modules that must NOT be embedded into Core.
Plugins that require these libraries should bundle them individually.
All libs here must not be embedded. If a library should be embedded, it should be moved to [libraries.core](#intellijmodulesetslibrariescorexml).
### [intellij.moduleSets.libraries.ktor.xml](intellij.moduleSets.libraries.ktor.xml)
A set of Ktor networking library modules that are embedded with the platform.
Includes ktor-io, ktor-utils, ktor-network-tls, ktor-client, and related modules.
### [intellij.moduleSets.libraries.temporaryBundled.xml](intellij.moduleSets.libraries.temporaryBundled.xml)
A set of library modules that are temporarily bundled with the platform but should eventually be moved elsewhere.
## Other Module Sets
### [intellij.moduleSets.xml.xml](intellij.moduleSets.xml.xml)
A set of modules providing XML language support and related functionality.
### [intellij.moduleSets.rd.common.xml](intellij.moduleSets.rd.common.xml)
A set of common Remote Development modules.
### [intellij.moduleSets.grid.core.xml](intellij.moduleSets.grid.core.xml)
A set of core grid-related modules.
### [intellij.moduleSets.elevation.xml](intellij.moduleSets.elevation.xml)
A set of modules related to privilege elevation functionality.
### [intellij.moduleSets.debugger.streams.xml](intellij.moduleSets.debugger.streams.xml)
A set of debugger stream tracing modules for Ultimate Edition IDEs.
Includes core stream tracing functionality, shared utilities, and backend integration.
This set is used by Rider, Aqua, and Ultimate.
### [intellij.moduleSets.ssh.xml](intellij.moduleSets.ssh.xml)
A set of SSH-related modules for remote development and deployment features.
Currently includes SSH UI components (`intellij.platform.ssh.ui`).
This set is used by all commercial IDEs, PyCharm Pro, Gateway, JetBrains Client, and Ultimate editions.
@@ -0,0 +1,209 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.productLayout
import kotlinx.serialization.json.Json
import org.jetbrains.intellij.build.BuildPaths
import org.jetbrains.intellij.build.ProductProperties
import org.jetbrains.intellij.build.dev.createProductProperties
import org.jetbrains.intellij.build.impl.jpsModuleOutputProvider
import org.jetbrains.jps.model.serialization.JpsMavenSettings
import org.jetbrains.jps.model.serialization.JpsSerializationManager
import java.nio.file.Files
import java.nio.file.Path
/**
* Represents a discovered product with all its metadata.
* Used by both XML and JSON generators to avoid code duplication.
* Test products have properties = null (they don't have ProductProperties classes).
*/
internal data class DiscoveredProduct(
val name: String,
val config: ProductConfiguration,
val properties: ProductProperties?,
val spec: ProductModulesContentSpec?,
val pluginXmlPath: String?
)
/**
* Converts a DiscoveredProduct to ProductSpec for JSON output.
* Extension function that has access to ProductProperties types.
* Includes full ProductModulesContentSpec for complete DSL representation.
*/
internal fun DiscoveredProduct.toProductSpec(projectRoot: Path): ProductSpec {
// For test products (properties = null), use "test-product" as source file
val sourceFile = if (properties != null) {
getProductPropertiesSourceFile(properties.javaClass, projectRoot)
} else {
"test-product"
}
return ProductSpec(
name = name,
className = config.className,
sourceFile = sourceFile,
pluginXmlPath = pluginXmlPath,
contentSpec = spec, // Pass full ProductModulesContentSpec for complete DSL serialization
buildModules = config.modules
)
}
/**
* Discovers all products from dev-build.json registry (internal representation).
* Returns DiscoveredProduct instances for internal use by XML generator.
*
* @param projectRoot The project root path
* @return List of discovered products with all metadata
*/
private suspend fun discoverAllProductsInternal(projectRoot: Path): List<DiscoveredProduct> {
val jsonContent = Files.readString(projectRoot.resolve(PRODUCT_REGISTRY_PATH))
val productToConfiguration = Json.decodeFromString<ProductConfigurationRegistry>(jsonContent).products
val project = JpsSerializationManager.getInstance().loadProject(
projectRoot.toString(),
mapOf("MAVEN_REPOSITORY" to JpsMavenSettings.getMavenRepositoryPath()),
false
)
val moduleOutputProvider = jpsModuleOutputProvider(project.modules)
val products = mutableListOf<DiscoveredProduct>()
for ((productName, productConfig) in productToConfiguration) {
val productProperties = createProductProperties(
productConfiguration = productConfig,
moduleOutputProvider = moduleOutputProvider,
projectDir = projectRoot,
platformPrefix = null
)
val spec = productProperties.getProductContentDescriptor()
products.add(
DiscoveredProduct(
name = productName,
config = productConfig,
properties = productProperties,
spec = spec,
pluginXmlPath = productConfig.pluginXmlPath
)
)
}
return products
}
/**
* Discovers test products from known locations.
* These are products used for testing that shouldn't be in dev-build.json.
* Retrieves contentSpec from module set registries (UltimateModuleSets).
*
* @param projectRoot The project root path
* @return List of test products as DiscoveredProduct with contentSpec
*/
private fun discoverTestProducts(projectRoot: Path): List<DiscoveredProduct> {
// Build a registry of test product name -> (xmlPath, contentSpec)
val testProductRegistry = mutableMapOf<String, Pair<String, ProductModulesContentSpec>>()
// Get ultimate test product specs (only if ultimate directory exists)
if (Files.exists(projectRoot.resolve("ultimate"))) {
try {
// Load UltimateModuleSets via reflection to avoid hard dependency
val ultimateModuleSetsClass = Class.forName("com.intellij.platform.commercial.buildScripts.productLayout.UltimateModuleSets")
val instanceField = ultimateModuleSetsClass.getDeclaredField("INSTANCE")
val ultimateModuleSetsInstance = instanceField.get(null)
val getTestProductSpecsMethod = ultimateModuleSetsClass.getDeclaredMethod("getTestProductSpecs")
@Suppress("UNCHECKED_CAST")
val ultimateSpecs = getTestProductSpecsMethod.invoke(ultimateModuleSetsInstance) as List<Pair<String, ProductModulesContentSpec>>
for ((name, spec) in ultimateSpecs) {
val xmlPath = "ultimate/platform-ultimate/testResources/META-INF/${name}Plugin.xml"
testProductRegistry[name] = xmlPath to spec
}
} catch (e: ClassNotFoundException) {
// Ultimate module not available, skip ultimate test products
}
}
// Build DiscoveredProduct instances for test products that exist on disk
return testProductRegistry.mapNotNull { (name, pair) ->
val (xmlPath, spec) = pair
val xmlFile = projectRoot.resolve(xmlPath)
if (!Files.exists(xmlFile)) return@mapNotNull null
DiscoveredProduct(
name = name,
config = ProductConfiguration(
className = "test-product", // test products don't have Properties class
modules = emptyList(),
pluginXmlPath = xmlPath
),
properties = null, // test products don't have ProductProperties
spec = spec,
pluginXmlPath = xmlPath
)
}
}
/**
* Discovers all products from dev-build.json registry and converts to ProductSpec for JSON output.
* Public function that can be called from other modules (like ultimate buildScripts).
*
* @param projectRoot The project root path
* @return List of products as ProductSpec (simple data class with no internal dependencies)
*/
suspend fun discoverAllProductsForJson(projectRoot: Path): List<ProductSpec> {
val regularProducts = discoverAllProductsInternal(projectRoot)
val testProducts = discoverTestProducts(projectRoot)
val allProducts = regularProducts + testProducts
return allProducts.map { it.toProductSpec(projectRoot) }
}
/**
* Discovers all products for validation purposes.
* Returns list of (productName, ProductModulesContentSpec) pairs.
*
* @param projectRoot The project root path
* @return List of product name and spec pairs for validation
*/
suspend fun discoverAllProductsForValidation(projectRoot: Path): List<Pair<String, ProductModulesContentSpec?>> {
val discovered = discoverAllProductsInternal(projectRoot)
return discovered.map { it.name to it.spec }
}
/**
* Generates product XMLs for all products using programmatic content.
* Discovers products from dev-build.json and test products, then generates complete plugin.xml files.
*
* @param projectRoot The project root path
* @return Result containing generation statistics
*/
suspend fun generateAllProductXmlFiles(projectRoot: Path): ProductGenerationResult {
val regularProducts = discoverAllProductsInternal(projectRoot)
val testProducts = discoverTestProducts(projectRoot)
val products = regularProducts + testProducts
// Detect if this is an Ultimate build (projectRoot != communityRoot)
val isUltimateBuild = projectRoot != BuildPaths.COMMUNITY_ROOT.communityRoot
val productResults = products.mapNotNull { discovered ->
// Skip products without pluginXmlPath or spec configured
val pluginXmlRelativePath = discovered.pluginXmlPath ?: return@mapNotNull null
val spec = discovered.spec ?: return@mapNotNull null
val pluginXmlPath = projectRoot.resolve(pluginXmlRelativePath)
val moduleOutputProvider = jpsModuleOutputProvider(
JpsSerializationManager.getInstance()
.loadProject(projectRoot.toString(), mapOf("MAVEN_REPOSITORY" to JpsMavenSettings.getMavenRepositoryPath()), false)
.modules
)
generateProductXml(
pluginXmlPath = pluginXmlPath,
spec = spec,
productName = discovered.name,
moduleOutputProvider = moduleOutputProvider,
productPropertiesClass = discovered.properties?.javaClass?.name ?: "test-product",
projectRoot = projectRoot,
isUltimateBuild = isUltimateBuild,
)
}
return ProductGenerationResult(productResults)
}
@@ -74,8 +74,8 @@ open class PyCharmCommunityProperties(protected val communityHome: Path) : PyCha
deprecatedInclude("intellij.pycharm.community", "META-INF/pycharm-core-customization.xml")
}
override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) {
super.copyAdditionalFiles(context, targetDir)
override suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) {
super.copyAdditionalFiles(targetDir, context)
val licenseTargetDir = targetDir.resolve("license")
copyFileToDir(context.paths.communityHomeDir.resolve("LICENSE.txt"), licenseTargetDir)
@@ -98,8 +98,8 @@ open class PyCharmCommunityProperties(protected val communityHome: Path) : PyCha
override fun getFullNameIncludingEdition(appInfo: ApplicationInfoProperties) = "PyCharm Community Edition"
override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path, arch: JvmArchitecture) {
super.copyAdditionalFiles(context, targetDir, arch)
override suspend fun copyAdditionalFiles(targetDir: Path, arch: JvmArchitecture, context: BuildContext) {
super.copyAdditionalFiles(targetDir, arch, context)
PyCharmBuildUtils.copySkeletons(context, targetDir, "skeletons-win*.zip")
}
@@ -32,7 +32,7 @@ abstract class PyCharmPropertiesBase(enlargeWelcomeScreen: Boolean) : JetBrainsP
))
}
override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) {
override suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) {
zipSourcesOfModules(
modules = listOf("intellij.python.community", "intellij.python.psi"),
targetFile = Path.of("$targetDir/lib/src/pycharm-openapi-src.zip"),
@@ -13,11 +13,11 @@
<xi:include href="/META-INF/pycharm-core-customization.xml"/>
<xi:include href="/META-INF/intellij.moduleSets.ide.common.xml"/>
<xi:include href="/META-INF/intellij.moduleSets.rd.common.xml"/>
<!-- <editor-fold desc="additional"> -->
<content namespace="jetbrains">
<!-- <editor-fold desc="additional"> -->
<module name="intellij.platform.ide.newUiOnboarding"/>
<module name="intellij.ide.startup.importSettings"/>
<module name="intellij.platform.tips"/>
<!-- </editor-fold> -->
</content>
<!-- </editor-fold> -->
</idea-plugin>