diff --git a/.idea/modules.xml b/.idea/modules.xml index 6a686fddb581..6dc3829ef071 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -989,10 +989,12 @@ + + diff --git a/build/bazel-generated-file-list.txt b/build/bazel-generated-file-list.txt index 75e9783e1eef..a89804546598 100644 --- a/build/bazel-generated-file-list.txt +++ b/build/bazel-generated-file-list.txt @@ -486,10 +486,12 @@ platform/bootstrap platform/bootstrap/coroutine platform/bootstrap/dev platform/build-scripts +platform/build-scripts/api platform/build-scripts/codeOptimizer platform/build-scripts/dev-server platform/build-scripts/downloader platform/build-scripts/icons +platform/build-scripts/product-dsl platform/build-scripts/testFramework platform/build-scripts/tests platform/build-scripts/usages diff --git a/build/src/org/jetbrains/intellij/build/IdeaCommunityProperties.kt b/build/src/org/jetbrains/intellij/build/IdeaCommunityProperties.kt index fa300ffd3196..0a7c9cae9205 100644 --- a/build/src/org/jetbrains/intellij/build/IdeaCommunityProperties.kt +++ b/build/src/org/jetbrains/intellij/build/IdeaCommunityProperties.kt @@ -40,12 +40,6 @@ internal suspend fun createCommunityBuildContext( ) open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrainsProductProperties() { - override val moduleSetsProviders: List - get() = listOf(CommunityModuleSets) - - override val baseFileName: String - get() = "idea" - init { configurePropertiesForAllEditionsOfIntelliJIdea(this) platformPrefix = "Idea" @@ -65,11 +59,11 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai productLayout.prepareCustomPluginRepositoryForPublishedPlugins = false productLayout.buildAllCompatiblePlugins = true - productLayout.pluginLayouts = CommunityRepositoryModules.COMMUNITY_REPOSITORY_PLUGINS.addAll(listOf( + productLayout.pluginLayouts = CommunityRepositoryModules.COMMUNITY_REPOSITORY_PLUGINS + persistentListOf( JavaPluginLayout.javaPlugin(), CommunityRepositoryModules.androidPlugin(allPlatforms = true), CommunityRepositoryModules.groovyPlugin(), - )) + ) productLayout.addPlatformSpec { layout, _ -> layout.withModule("intellij.platform.structuralSearch") @@ -78,12 +72,12 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai productLayout.skipUnresolvedContentModules = true mavenArtifacts.forIdeModules = true - mavenArtifacts.additionalModules = mavenArtifacts.additionalModules.addAll(MAVEN_ARTIFACTS_ADDITIONAL_MODULES) - mavenArtifacts.squashedModules = mavenArtifacts.squashedModules.addAll(persistentListOf( + mavenArtifacts.additionalModules += MAVEN_ARTIFACTS_ADDITIONAL_MODULES + mavenArtifacts.squashedModules += persistentListOf( "intellij.platform.util.base", "intellij.platform.util.base.multiplatform", "intellij.platform.util.zip", - )) + ) mavenArtifacts.validateForMavenCentralPublication = { module -> JewelMavenArtifacts.isPublishedJewelModule(module) } @@ -120,47 +114,19 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai additionalVmOptions = persistentListOf("-Dllm.show.ai.promotion.window.on.start=false") } - override fun getProductContentDescriptor(): ProductModulesContentSpec { - return getIntelliJCommunityProductContentDescriptor(includeCommunityExtensions = true) + override val moduleSetsProviders: List + get() = listOf(CommunityModuleSets) + + override val baseFileName: String + get() = "idea" + + override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules { + include(intellijCommunityBaseFragment()) + include(communityExtensionsFragment()) } - protected fun getIntelliJCommunityProductContentDescriptor( - includeCommunityExtensions: Boolean, - ): ProductModulesContentSpec = productModules { - alias("com.intellij.modules.idea") - alias("com.intellij.modules.idea.community") - alias("com.intellij.modules.java-capable") - alias("com.intellij.modules.python-core-capable") - alias("com.intellij.modules.python-in-non-pycharm-ide-capable") - alias("com.intellij.platform.ide.provisioner") - - deprecatedInclude("intellij.java.ide.resources", "META-INF/JavaIdePlugin.xml") - deprecatedInclude("intellij.idea.community.customization", "META-INF/tips-intellij-idea-community.xml") - - moduleSet(CommunityModuleSets.debuggerStreams()) - - module("intellij.platform.coverage") - module("intellij.platform.coverage.agent") - module("intellij.xml.xmlbeans") - module("intellij.platform.ide.newUiOnboarding") - module("intellij.platform.ide.newUsersOnboarding") - module("intellij.ide.startup.importSettings") - module("intellij.platform.customization.min") - module("intellij.idea.customization.base") - module("intellij.idea.customization.backend") - module("intellij.platform.tips") - - moduleSet(CommunityModuleSets.ideCommon()) - moduleSet(CommunityModuleSets.rdCommon()) - - if (includeCommunityExtensions) { - deprecatedInclude("intellij.platform.extended.community.impl", "META-INF/community-extensions.xml", ultimateOnly = true) - } - deprecatedInclude("intellij.idea.community.customization", "META-INF/community-customization.xml") - } - - override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) { - super.copyAdditionalFiles(context, targetDir) + override suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) { + super.copyAdditionalFiles(targetDir, context) copyFileToDir(context.paths.communityHomeDir.resolve("LICENSE.txt"), targetDir) copyFileToDir(context.paths.communityHomeDir.resolve("NOTICE.txt"), targetDir) @@ -211,12 +177,12 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai override fun getRootDirectoryName(appInfo: ApplicationInfoProperties, buildNumber: String): String = "idea-IC-$buildNumber" override fun generateExecutableFilesPatterns( - context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture, targetLibcImpl: LibcImpl, + context: BuildContext, ): Sequence { - return super.generateExecutableFilesPatterns(context, includeRuntime, arch, targetLibcImpl) + return super.generateExecutableFilesPatterns(includeRuntime, arch, targetLibcImpl, context) .plus(KotlinBinaries.kotlinCompilerExecutables) .filterNot { it == "plugins/**/*.sh" } } @@ -242,8 +208,8 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai } } - override fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture): Sequence { - return super.generateExecutableFilesPatterns(context, includeRuntime, arch) + override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, context: BuildContext): Sequence { + return super.generateExecutableFilesPatterns(includeRuntime, arch, context) .plus(KotlinBinaries.kotlinCompilerExecutables) .filterNot { it == "plugins/**/*.sh" } } @@ -257,3 +223,45 @@ open class IdeaCommunityProperties(private val communityHomeDir: Path) : JetBrai override fun getOutputDirectoryName(appInfo: ApplicationInfoProperties): String = "idea-ce" } + +/** + * Base IntelliJ Community content fragment. + * This fragment is composable - subclasses can include this and optionally add community extensions. + */ +fun intellijCommunityBaseFragment(): ProductModulesContentSpec = productModules { + alias("com.intellij.modules.idea") + alias("com.intellij.modules.idea.community") + alias("com.intellij.modules.java-capable") + alias("com.intellij.modules.python-core-capable") + alias("com.intellij.modules.python-in-non-pycharm-ide-capable") + alias("com.intellij.platform.ide.provisioner") + + deprecatedInclude("intellij.java.ide.resources", "META-INF/JavaIdePlugin.xml") + deprecatedInclude("intellij.idea.community.customization", "META-INF/tips-intellij-idea-community.xml") + + moduleSet(CommunityModuleSets.debuggerStreams()) + + module("intellij.platform.coverage") + module("intellij.platform.coverage.agent") + module("intellij.xml.xmlbeans") + module("intellij.platform.ide.newUiOnboarding") + module("intellij.platform.ide.newUsersOnboarding") + module("intellij.ide.startup.importSettings") + module("intellij.platform.customization.min") + module("intellij.idea.customization.base") + module("intellij.idea.customization.backend") + module("intellij.platform.tips") + + moduleSet(CommunityModuleSets.ideCommon()) + moduleSet(CommunityModuleSets.rdCommon()) + + deprecatedInclude("intellij.idea.community.customization", "META-INF/community-customization.xml") +} + +/** + * Community extensions fragment for Ultimate builds. + * This fragment is composable - subclasses can choose to include or exclude it. + */ +fun communityExtensionsFragment(): ProductModulesContentSpec = productModules { + deprecatedInclude("intellij.platform.extended.community.impl", "META-INF/community-extensions.xml", ultimateOnly = true) +} \ No newline at end of file diff --git a/community-resources/resources/META-INF/IdeaPlugin.xml b/community-resources/resources/META-INF/IdeaPlugin.xml index 8dcc33330e24..7c30501c24ad 100644 --- a/community-resources/resources/META-INF/IdeaPlugin.xml +++ b/community-resources/resources/META-INF/IdeaPlugin.xml @@ -11,15 +11,15 @@ + - + - @@ -30,6 +30,6 @@ - + diff --git a/platform/build-scripts/BUILD.bazel b/platform/build-scripts/BUILD.bazel index 641b69dcf636..c48885f64843 100644 --- a/platform/build-scripts/BUILD.bazel +++ b/platform/build-scripts/BUILD.bazel @@ -88,6 +88,12 @@ jvm_library( "@lib//:platform-build_scripts-jetbrains-intellij-deps-coverage-reporter", "//libraries/ktor/client", "//platform/plugins/parser/impl", + "//platform/build-scripts/api", + "//platform/build-scripts/product-dsl", + ], + exports = [ + "//platform/build-scripts/api", + "//platform/build-scripts/product-dsl", ], runtime_deps = [ "//libraries/commons/cli", diff --git a/platform/build-scripts/api/BUILD.bazel b/platform/build-scripts/api/BUILD.bazel new file mode 100644 index 000000000000..e69e7c5f9296 --- /dev/null +++ b/platform/build-scripts/api/BUILD.bazel @@ -0,0 +1,21 @@ +### auto-generated section `build intellij.platform.buildScripts.api` start +load("@rules_jvm//:jvm.bzl", "jvm_library") + +jvm_library( + name = "api", + module_name = "intellij.platform.buildScripts.api", + visibility = ["//visibility:public"], + srcs = glob(["src/**/*.kt", "src/**/*.java", "src/**/*.form"], allow_empty = True), + deps = [ + "@lib//:kotlin-stdlib", + "//plugins/groovy/rt/classLoader", + "//platform/util-class-loader:util-classLoader", + "//platform/util", + "@lib//:jetbrains-annotations", + "//platform/build-scripts/downloader", + "//jps/model-api:model", + "//platform/util/zip", + "@community//build:zip", + ] +) +### auto-generated section `build intellij.platform.buildScripts.api` end \ No newline at end of file diff --git a/platform/build-scripts/api/intellij.platform.buildScripts.api.iml b/platform/build-scripts/api/intellij.platform.buildScripts.api.iml new file mode 100644 index 000000000000..a8cd9306d3e4 --- /dev/null +++ b/platform/build-scripts/api/intellij.platform.buildScripts.api.iml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/BuildPaths.kt b/platform/build-scripts/api/src/BuildPaths.kt similarity index 99% rename from platform/build-scripts/src/org/jetbrains/intellij/build/BuildPaths.kt rename to platform/build-scripts/api/src/BuildPaths.kt index fe6450c1a4c3..bfee0992fcd0 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/BuildPaths.kt +++ b/platform/build-scripts/api/src/BuildPaths.kt @@ -72,4 +72,4 @@ class BuildPaths( */ @Deprecated("Use [artifactDir] or [tempDir] instead") val jpsArtifacts: Path = buildOutputDir.resolve("jps-artifacts") -} +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/IdeaProjectLoaderUtil.kt b/platform/build-scripts/api/src/IdeaProjectLoaderUtil.kt similarity index 95% rename from platform/build-scripts/src/org/jetbrains/intellij/build/IdeaProjectLoaderUtil.kt rename to platform/build-scripts/api/src/IdeaProjectLoaderUtil.kt index 5d988defd018..dcd5af382176 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/IdeaProjectLoaderUtil.kt +++ b/platform/build-scripts/api/src/IdeaProjectLoaderUtil.kt @@ -1,9 +1,8 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build import com.intellij.openapi.application.PathManager import com.intellij.util.lang.UrlClassLoader -import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.ApiStatus.Internal import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot import java.nio.file.Path import java.nio.file.Paths @@ -60,7 +59,7 @@ internal object IdeaProjectLoaderUtil { /** * This method only for internal usage. Use [BuildPaths.ULTIMATE_HOME] instead. */ - @ApiStatus.Internal + @Internal fun guessUltimateHome(): Path { return searchForAnyMarkerFile(listOf(ULTIMATE_REPO_MARKER_FILE)) } @@ -68,7 +67,7 @@ internal object IdeaProjectLoaderUtil { /** * This method only for internal usage. Use [BuildPaths.COMMUNITY_ROOT] instead. */ - @ApiStatus.Internal + @Internal fun guessCommunityHome(): BuildDependenciesCommunityRoot { val directMarker = COMMUNITY_REPO_MARKER_FILE val inSubdirMarker = Path.of("community").resolve(COMMUNITY_REPO_MARKER_FILE) @@ -85,7 +84,7 @@ internal object IdeaProjectLoaderUtil { /** * This method only for internal usage. Use [BuildPaths.MAYBE_ULTIMATE_HOME] instead. */ - @ApiStatus.Internal + @Internal fun maybeUltimateHome(): Path? { return searchForOptionalMarkerFile(listOf(ULTIMATE_REPO_MARKER_FILE)) } @@ -134,4 +133,4 @@ internal object IdeaProjectLoaderUtil { checkNotNull(classFileURL) { "Could not get .class file location from class " + klass.getName() } return UrlClassLoader.urlToFilePath(classFileURL.path) } -} +} \ No newline at end of file diff --git a/platform/build-scripts/api/src/ModuleOutputProvider.kt b/platform/build-scripts/api/src/ModuleOutputProvider.kt new file mode 100644 index 000000000000..8fa4026efe24 --- /dev/null +++ b/platform/build-scripts/api/src/ModuleOutputProvider.kt @@ -0,0 +1,15 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build + +import org.jetbrains.jps.model.module.JpsModule +import java.nio.file.Path + +interface ModuleOutputProvider { + fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean = false): ByteArray? + + fun findModule(name: String): JpsModule? + + fun findRequiredModule(name: String): JpsModule + + fun getModuleOutputRoots(module: JpsModule, forTests: Boolean = false): List +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/moduleContentUtil.kt b/platform/build-scripts/api/src/moduleContentUtil.kt similarity index 87% rename from platform/build-scripts/src/org/jetbrains/intellij/build/moduleContentUtil.kt rename to platform/build-scripts/api/src/moduleContentUtil.kt index 559c71f70a29..810845a2c72a 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/moduleContentUtil.kt +++ b/platform/build-scripts/api/src/moduleContentUtil.kt @@ -3,8 +3,6 @@ package org.jetbrains.intellij.build import com.intellij.util.lang.ImmutableZipFile import org.jetbrains.annotations.ApiStatus.Internal -import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH -import org.jetbrains.intellij.build.impl.ModuleOutputProvider import org.jetbrains.intellij.build.io.ZipEntryProcessorResult import org.jetbrains.intellij.build.io.readZipFile import org.jetbrains.jps.model.java.JavaResourceRootType @@ -19,6 +17,9 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.attribute.BasicFileAttributes +const val PLUGIN_XML_RELATIVE_PATH: String = "META-INF/plugin.xml" +val useTestSourceEnabled: Boolean = System.getProperty("idea.build.pack.test.source.enabled", "true").toBoolean() + fun getUnprocessedPluginXmlContent(module: JpsModule, context: ModuleOutputProvider): ByteArray { return requireNotNull(findUnprocessedDescriptorContent(module = module, path = PLUGIN_XML_RELATIVE_PATH, context = context)) { "META-INF/plugin.xml not found in ${module.name} module output" @@ -40,7 +41,7 @@ fun findUnprocessedDescriptorContent(module: JpsModule, path: String, context: M private val rootTypeOrder = arrayOf(JavaResourceRootType.RESOURCE, JavaSourceRootType.SOURCE, JavaResourceRootType.TEST_RESOURCE, JavaSourceRootType.TEST_SOURCE) -internal fun findFileInModuleSources(module: JpsModule, relativePath: String, onlyProductionSources: Boolean = false): Path? { +fun findFileInModuleSources(module: JpsModule, relativePath: String, onlyProductionSources: Boolean = false): Path? { for (type in rootTypeOrder) { for (root in module.sourceRoots) { if (type != root.rootType || (onlyProductionSources && !(root.rootType == JavaResourceRootType.RESOURCE || root.rootType == JavaSourceRootType.SOURCE))) { @@ -55,9 +56,9 @@ internal fun findFileInModuleSources(module: JpsModule, relativePath: String, on return null } -internal fun isModuleNameLikeFilename(relativePath: String): Boolean = relativePath.startsWith("intellij.") || relativePath.startsWith("fleet.") +fun isModuleNameLikeFilename(relativePath: String): Boolean = relativePath.startsWith("intellij.") || relativePath.startsWith("fleet.") -internal fun findFileInModuleLibraryDependencies(module: JpsModule, relativePath: String): ByteArray? { +fun findFileInModuleLibraryDependencies(module: JpsModule, relativePath: String): ByteArray? { for (dependency in module.dependenciesList.dependencies) { if (dependency is JpsLibraryDependency) { val library = dependency.library ?: continue @@ -71,14 +72,14 @@ internal fun findFileInModuleLibraryDependencies(module: JpsModule, relativePath return null } -internal fun findProductModulesFile(clientMainModuleName: String, context: CompilationContext): Path? { +fun findProductModulesFile(clientMainModuleName: String, context: ModuleOutputProvider): Path? { return findFileInModuleSources(context.findRequiredModule(clientMainModuleName), "META-INF/$clientMainModuleName/product-modules.xml") } -internal fun findFileInModuleDependencies( +fun findFileInModuleDependencies( module: JpsModule, relativePath: String, - context: CompilationContext, + context: ModuleOutputProvider, processedModules: MutableSet, recursiveModuleExclude: String? = null, ): ByteArray? { @@ -98,7 +99,7 @@ internal fun findFileInModuleDependencies( private fun findFileInModuleDependenciesRecursive( module: JpsModule, relativePath: String, - context: CompilationContext, + context: ModuleOutputProvider, processedModules: MutableSet, recursiveModuleExclude: String?, ): ByteArray? { @@ -134,7 +135,7 @@ private fun findFileInModuleDependenciesRecursive( } @Internal -fun hasModuleOutputPath(module: JpsModule, relativePath: String, context: CompilationContext): Boolean { +fun hasModuleOutputPath(module: JpsModule, relativePath: String, context: ModuleOutputProvider): Boolean { return context.getModuleOutputRoots(module).any { output -> val attributes = try { Files.readAttributes(output, BasicFileAttributes::class.java) diff --git a/platform/build-scripts/intellij.platform.buildScripts.iml b/platform/build-scripts/intellij.platform.buildScripts.iml index cf05b5bc6997..5a952ab749dc 100644 --- a/platform/build-scripts/intellij.platform.buildScripts.iml +++ b/platform/build-scripts/intellij.platform.buildScripts.iml @@ -420,5 +420,7 @@ + + \ No newline at end of file diff --git a/platform/build-scripts/product-dsl/BUILD.bazel b/platform/build-scripts/product-dsl/BUILD.bazel new file mode 100644 index 000000000000..58ed9f5c6c15 --- /dev/null +++ b/platform/build-scripts/product-dsl/BUILD.bazel @@ -0,0 +1,65 @@ +### auto-generated section `build intellij.platform.buildScripts.productDsl` start +load("//build:compiler-options.bzl", "create_kotlinc_options") +load("@rules_jvm//:jvm.bzl", "jvm_library") + +create_kotlinc_options( + name = "custom_product-dsl", + opt_in = [ + "kotlin.RequiresOptIn", + "kotlinx.serialization.ExperimentalSerializationApi", + "kotlinx.coroutines.ExperimentalCoroutinesApi", + ] +) + +jvm_library( + name = "product-dsl", + module_name = "intellij.platform.buildScripts.productDsl", + visibility = ["//visibility:public"], + srcs = glob(["src/**/*.kt", "src/**/*.java", "src/**/*.form"], allow_empty = True), + kotlinc_opts = ":custom_product-dsl", + deps = [ + "@lib//:kotlin-stdlib", + "//platform/plugins/parser/impl", + "//platform/build-scripts/api", + "//libraries/kotlinx/serialization/core", + "//libraries/kotlinx/serialization/json", + "//jps/model-api:model", + "//jps/model-serialization", + "//platform/util", + "//platform/util/jdom", + "//libraries/jackson/jackson", + ] +) + +jvm_library( + name = "product-dsl_test_lib", + visibility = ["//visibility:public"], + srcs = glob(["testSrc/**/*.kt", "testSrc/**/*.java", "testSrc/**/*.form"], allow_empty = True), + kotlinc_opts = ":custom_product-dsl", + associates = [":product-dsl"], + deps = [ + "@lib//:kotlin-stdlib", + "//platform/plugins/parser/impl", + "//platform/plugins/parser/impl:impl_test_lib", + "//platform/build-scripts/api", + "//libraries/kotlinx/serialization/core", + "//libraries/kotlinx/serialization/json", + "//jps/model-api:model", + "//jps/model-serialization", + "//platform/util", + "//platform/util/jdom", + "@lib//:junit5", + "@lib//:assert_j", + "//libraries/jackson/jackson", + ] +) +### auto-generated section `build intellij.platform.buildScripts.productDsl` end + +### auto-generated section `test intellij.platform.buildScripts.productDsl` start +load("@community//build:tests-options.bzl", "jps_test") + +jps_test( + name = "product-dsl_test", + runtime_deps = [":product-dsl_test_lib"] +) +### auto-generated section `test intellij.platform.buildScripts.productDsl` end \ No newline at end of file diff --git a/platform/build-scripts/product-dsl/intellij.platform.buildScripts.productDsl.iml b/platform/build-scripts/product-dsl/intellij.platform.buildScripts.productDsl.iml new file mode 100644 index 000000000000..1ade7fd58b9b --- /dev/null +++ b/platform/build-scripts/product-dsl/intellij.platform.buildScripts.productDsl.iml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + $KOTLIN_BUNDLED$/lib/kotlinx-serialization-compiler-plugin.jar + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/platform/build-scripts/product-dsl/module-sets.md b/platform/build-scripts/product-dsl/module-sets.md new file mode 100644 index 000000000000..e29c94c9033e --- /dev/null +++ b/platform/build-scripts/product-dsl/module-sets.md @@ -0,0 +1,56 @@ +# Module Sets + +Module sets are collections of modules that can be referenced as a single entity in product configurations. + +## Overview + +Module sets are now **defined in Kotlin code** in the following locations: +- **Community module sets**: `community/platform/build-scripts/product-dsl/src/CommunityModuleSets.kt` +- **Ultimate module sets**: `platform/buildScripts/src/productLayout/UltimateModuleSets.kt` + +Each module set is defined as a Kotlin function returning a `ModuleSet` object. The XML files (following the pattern `intellij.moduleSets...xml`) are **auto-generated** from this Kotlin code. + +### Generating XML Files + +To regenerate XML files from Kotlin code, run: +```bash +# For all products (community + ultimate) +UltimateModuleSets.main() + +# For community products only +CommunityModuleSets.main() +``` + +Or use the IDE's "Generate Product Layouts" run configuration. + +### Querying Module Sets + +To query module set structure, relationships, and usage programmatically, use the **JSON analysis endpoint**: +```bash +UltimateModuleSets.main(args = ["--json"]) +``` + +See the [Programmatic Content](programmatic-content.md#json-analysis-endpoint) documentation for details. + +## Creating a New Module Set + +See `/create-module-set` slash command for detailed instructions on creating a new module set. + +## Available Module Sets + +Module sets are defined in Kotlin code: + +- **Community module sets**: See `community/platform/build-scripts/product-dsl/src/CommunityModuleSets.kt` +- **Ultimate module sets**: See `platform/buildScripts/src/productLayout/UltimateModuleSets.kt` + +Each module set is defined as a function (e.g., `fun essential(): ModuleSet`) within these classes. + +### Querying Module Sets + +To discover available module sets and their composition: + +1. **Browse the source code** - Open the Kotlin files listed above to see all available module sets +2. **Use the JSON endpoint** - Run `UltimateModuleSets.main(args = ["--json"])` to get comprehensive analysis +3. **Check generated XML** - The generated XML files (e.g., `intellij.moduleSets.essential.xml`) contain expanded module lists + +For detailed module composition, includes/includedBy relationships, and product usage, use the [JSON analysis endpoint](#querying-module-sets). \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/programmatic-content.md b/platform/build-scripts/product-dsl/programmatic-content.md similarity index 71% rename from platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/programmatic-content.md rename to platform/build-scripts/product-dsl/programmatic-content.md index 4a682ff95b73..7727da4e8f2a 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/programmatic-content.md +++ b/platform/build-scripts/product-dsl/programmatic-content.md @@ -283,6 +283,173 @@ Use `ultimateOnly = true` when: - **`generateAllProductXmlFiles()`** (generator.kt): Batch generation for all registered products - **`collectAndValidateAliases()`** (generator.kt): Validates module aliases for duplicates +## JSON Analysis Endpoint + +The module set system provides a JSON analysis endpoint for programmatic querying and tooling integration. This endpoint is used by the Plugin Model Analyzer MCP server and other build tools. + +### Usage + +Run the module set main function with the `--json` flag: + +```bash +# Generate complete analysis for all products and module sets +UltimateModuleSets.main(args = ["--json"]) + +# Community products only +CommunityModuleSets.main(args = ["--json"]) +``` + +### Filtering Output + +Use the `--json` flag with a filter to get specific sections: + +```bash +# Get only products +--json='{"filter":"products"}' + +# Get only module sets +--json='{"filter":"moduleSets"}' + +# Include duplicate analysis +--json='{"includeDuplicates":true}' +``` + +### Output Structure + +The JSON output contains comprehensive analysis of the module system: + +#### 1. Module Distribution + +Maps each module to the module sets and products that include it: + +```json +{ + "moduleDistribution": { + "intellij.platform.vcs.impl": { + "inModuleSets": ["vcs", "ide.common"], + "inProducts": ["WebStorm", "GoLand", "CLion", "PyCharm", ...] + } + } +} +``` + +**Use case:** Find where a specific module is used across the codebase. + +#### 2. Module Set Hierarchy + +Shows the include relationships between module sets: + +```json +{ + "moduleSetHierarchy": { + "ide.common": { + "includes": ["essential", "vcs"], + "includedBy": ["ide.ultimate"], + "moduleCount": 145 + } + } +} +``` + +**Use case:** Understand module set dependencies and nesting structure. + +#### 3. Module Usage Index + +Comprehensive reverse lookup with source file paths: + +```json +{ + "moduleUsageIndex": { + "modules": { + "intellij.platform.vcs.impl": { + "moduleSets": [ + { + "name": "vcs", + "location": "community", + "sourceFile": "community/platform/build-scripts/product-dsl/src/CommunityModuleSets.kt" + } + ], + "products": [ + { + "name": "WebStorm", + "sourceFile": "platform/buildScripts/src/productLayout/UltimateModuleSets.kt" + } + ] + } + } + } +} +``` + +**Use case:** Trace module ownership and find where to make changes. + +#### 4. Product Composition Analysis + +Detailed breakdown of each product's composition: + +```json +{ + "productCompositionAnalysis": { + "CLion": { + "composition": { + "totalAliases": 3, + "totalModuleSets": 12, + "totalDirectModules": 45, + "totalModules": 523 + }, + "operations": [ + {"type": "alias", "value": "com.jetbrains.modules.cidr.lang"}, + {"type": "moduleSet", "value": "commercial"}, + {"type": "module", "value": "intellij.clion.core"} + ] + } + } +} +``` + +**Use case:** Analyze product composition and optimize module dependencies. + +#### 5. Duplicate Analysis (Optional) + +When `includeDuplicates: true` is set, detects duplicate xi:include elements: + +```json +{ + "duplicateAnalysis": { + "ReSharper Backend": { + "/META-INF/intellij.moduleSets.essential.xml": [ + { + "directInclude": true, + "deprecatedIncludeRefs": [ + "intellij.platform.resources -> /META-INF/PlatformLangPlugin.xml" + ] + } + ] + } + } +} +``` + +**Use case:** Identify redundant includes that can be removed. + +### Integration with MCP Server + +The Plugin Model Analyzer MCP server (`build/mcp-servers/module-analyzer`) uses this JSON endpoint to provide: + +- `analyze_module_structure` - Complete module system analysis +- `get_module_info` - Query specific module details +- `find_module_paths` - Trace module to product paths +- `get_module_set_hierarchy` - Query module set relationships +- `list_products` - List products filtered by criteria +- `validate_community_products` - Ensure community/ultimate separation + +### Implementation + +The JSON generation is implemented in: +- `ModuleSetRunner.kt` - Orchestration and CLI parsing +- `ModuleSetJsonExport.kt` - JSON generation logic +- `ModuleSetDiscovery.kt` - Module set discovery via reflection + ## Benefits 1. **Type safety**: Kotlin code with IDE support (autocomplete, refactoring) @@ -290,6 +457,7 @@ Use `ultimateOnly = true` when: 3. **Single source of truth**: One Kotlin definition for both dev and non-dev modes 4. **Maintainability**: Easier to see what modules a product includes 5. **VCS-friendly**: Static files work without dev mode infrastructure +6. **Programmatic access**: JSON endpoint enables tooling and automation ## See Also diff --git a/platform/build-scripts/product-dsl/src/ContentBlockBuilder.kt b/platform/build-scripts/product-dsl/src/ContentBlockBuilder.kt new file mode 100644 index 000000000000..2df37a088c9a --- /dev/null +++ b/platform/build-scripts/product-dsl/src/ContentBlockBuilder.kt @@ -0,0 +1,152 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.intellij.build.productLayout + +import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule + +/** + * Builds content blocks and module-to-set chain mapping in a single hierarchical traversal. + * This optimized version eliminates redundant tree walking by computing all results simultaneously. + * + * @param spec The product modules specification + * @param collectModuleSetAliases Whether to collect module set aliases during traversal (for inlineModuleSets mode) + * @return Triple of (content blocks, module-to-set chain mapping, module set alias to source mapping) + */ +internal fun buildContentBlocksAndChainMapping( + spec: ProductModulesContentSpec, + collectModuleSetAliases: Boolean = false +): Triple, Map>, Map> { + val contentBlocks = mutableListOf() + val moduleToChain = mutableMapOf>() + val moduleToSets = mutableMapOf>() + val aliasToSource = if (collectModuleSetAliases) mutableMapOf() else null + val processedSets = HashSet() + val contentBlockByName = mutableMapOf() + + fun traverse(moduleSet: ModuleSet, chain: List, overrides: Map) { + val setName = "$MODULE_SET_PREFIX${moduleSet.name}" + + // Check if already processed + val alreadyProcessed = !processedSets.add(setName) + if (alreadyProcessed) { + // If already processed but now we have overrides, update the existing content block + if (overrides.isNotEmpty()) { + val existingBlock = contentBlockByName[moduleSet.name] + if (existingBlock != null) { + // Check if existing block has any loading attributes + val hasExistingOverrides = existingBlock.modules.any { it.loading != null } + if (!hasExistingOverrides) { + // Reuse the filtered module list from existing block (already filtered by first pass) + val updatedModules = mutableListOf() + for (existingModule in existingBlock.modules) { + val effectiveLoading = overrides[existingModule.name] ?: existingModule.loading + updatedModules.add(ModuleWithLoading(existingModule.name, effectiveLoading)) + } + + // Create new content block with overrides and replace the old one + val updatedBlock = ContentBlock(moduleSet.name, updatedModules) + val oldBlockIndex = contentBlocks.indexOf(existingBlock) + if (oldBlockIndex >= 0) { + contentBlocks[oldBlockIndex] = updatedBlock + contentBlockByName[moduleSet.name] = updatedBlock + } + // Don't re-add to moduleToSets or moduleToChain - already there from first pass + } + // else: Both have overrides, keep the first one (shouldn't happen in practice) + } + } + // Already processed, don't reprocess + return + } + // Collect module set alias if requested + if (aliasToSource != null && moduleSet.alias != null) { + validateAndRecordAlias( + alias = moduleSet.alias, + source = "module set '${moduleSet.name}'", + aliasToSource = aliasToSource + ) + } + + val currentChain = chain + setName + + // Get direct modules for this set + val directModules = getDirectModules(moduleSet, spec.excludedModules) + + // Build content block and track chains/duplicates in single pass + val modulesWithLoading = mutableListOf() + for (module in directModules) { + // Track for duplicate detection + moduleToSets.computeIfAbsent(module.name) { mutableListOf() }.add(moduleSet.name) + // Track chain + moduleToChain[module.name] = currentChain + // Build loading info - apply overrides from module set + val effectiveLoading = overrides[module.name] ?: module.loading + modulesWithLoading.add(ModuleWithLoading(module.name, effectiveLoading)) + } + + if (modulesWithLoading.isNotEmpty()) { + val block = ContentBlock(moduleSet.name, modulesWithLoading) + contentBlocks.add(block) + contentBlockByName[moduleSet.name] = block + } + + // Recursively process nested sets (no override cascading - each set must be referenced directly for overrides) + for (nestedSet in moduleSet.nestedSets) { + traverse(nestedSet, currentChain, emptyMap()) + } + } + + // Process all top-level module sets + for (moduleSetWithOverrides in spec.moduleSets) { + traverse(moduleSetWithOverrides.moduleSet, emptyList(), moduleSetWithOverrides.loadingOverrides) + } + + // Validate that all overridden modules exist as direct modules in their respective module sets + for (moduleSetWithOverrides in spec.moduleSets) { + validateModuleSetOverrides(moduleSetWithOverrides, spec) + } + + // Check for duplicates and FAIL if found + validateNoDuplicateModules(moduleToSets) + + // Add additional modules if any + val additionalModulesWithLoading = mutableListOf() + for (module in spec.additionalModules) { + if (module.name !in spec.excludedModules) { + additionalModulesWithLoading.add(ModuleWithLoading(module.name, module.loading)) + } + } + + if (additionalModulesWithLoading.isNotEmpty()) { + contentBlocks.add(ContentBlock(ADDITIONAL_MODULES_BLOCK, additionalModulesWithLoading)) + } + + return Triple(contentBlocks, moduleToChain, aliasToSource ?: emptyMap()) +} + +/** + * Collects and validates product-level aliases and merges with module set aliases. + * Checks for duplicates between product-level and module set aliases. + * + * @param spec The product modules specification + * @param moduleSetAliases Aliases collected from module sets during traversal + * @return List of validated unique aliases + */ +internal fun collectAndValidateAliases( + spec: ProductModulesContentSpec, + moduleSetAliases: Map +): List { + val allAliases = moduleSetAliases.toMutableMap() + + // Collect product-level aliases and check for conflicts with module set aliases + for (alias in spec.productModuleAliases) { + validateAndRecordAlias( + alias = alias, + source = "product level", + aliasToSource = allAliases + ) + } + + return allAliases.keys.sorted() +} diff --git a/platform/build-scripts/product-dsl/src/DuplicateIncludeDetector.kt b/platform/build-scripts/product-dsl/src/DuplicateIncludeDetector.kt new file mode 100644 index 000000000000..8385d0fd31e0 --- /dev/null +++ b/platform/build-scripts/product-dsl/src/DuplicateIncludeDetector.kt @@ -0,0 +1,288 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +import com.intellij.openapi.util.JDOMUtil +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import org.jdom.Element +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.isRegularFile +import kotlin.io.path.readText + +/** + * Detects duplicate xi:include references in product plugin.xml files. + * Analyzes both direct xi:includes and nested includes from deprecatedInclude XML files. + */ +object DuplicateIncludeDetector { + + /** + * Detects duplicate xi:include references across all provided product XML files. + * + * @param productXmlFiles List of product plugin.xml file paths to analyze + * @param projectRoot Project root directory for resolving relative paths + * @return Report containing all detected duplicates + */ + fun detectDuplicates( + productXmlFiles: List, + projectRoot: Path + ): DuplicateIncludesReport { + val productsWithDuplicates = mutableListOf() + + for (productFile in productXmlFiles) { + if (!productFile.exists() || !productFile.isRegularFile()) { + continue + } + + val allIncludes = mutableListOf>() + + // 1. Parse direct xi:includes from product file + val directIncludes = parseXiIncludes(productFile) + for (include in directIncludes) { + allIncludes.add(include.href to IncludeSource( + file = productFile.toString(), + sourceType = "xi:include", + lineNumber = include.lineNumber + )) + + // 2. If this xi:include points to a potential deprecatedInclude XML file, parse it too + val resolvedPath = resolveIncludePath(include.href, productFile, projectRoot) + if (resolvedPath != null && resolvedPath.exists() && isDeprecatedIncludeFile(resolvedPath)) { + val nestedIncludes = parseXiIncludes(resolvedPath) + for (nested in nestedIncludes) { + allIncludes.add(nested.href to IncludeSource( + file = resolvedPath.toString(), + sourceType = "deprecatedInclude-nested", + lineNumber = nested.lineNumber + )) + } + } + } + + // 3. Find duplicates + val grouped = allIncludes.groupBy { it.first } + val duplicates = grouped.filter { it.value.size > 1 }.map { (href, sources) -> + DuplicateInclude( + href = href, + resolvedPath = resolveIncludePath(href, productFile, projectRoot)?.toString(), + count = sources.size, + sources = sources.map { it.second } + ) + } + + if (duplicates.isNotEmpty()) { + productsWithDuplicates.add(ProductDuplicates( + name = productFile.fileName.toString().removeSuffix(".xml"), + file = productFile.toString(), + duplicates = duplicates + )) + } + } + + return DuplicateIncludesReport( + timestamp = System.currentTimeMillis().toString(), + products = productsWithDuplicates, + summary = DuplicateSummary( + totalProducts = productXmlFiles.size, + productsWithDuplicates = productsWithDuplicates.size, + totalDuplicateFiles = productsWithDuplicates.sumOf { it.duplicates.size } + ) + ) + } + + /** + * Parses xi:include elements from an XML file. + * Returns list of XiInclude objects with href and line number. + */ + private fun parseXiIncludes(xmlFile: Path): List { + val includes = mutableListOf() + + try { + val content = xmlFile.readText() + val root = JDOMUtil.load(xmlFile) + + // Find all xi:include elements (recursively) + val includeElements = findElementsByName(root, "include") + for (element in includeElements) { + val href = element.getAttributeValue("href") + if (!href.isNullOrEmpty()) { + // Try to find line number by searching in the file content + val lineNumber = findLineNumber(content, href) + includes.add(XiInclude(href, lineNumber)) + } + } + } + catch (e: Exception) { + // Ignore parsing errors + } + + return includes + } + + /** + * Recursively finds all elements with the given local name (ignoring namespace). + */ + private fun findElementsByName(element: Element, localName: String): List { + val result = mutableListOf() + + // Check if current element matches (ignoring namespace prefix) + if (element.name == localName || element.name.endsWith(":$localName")) { + result.add(element) + } + + // Recursively search children + for (child in element.children) { + result.addAll(findElementsByName(child, localName)) + } + + return result + } + + /** + * Attempts to find the line number where an href appears in the XML content. + */ + private fun findLineNumber(content: String, href: String): Int? { + val lines = content.lines() + for ((index, line) in lines.withIndex()) { + if (line.contains("href=\"$href\"")) { + return index + 1 // 1-based line numbers + } + } + return null + } + + /** + * Resolves an xi:include href to an absolute file path. + */ + private fun resolveIncludePath(href: String, currentFile: Path, projectRoot: Path): Path? { + if (href.startsWith("/META-INF/")) { + // Absolute path - search in known resource locations + val locations = listOf( + projectRoot.resolve("community/platform/platform-resources/src$href"), + projectRoot.resolve("community/platform/platform-resources/generated$href"), + projectRoot.resolve("community/java/ide-resources/resources$href"), + projectRoot.resolve("ultimate/platform-ultimate/resources$href"), + projectRoot.resolve("licenseCommon/resources$href"), + projectRoot.resolve("licenseCommon/generated$href"), + // Also check CIDR and other products + projectRoot.resolve("CIDR/clion/main/nolang/resources$href"), + projectRoot.resolve("goland/resources$href"), + projectRoot.resolve("ruby/resources$href"), + projectRoot.resolve("WebStorm/resources$href"), + projectRoot.resolve("dbe/ide/resources$href"), + projectRoot.resolve("aqua/branding/resources$href"), + projectRoot.resolve("rider/resources$href"), + ) + + for (loc in locations) { + if (loc.exists()) { + return loc + } + } + return null + } + else if (href.startsWith("/")) { + // Handle paths like /something.xml + return resolveIncludePath("/META-INF$href", currentFile, projectRoot) + } + else { + // Relative path + val dir = currentFile.parent + return dir?.resolve(href) + } + } + + /** + * Checks if the given path is a deprecatedInclude XML file that might contain nested xi:includes. + */ + private fun isDeprecatedIncludeFile(path: Path): Boolean { + val fileName = path.fileName.toString() + return fileName.endsWith("-customization.xml") || + fileName == "ultimate.xml" || + fileName == "PlatformLangPlugin.xml" || + fileName == "JavaIdePlugin.xml" || + fileName == "structuralsearch.xml" || + fileName.endsWith("Plugin.xml") + } + + /** + * Represents a single xi:include element. + */ + private data class XiInclude( + val href: String, + val lineNumber: Int? + ) +} + +/** + * Detects and prints duplicate xi:include elements in product plugin.xml files. + * This is a convenience function that takes discovered products and outputs JSON report. + * + * @param products List of discovered products with their plugin XML paths + * @param projectRoot Project root directory for resolving relative paths + */ +fun detectAndPrintDuplicateIncludes(products: List, projectRoot: Path) { + // Convert ProductSpec plugin paths to actual file paths + val productFiles = products + .mapNotNull { it.pluginXmlPath } + .map { projectRoot.resolve(it) } + .filter { it.exists() && it.isRegularFile() } + + // Run detection + val report = DuplicateIncludeDetector.detectDuplicates(productFiles, projectRoot) + + // Output JSON + val json = Json { prettyPrint = true } + println(json.encodeToString(report)) +} + +/** + * Report containing all detected duplicate includes. + */ +@Serializable +data class DuplicateIncludesReport( + val timestamp: String, + val products: List, + val summary: DuplicateSummary +) + +/** + * Duplicates found in a single product. + */ +@Serializable +data class ProductDuplicates( + val name: String, + val file: String, + val duplicates: List +) + +/** + * A single duplicate include with its sources. + */ +@Serializable +data class DuplicateInclude( + val href: String, + val resolvedPath: String?, + val count: Int, + val sources: List +) + +/** + * Source of an include (where it was found). + */ +@Serializable +data class IncludeSource( + val file: String, + val sourceType: String, // "xi:include" or "deprecatedInclude-nested" + val lineNumber: Int? +) + +/** + * Summary statistics. + */ +@Serializable +data class DuplicateSummary( + val totalProducts: Int, + val productsWithDuplicates: Int, + val totalDuplicateFiles: Int +) diff --git a/platform/build-scripts/product-dsl/src/GeneratorModel.kt b/platform/build-scripts/product-dsl/src/GeneratorModel.kt new file mode 100644 index 000000000000..2be7149b58d8 --- /dev/null +++ b/platform/build-scripts/product-dsl/src/GeneratorModel.kt @@ -0,0 +1,64 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.intellij.build.productLayout + +import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule + +// Constants for magic strings used throughout the generator +internal const val ADDITIONAL_MODULES_BLOCK = "additional" +internal const val JETBRAINS_NAMESPACE = "jetbrains" +internal const val GENERATOR_SUFFIX = ".main()" +internal const val MODULE_SET_PREFIX = "intellij.moduleSets." + +/** + * Type-safe wrapper for module set names. + * Prevents accidental mixing of module set names with other string types. + */ +@JvmInline +value class ModuleSetName(val value: String) { + override fun toString(): String = value +} + +/** + * Represents a single content block in a product plugin.xml. + * Each block corresponds to a module set or additional modules section. + */ +data class ContentBlock( + /** Source identifier for the block (e.g., "essential", "vcs", "additional") */ + @JvmField val source: String, + /** List of modules with their effective loading modes */ + @JvmField val modules: List, +) + +/** + * A module with its effective loading mode after applying overrides and exclusions. + */ +data class ModuleWithLoading( + /** Module name */ + @JvmField val name: String, + /** Effective loading mode (null means default/no attribute) */ + @JvmField val loading: ModuleLoadingRule?, +) + +/** + * Result of building product content XML. + * Contains the generated XML string, content blocks, and module-to-set chain mapping. + */ +data class ProductContentBuildResult( + /** Generated XML content as string */ + @JvmField val xml: String, + /** List of content blocks generated from the spec */ + @JvmField val contentBlocks: List, + /** Mapping from module name to its module set chain as list (e.g., ["parent", "child"]) */ + @JvmField val moduleToSetChainMapping: Map>, +) + +/** + * Result of building module set XML. + * Contains the XML string and count of direct modules (excluding nested). + */ +internal data class ModuleSetBuildResult( + @JvmField val xml: String, + @JvmField val directModuleCount: Int, +) \ No newline at end of file diff --git a/platform/build-scripts/product-dsl/src/ModuleLoadingOverrideBuilder.kt b/platform/build-scripts/product-dsl/src/ModuleLoadingOverrideBuilder.kt new file mode 100644 index 000000000000..552be24afb24 --- /dev/null +++ b/platform/build-scripts/product-dsl/src/ModuleLoadingOverrideBuilder.kt @@ -0,0 +1,55 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplacePutWithAssignment") + +package org.jetbrains.intellij.build.productLayout + +import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule +import kotlinx.serialization.Serializable + +/** + * Wrapper for a module set with optional loading overrides. + * When overrides are present, the module set will be inlined in the product XML + * instead of being referenced via xi:include. + */ +@Serializable +data class ModuleSetWithOverrides( + @JvmField val moduleSet: ModuleSet, + @JvmField val loadingOverrides: Map = emptyMap(), +) { + val hasOverrides: Boolean + get() = loadingOverrides.isNotEmpty() +} + +/** + * DSL builder for module loading overrides within a module set. + */ +@ProductDslMarker +class ModuleLoadingOverrideBuilder { + private val overrides = HashMap() + + /** + * Override a module in this module set to be loaded as embedded (loading="embedded"). + */ + fun overrideAsEmbedded(moduleName: String) { + overrides.put(moduleName, ModuleLoadingRule.EMBEDDED) + } + + /** + * Override a module in this module set to be loaded as required (loading="required"). + */ + fun overrideAsRequired(moduleName: String) { + overrides.put(moduleName, ModuleLoadingRule.REQUIRED) + } + + /** + * Set custom loading rule for modules. + */ + fun loading(rule: ModuleLoadingRule, vararg moduleNames: String) { + for (name in moduleNames) { + overrides.put(name, rule) + } + } + + @PublishedApi + internal fun build(): Map = overrides.toMap() +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ModuleSetBuilder.kt b/platform/build-scripts/product-dsl/src/ModuleSetBuilder.kt similarity index 86% rename from platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ModuleSetBuilder.kt rename to platform/build-scripts/product-dsl/src/ModuleSetBuilder.kt index f739b3cbb440..5d811565ccae 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ModuleSetBuilder.kt +++ b/platform/build-scripts/product-dsl/src/ModuleSetBuilder.kt @@ -2,12 +2,23 @@ package org.jetbrains.intellij.build.productLayout import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule +import kotlinx.serialization.Serializable import org.jetbrains.intellij.build.BuildPaths -import java.lang.invoke.MethodHandles -import java.lang.invoke.MethodType import java.nio.file.Files import java.nio.file.Path +/** + * Represents a content module with optional loading attribute. + * + * @param name Module name + * @param loading Optional loading mode (e.g., ModuleLoadingRule.EMBEDDED) + */ +@Serializable +data class ContentModule( + @JvmField val name: String, + @JvmField val loading: ModuleLoadingRule? = null, +) + /** * Represents a named collection of content modules. * The name serves as metadata for debugging and XML generation (used in the 'source' attribute). @@ -17,6 +28,7 @@ import java.nio.file.Path * @param nestedSets List of nested module sets (for xi:include generation) * @param alias Optional module alias for `` declaration (e.g., "com.intellij.modules.xml") */ +@Serializable data class ModuleSet( @JvmField val name: String, @JvmField val modules: List, @@ -38,6 +50,7 @@ interface ModuleSetProvider { /** * DSL builder for creating ModuleSets with reduced boilerplate. */ +@ProductDslMarker class ModuleSetBuilder { private val modules = mutableListOf() private val nestedSets = mutableListOf() @@ -146,15 +159,6 @@ private fun appendModuleSetContent(sb: StringBuilder, moduleSet: ModuleSet, inde } } -/** - * Result of building module set XML. - * Contains the XML string and count of direct modules (excluding nested). - */ -data class ModuleSetBuildResult( - val xml: String, - val directModuleCount: Int, -) - /** * Builds the XML content for a module set. * @@ -202,32 +206,6 @@ internal fun buildModuleSetXml(moduleSet: ModuleSet, label: String): ModuleSetBu return ModuleSetBuildResult(xml, directModuleCount) } -/** - * Discovers all module set functions in the given object using reflection. - * Returns all public functions that: - * - Return ModuleSet - * - Take no parameters - * - Are not named 'main' - * - * @param obj The object to scan for module set functions (e.g., CommunityModuleSets, UltimateModuleSets) - * @return List of all discovered ModuleSets - */ -private fun discoverModuleSets(obj: Any): List { - val lookup = MethodHandles.lookup() - val clazz = obj.javaClass - val methodType = MethodType.methodType(ModuleSet::class.java) - - val declaredMethods = clazz.declaredMethods - val result = ArrayList(declaredMethods.size) - for (method in declaredMethods) { - if (method.parameterCount == 0 && java.lang.reflect.Modifier.isPublic(method.modifiers) && method.returnType == ModuleSet::class.java) { - val moduleSet = lookup.findVirtual(clazz, method.name, methodType).invoke(obj) as ModuleSet - result.add(moduleSet) - } - } - return result -} - /** * Generates all module set XMLs for the given object. * Discovers all ModuleSet functions via reflection, generates XML files, and prints results. diff --git a/platform/build-scripts/product-dsl/src/ModuleSetDiscovery.kt b/platform/build-scripts/product-dsl/src/ModuleSetDiscovery.kt new file mode 100644 index 000000000000..c0f8f82affbc --- /dev/null +++ b/platform/build-scripts/product-dsl/src/ModuleSetDiscovery.kt @@ -0,0 +1,29 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +import java.lang.invoke.MethodHandles +import java.lang.invoke.MethodType + +/** + * Discovers all module sets from an object using reflection. + * Returns all public functions that return ModuleSet and take no parameters. + * + * This consolidates the duplicated discovery logic that was in both UltimateModuleSets and ultimateGenerator. + */ +fun discoverModuleSets(provider: Any): List { + val lookup = MethodHandles.lookup() + val clazz = provider.javaClass + val methodType = MethodType.methodType(ModuleSet::class.java) + + val declaredMethods = clazz.declaredMethods + val result = ArrayList(declaredMethods.size) + for (method in declaredMethods) { + if (method.parameterCount == 0 && + java.lang.reflect.Modifier.isPublic(method.modifiers) && + method.returnType == ModuleSet::class.java) { + val moduleSet = lookup.findVirtual(clazz, method.name, methodType).invoke(provider) as ModuleSet + result.add(moduleSet) + } + } + return result +} diff --git a/platform/build-scripts/product-dsl/src/ModuleSetJsonExport.kt b/platform/build-scripts/product-dsl/src/ModuleSetJsonExport.kt new file mode 100644 index 000000000000..8164193456f8 --- /dev/null +++ b/platform/build-scripts/product-dsl/src/ModuleSetJsonExport.kt @@ -0,0 +1,1917 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +import com.fasterxml.jackson.core.JsonEncoding +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.JsonGenerator +import com.intellij.openapi.util.JDOMUtil +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import java.nio.file.Files +import java.nio.file.Path +import java.time.Instant + +/** + * Metadata about a module set including its location and source file. + * + * @param moduleSet The module set instance + * @param location The location category ("community" or "ultimate") + * @param sourceFile The Kotlin source file path relative to project root where this module set is defined + */ +data class ModuleSetMetadata( + val moduleSet: ModuleSet, + val location: String, + val sourceFile: String +) + +/** + * JSON filter for selective analysis output. + * Supports various filter types: products, moduleSets, composition, duplicates, or specific items. + */ +@Serializable +data class JsonFilter( + val filter: String, // "products", "moduleSets", "composition", "duplicates", "product", "moduleSet" + val value: String? = null, // Product/module set name when filter is "product" or "moduleSet" + val includeDuplicates: Boolean = false // Include duplicate xi:include detection in output (for future unification) +) + +/** + * Product specification for JSON output. + * Contains essential product metadata with source file paths for AI navigation, + * plus complete ProductModulesContentSpec for full DSL representation. + */ +data class ProductSpec( + val name: String, + val className: String?, + val sourceFile: String, + val pluginXmlPath: String?, + val contentSpec: ProductModulesContentSpec?, + val buildModules: List, + val totalModuleCount: Int = 0, // All modules including from module sets + val directModuleCount: Int = 0, // Just additionalModules count + val moduleSetCount: Int = 0, // Number of module sets included + val uniqueModuleCount: Int = 0 // Deduplicated module count +) + +/** + * Module location information from .idea/modules.xml. + * + * @param location Module location: "community", "ultimate", or "unknown" + * @param imlPath Absolute path to the .iml file + */ +data class ModuleLocationInfo( + val location: String, // "community", "ultimate", or "unknown" + val imlPath: String? +) + +/** + * Parses .idea/modules.xml to determine module locations (community vs ultimate). + * This information is used by validation functions to ensure architectural constraints. + * + * @param projectRoot Absolute path to the project root directory + * @return Map of module name to ModuleLocationInfo + */ +fun parseModulesXml(projectRoot: Path): Map { + val modulesXmlPath = projectRoot.resolve(".idea/modules.xml") + + if (!Files.exists(modulesXmlPath)) { + return emptyMap() + } + + val modules = mutableMapOf() + + try { + val document = JDOMUtil.load(modulesXmlPath) + + // Get all elements under + val projectModuleManager = document.getChildren("component") + .find { it.getAttributeValue("name") == "ProjectModuleManager" } + + val modulesParent = projectModuleManager?.getChild("modules") ?: return emptyMap() + + for (moduleElement in modulesParent.getChildren("module")) { + var filepath = moduleElement.getAttributeValue("filepath") ?: continue + + // Replace $PROJECT_DIR$ with actual project root + filepath = filepath.replace("\$PROJECT_DIR\$", projectRoot.toString()) + + // Extract module name from .iml filename + val moduleName = Path.of(filepath).fileName.toString().removeSuffix(".iml") + + // Determine location based on filepath + val location = when { + filepath.contains("/community/") -> "community" + filepath.contains("/ultimate/") -> "ultimate" + else -> "unknown" + } + + modules[moduleName] = ModuleLocationInfo(location, filepath) + } + } + catch (e: Exception) { + // If parsing fails, return empty map (validation will report as unknown) + System.err.println("Warning: Failed to parse .idea/modules.xml: ${e.message}") + } + + return modules +} + +/** + * Derives source file path from ProductProperties class name. + * Maps package structure to file system path relative to project root. + */ +fun getProductPropertiesSourceFile(clazz: Class<*>, @Suppress("UNUSED_PARAMETER") projectRoot: Path): String { + val className = clazz.name + + // Map package to directory structure + // org.jetbrains.intellij.build.IdeaUltimateProperties -> build/src/org/jetbrains/intellij/build/IdeaUltimateProperties.kt + // org.jetbrains.intellij.build.goland.GoLandProperties -> goland/intellij-go-build/src/org/jetbrains/intellij/build/goland/GoLandProperties.kt + + val packagePath = className.replace('.', '/') + ".kt" + + // Handle special cases based on package + return when { + className.startsWith("org.jetbrains.intellij.build.goland.") -> + "goland/intellij-go-build/src/$packagePath" + className.startsWith("org.jetbrains.intellij.build.clion.") -> + "CIDR/clion-build/src/$packagePath" + className.startsWith("com.jetbrains.rider.build.") -> + "rider/build/src/$packagePath" + className.startsWith("org.jetbrains.intellij.build.dataGrip.") -> + "dbe/build/src/$packagePath" + className.startsWith("org.jetbrains.intellij.build.") && !className.contains("community") -> + "build/src/$packagePath" + className.startsWith("org.jetbrains.intellij.build.") -> + "community/build/src/$packagePath" + else -> packagePath // Fallback to package path + } +} + +// kotlinx.serialization Json instance for serializing data structures +private val kotlinxJson = Json { + prettyPrint = false + encodeDefaults = true +} + +/** + * Enriches products with calculated metrics. + * Calculates totalModuleCount, directModuleCount, moduleSetCount, and uniqueModuleCount for each product. + */ +private fun enrichProductsWithMetrics( + products: List, + moduleSets: List +): List { + return products.map { product -> + val contentSpec = product.contentSpec + if (contentSpec == null) { + product // Return as-is if no contentSpec + } else { + // Calculate metrics + val allModules = mutableSetOf() + + // Collect modules from module sets + for (msRef in contentSpec.moduleSets) { + val modulesFromSet = collectAllModuleNamesFromSet(moduleSets, msRef.moduleSet.name) + allModules.addAll(modulesFromSet) + } + + // Add additional modules + for (module in contentSpec.additionalModules) { + allModules.add(module.name) + } + + // Remove excluded modules + for (excludedModule in contentSpec.excludedModules) { + allModules.remove(excludedModule) + } + + val totalModuleCount = allModules.size + val directModuleCount = contentSpec.additionalModules.size + val moduleSetCount = contentSpec.moduleSets.size + val uniqueModuleCount = allModules.size // Same as total after deduplication + + product.copy( + totalModuleCount = totalModuleCount, + directModuleCount = directModuleCount, + moduleSetCount = moduleSetCount, + uniqueModuleCount = uniqueModuleCount + ) + } + } +} + +/** + * Streams comprehensive module set and product analysis data as JSON to stdout. + * Uses hybrid approach: + * - Jackson JsonGenerator for overall structure and analysis sections + * - kotlinx.serialization for automatic serialization of DSL data structures (ModuleSet, ProductModulesContentSpec) + * + * Generic function that accepts discovered module sets and products - similar to buildProductContentXml pattern. + * Caller is responsible for discovering module sets and products and providing them. + * + * Output includes: + * - Module sets with complete ModuleSet structure (modules, nested sets, aliases) + * - Products with full ProductModulesContentSpec (module sets with overrides, additional modules, exclusions) + * - Source file paths (relative to project root) for easy navigation + * - Duplicate analysis (modules in multiple sets) + * - Set overlap analysis (for unification opportunities) + * + * This data enables AI to: + * - See complete product DSL specifications with all modules and overrides + * - Detect duplicate modules across module sets + * - Find unification opportunities + * - Identify redundant includes or module specifications + * - Analyze module set overlap and relationships + * - Navigate to source files to make changes + * + * @param allModuleSets List of ModuleSetMetadata instances with module set, location, and source file path + * @param products List of ProductSpec instances + * @param projectRoot Project root path for resolving .idea/modules.xml + * @param filter Optional filter: null (full), or JsonFilter object with filter type and optional value + */ +fun streamModuleAnalysisJson( + allModuleSets: List, + products: List, + projectRoot: Path, + filter: JsonFilter? = null +) { + // Validate product specifications using shared validation (always validate, even when filtering) + val moduleSets = allModuleSets.map { it.moduleSet } + val productSpecs = products.map { it.name to it.contentSpec } + validateNoRedundantModuleSets(moduleSets, productSpecs) + + // Enrich products with calculated metrics + val enrichedProducts = enrichProductsWithMetrics(products, moduleSets) + + val generator = JsonFactory() + .createGenerator(System.out, JsonEncoding.UTF8) + .configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false) + .configure(JsonGenerator.Feature.FLUSH_PASSED_TO_STREAM, false) + .useDefaultPrettyPrinter() + + generator.use { gen -> + gen.writeStartObject() + gen.writeStringField("timestamp", Instant.now().toString()) + + // Apply filter + when { + filter == null -> { + // Full JSON (no filter) + writeAllSections(gen, allModuleSets, enrichedProducts, projectRoot) + } + filter.filter == "products" -> { + gen.writeArrayFieldStart("products") + for (product in enrichedProducts) { + writeProduct(gen, product) + } + gen.writeEndArray() + } + filter.filter == "moduleSets" -> { + gen.writeArrayFieldStart("moduleSets") + for ((moduleSet, location, sourceFilePath) in allModuleSets) { + writeModuleSet(gen, moduleSet, location, sourceFilePath, moduleSets) + } + gen.writeEndArray() + } + filter.filter == "composition" -> { + gen.writeObjectFieldStart("productCompositionAnalysis") + writeProductCompositionAnalysis(gen, enrichedProducts) + gen.writeEndObject() + } + filter.filter == "duplicates" -> { + gen.writeObjectFieldStart("duplicateAnalysis") + writeDuplicateAnalysis(gen, allModuleSets) + gen.writeEndObject() + } + filter.filter == "product" && filter.value != null -> { + val productName = filter.value + val product = enrichedProducts.firstOrNull { it.name == productName } + if (product != null) { + gen.writeFieldName("product") + gen.writeStartObject() + // Copy writeProduct logic inline but without outer object wrapper + gen.writeStringField("name", product.name) + gen.writeStringField("className", product.className) + gen.writeStringField("sourceFile", product.sourceFile) + if (product.pluginXmlPath != null) { + gen.writeStringField("pluginXmlPath", product.pluginXmlPath) + } + if (product.contentSpec != null) { + val contentSpecJson = kotlinxJson.encodeToString(product.contentSpec) + gen.writeFieldName("contentSpec") + gen.writeRawValue(contentSpecJson) + } + gen.writeArrayFieldStart("buildModules") + for (buildModule in product.buildModules) { + gen.writeString(buildModule) + } + gen.writeEndArray() + gen.writeEndObject() + } else { + gen.writeStringField("error", "Product '$productName' not found") + } + } + filter.filter == "moduleSet" && filter.value != null -> { + val moduleSetName = filter.value + val moduleSetEntry = allModuleSets.firstOrNull { it.moduleSet.name == moduleSetName } + if (moduleSetEntry != null) { + gen.writeFieldName("moduleSet") + gen.writeStartObject() + gen.writeStringField("name", moduleSetEntry.moduleSet.name) + gen.writeStringField("location", moduleSetEntry.location) + gen.writeStringField("sourceFile", moduleSetEntry.sourceFile) + val moduleSetJson = kotlinxJson.encodeToString(moduleSetEntry.moduleSet) + gen.writeFieldName("moduleSet") + gen.writeRawValue(moduleSetJson) + gen.writeEndObject() + } else { + gen.writeStringField("error", "Module set '$moduleSetName' not found") + } + } + else -> { + gen.writeStringField("error", "Unknown filter: ${filter.filter}") + } + } + + gen.writeEndObject() + } + + // Flush stdout to ensure all data is written + System.out.flush() +} + +/** + * Writes all sections of the analysis JSON (used when no filter is specified). + */ +private fun writeAllSections( + gen: JsonGenerator, + allModuleSets: List, + products: List, + projectRoot: Path +) { + // Write module sets + val moduleSets = allModuleSets.map { it.moduleSet } + gen.writeArrayFieldStart("moduleSets") + for ((moduleSet, location, sourceFilePath) in allModuleSets) { + writeModuleSet(gen, moduleSet, location, sourceFilePath, moduleSets) + } + gen.writeEndArray() + + // Write products + gen.writeArrayFieldStart("products") + for (product in products) { + writeProduct(gen, product) + } + gen.writeEndArray() + + // Write duplicate analysis + gen.writeObjectFieldStart("duplicateAnalysis") + writeDuplicateAnalysis(gen, allModuleSets) + gen.writeEndObject() + + // Write product composition analysis + gen.writeObjectFieldStart("productCompositionAnalysis") + writeProductCompositionAnalysis(gen, products) + gen.writeEndObject() + + // Write module distribution analysis + gen.writeObjectFieldStart("moduleDistribution") + writeModuleDistribution(gen, allModuleSets, products, projectRoot) + gen.writeEndObject() + + // Write module set hierarchy + gen.writeObjectFieldStart("moduleSetHierarchy") + writeModuleSetHierarchy(gen, allModuleSets) + gen.writeEndObject() + + // Write module usage index + gen.writeObjectFieldStart("moduleUsageIndex") + writeModuleUsageIndex(gen, allModuleSets, products) + gen.writeEndObject() + + // Parse module locations for validation + val moduleLocations = parseModulesXml(projectRoot) + + // Validate community products don't use ultimate modules + val communityViolations = validateCommunityProducts(products, allModuleSets, moduleLocations, projectRoot) + gen.writeObjectFieldStart("communityProductViolations") + writeCommunityProductViolations(gen, communityViolations) + gen.writeEndObject() + + // Validate module sets are in correct locations + val locationViolations = validateModuleSetLocations(allModuleSets, moduleLocations, projectRoot) + gen.writeObjectFieldStart("moduleSetLocationViolations") + writeModuleSetLocationViolations(gen, locationViolations) + gen.writeEndObject() + + // Analyze product similarity for refactoring recommendations + val similarityPairs = analyzeProductSimilarity(products, similarityThreshold = 0.7) + gen.writeObjectFieldStart("productSimilarity") + writeProductSimilarityAnalysis(gen, similarityPairs, 0.7) + gen.writeEndObject() + + // Detect module set overlaps (with nested set filtering to avoid false positives) + val moduleSetOverlaps = detectModuleSetOverlap(allModuleSets, minOverlapPercent = 50) + gen.writeObjectFieldStart("moduleSetOverlap") + writeModuleSetOverlapAnalysis(gen, moduleSetOverlaps, 50) + gen.writeEndObject() + + // Generate unification suggestions based on overlaps and similarity + val unificationSuggestions = suggestModuleSetUnification( + allModuleSets = allModuleSets, + products = products, + overlaps = moduleSetOverlaps, + similarityPairs = similarityPairs, + maxSuggestions = 10, + strategy = "all" + ) + gen.writeObjectFieldStart("unificationSuggestions") + writeUnificationSuggestions(gen, unificationSuggestions) + gen.writeEndObject() +} + +/** + * Writes a single module set to JSON. + * Uses kotlinx.serialization to serialize the ModuleSet structure directly, + * then embeds the raw JSON using writeRawValue(). + */ +private fun writeModuleSet( + gen: JsonGenerator, + moduleSet: ModuleSet, + location: String, + sourceFilePath: String, + allModuleSets: List +) { + gen.writeStartObject() + + // Metadata fields + gen.writeStringField("name", moduleSet.name) + gen.writeStringField("location", location) + gen.writeStringField("sourceFile", sourceFilePath) + + // Serialize ModuleSet using kotlinx.serialization and write raw JSON + val moduleSetJson = kotlinxJson.encodeToString(moduleSet) + gen.writeFieldName("moduleSet") + gen.writeRawValue(moduleSetJson) + + // Add flattened list of all modules (including from nested sets) + gen.writeArrayFieldStart("allModulesFlattened") + val allModules = collectAllModuleNamesFromSet(allModuleSets, moduleSet.name) + for (moduleName in allModules.sorted()) { + gen.writeString(moduleName) + } + gen.writeEndArray() + + gen.writeEndObject() +} + +/** + * Writes a single product to JSON. + * Uses kotlinx.serialization to serialize ProductModulesContentSpec directly, + * providing complete DSL structure with all modules, overrides, and exclusions. + */ +private fun writeProduct( + gen: JsonGenerator, + product: ProductSpec +) { + gen.writeStartObject() + gen.writeStringField("name", product.name) + gen.writeStringField("className", product.className) + gen.writeStringField("sourceFile", product.sourceFile) + + if (product.pluginXmlPath != null) { + gen.writeStringField("pluginXmlPath", product.pluginXmlPath) + } + + // Serialize ProductModulesContentSpec using kotlinx.serialization and write raw JSON + if (product.contentSpec != null) { + val contentSpecJson = kotlinxJson.encodeToString(product.contentSpec) + gen.writeFieldName("contentSpec") + gen.writeRawValue(contentSpecJson) + } + + // Build modules + gen.writeArrayFieldStart("buildModules") + for (buildModule in product.buildModules) { + gen.writeString(buildModule) + } + gen.writeEndArray() + + // Metrics + gen.writeNumberField("totalModuleCount", product.totalModuleCount) + gen.writeNumberField("directModuleCount", product.directModuleCount) + gen.writeNumberField("moduleSetCount", product.moduleSetCount) + gen.writeNumberField("uniqueModuleCount", product.uniqueModuleCount) + + gen.writeEndObject() +} + +/** + * Writes duplicate analysis section. + */ +private fun writeDuplicateAnalysis( + gen: JsonGenerator, + allModuleSets: List +) { + // Find modules that appear in multiple module sets + val moduleToSets = mutableMapOf>() + for ((moduleSet, _, _) in allModuleSets) { + val allModules = collectAllModuleNames(moduleSet) + for (moduleName in allModules) { + moduleToSets.getOrPut(moduleName) { mutableListOf() }.add(moduleSet.name) + } + } + + val duplicateModules = moduleToSets.filter { it.value.size > 1 } + + gen.writeArrayFieldStart("modulesInMultipleSets") + for ((moduleName, setNames) in duplicateModules.entries.sortedBy { it.key }) { + gen.writeStartObject() + gen.writeStringField("moduleName", moduleName) + + gen.writeArrayFieldStart("appearsInSets") + for (setName in setNames.sorted()) { + gen.writeString(setName) + } + gen.writeEndArray() + + gen.writeEndObject() + } + gen.writeEndArray() + + // Set overlap analysis + gen.writeArrayFieldStart("setOverlapAnalysis") + for (i in allModuleSets.indices) { + for (j in i + 1 until allModuleSets.size) { + val (set1, _, _) = allModuleSets[i] + val (set2, _, _) = allModuleSets[j] + + val modules1 = collectAllModuleNames(set1) + val modules2 = collectAllModuleNames(set2) + + val overlap = modules1.intersect(modules2) + if (overlap.size > 5) { // Only report significant overlaps + val uniqueToSet1 = modules1 - modules2 + val uniqueToSet2 = modules2 - modules1 + + gen.writeStartObject() + gen.writeStringField("set1", set1.name) + gen.writeStringField("set2", set2.name) + gen.writeNumberField("overlapCount", overlap.size) + gen.writeNumberField("set1TotalModules", modules1.size) + gen.writeNumberField("set2TotalModules", modules2.size) + + val overlapPercentage = (overlap.size.toDouble() / minOf(modules1.size, modules2.size)) * 100 + gen.writeNumberField("overlapPercentage", overlapPercentage) + + if (uniqueToSet1.isNotEmpty()) { + gen.writeArrayFieldStart("uniqueToSet1") + for (moduleName in uniqueToSet1.sorted().take(10)) { // Limit to first 10 + gen.writeString(moduleName) + } + gen.writeEndArray() + } + + if (uniqueToSet2.isNotEmpty()) { + gen.writeArrayFieldStart("uniqueToSet2") + for (moduleName in uniqueToSet2.sorted().take(10)) { // Limit to first 10 + gen.writeString(moduleName) + } + gen.writeEndArray() + } + + gen.writeEndObject() + } + } + } + gen.writeEndArray() +} + +/** + * Analyzes product composition graphs and writes insights to JSON. + * Provides statistics and recommendations for each product based on its composition. + */ +private fun writeProductCompositionAnalysis( + gen: JsonGenerator, + products: List +) { + gen.writeArrayFieldStart("products") + + for (product in products) { + val contentSpec = product.contentSpec ?: continue + val compositionGraph = contentSpec.compositionGraph + + if (compositionGraph.isEmpty()) continue + + gen.writeStartObject() + gen.writeStringField("productName", product.name) + + // Count composition types + val typeCounts = compositionGraph.groupBy { it.type }.mapValues { it.value.size } + gen.writeObjectFieldStart("compositionCounts") + for ((type, count) in typeCounts.entries.sortedBy { it.key.name }) { + gen.writeNumberField(type.name.lowercase(), count) + } + gen.writeEndObject() + + // Total operations + gen.writeNumberField("totalCompositionOperations", compositionGraph.size) + + // List module set references + val moduleSetRefs = compositionGraph.filter { it.type == CompositionType.MODULE_SET_REF } + if (moduleSetRefs.isNotEmpty()) { + gen.writeArrayFieldStart("moduleSetReferences") + for (ref in moduleSetRefs) { + gen.writeStartObject() + gen.writeStringField("name", ref.reference ?: "unknown") + gen.writeArrayFieldStart("path") + for (pathItem in ref.path) { + gen.writeString(pathItem) + } + gen.writeEndArray() + gen.writeEndObject() + } + gen.writeEndArray() + } + + // List inline spec includes + val inlineSpecs = compositionGraph.filter { it.type == CompositionType.INLINE_SPEC } + if (inlineSpecs.isNotEmpty()) { + gen.writeArrayFieldStart("inlineSpecIncludes") + for (spec in inlineSpecs) { + gen.writeStartObject() + gen.writeStringField("reference", spec.reference ?: "unknown") + if (spec.sourceLocation != null) { + gen.writeStringField("sourceFile", spec.sourceLocation) + } + gen.writeEndObject() + } + gen.writeEndArray() + } + + gen.writeEndObject() + } + + gen.writeEndArray() +} + +/** + * Writes module distribution analysis. + * For each module, lists which module sets and products use it, plus location information. + * Replaces TypeScript analyzeModuleDistribution() function. + * + * Output format: { "moduleName": { "inModuleSets": [...], "inProducts": [...], "location": "...", "imlPath": "..." } } + */ +private fun writeModuleDistribution( + gen: JsonGenerator, + allModuleSets: List, + products: List, + projectRoot: Path +) { + // Parse module locations from .idea/modules.xml + val moduleLocations = parseModulesXml(projectRoot) + + // Build module → {inModuleSets: [], inProducts: []} mapping + val moduleMap = mutableMapOf() + + // Collect modules from module sets + for ((moduleSet, _, _) in allModuleSets) { + for (module in moduleSet.modules) { + val info = moduleMap.getOrPut(module.name) { ModuleDistributionInfo() } + info.inModuleSets.add(moduleSet.name) + } + } + + // Collect modules from products + for (product in products) { + val contentSpec = product.contentSpec ?: continue + + // Collect all modules used by this product (from module sets + additional modules) + val allModulesInProduct = mutableSetOf() + + // Collect from module sets (recursively) + for (msRef in contentSpec.moduleSets) { + val modulesFromSet = collectAllModuleNamesFromSet(allModuleSets.map { it.moduleSet }, msRef.moduleSet.name) + allModulesInProduct.addAll(modulesFromSet) + } + + // Add additional modules + for (module in contentSpec.additionalModules) { + allModulesInProduct.add(module.name) + } + + // Add to module map + for (moduleName in allModulesInProduct) { + val info = moduleMap.getOrPut(moduleName) { ModuleDistributionInfo() } + if (product.name !in info.inProducts) { + info.inProducts.add(product.name) + } + } + } + + // Set location information from .idea/modules.xml + for ((moduleName, info) in moduleMap) { + val locationInfo = moduleLocations[moduleName] + if (locationInfo != null) { + info.location = locationInfo.location + info.imlPath = locationInfo.imlPath + } + } + + // Write JSON as object (not array) for direct access by module name + for ((moduleName, info) in moduleMap.entries.sortedBy { it.key }) { + gen.writeObjectFieldStart(moduleName) + + gen.writeArrayFieldStart("inModuleSets") + for (setName in info.inModuleSets.sorted()) { + gen.writeString(setName) + } + gen.writeEndArray() + + gen.writeArrayFieldStart("inProducts") + for (productName in info.inProducts.sorted()) { + gen.writeString(productName) + } + gen.writeEndArray() + + gen.writeStringField("location", info.location) + if (info.imlPath != null) { + gen.writeStringField("imlPath", info.imlPath) + } + + gen.writeEndObject() + } +} + +/** + * Helper data class for module distribution analysis. + */ +private data class ModuleDistributionInfo( + val inModuleSets: MutableList = mutableListOf(), + val inProducts: MutableList = mutableListOf(), + var location: String = "unknown", // "community", "ultimate", or "unknown" + var imlPath: String? = null +) + +/** + * Writes module set hierarchy analysis. + * For each module set, lists what it includes and what includes it. + * Replaces TypeScript buildModuleSetHierarchy() function. + * + * Output format: { "setName": { "includes": [...], "includedBy": [...], "moduleCount": N } } + */ +private fun writeModuleSetHierarchy( + gen: JsonGenerator, + allModuleSets: List +) { + // Build hierarchy map: moduleSetName → {includes: [], includedBy: [], moduleCount: N} + val hierarchy = mutableMapOf() + + // First pass: build includes and module counts + for ((moduleSet, _, _) in allModuleSets) { + val info = ModuleSetHierarchyInfo( + includes = moduleSet.nestedSets.map { it.name }, + moduleCount = moduleSet.modules.size + ) + hierarchy[moduleSet.name] = info + } + + // Second pass: build reverse references (includedBy) + for ((moduleSet, _, _) in allModuleSets) { + for (nestedSet in moduleSet.nestedSets) { + hierarchy[nestedSet.name]?.includedBy?.add(moduleSet.name) + } + } + + // Write JSON as flat object (no "moduleSets" wrapper) for direct access + for ((setName, info) in hierarchy.entries.sortedBy { it.key }) { + gen.writeObjectFieldStart(setName) + + gen.writeArrayFieldStart("includes") + for (includedSet in info.includes.sorted()) { + gen.writeString(includedSet) + } + gen.writeEndArray() + + gen.writeArrayFieldStart("includedBy") + for (parentSet in info.includedBy.sorted()) { + gen.writeString(parentSet) + } + gen.writeEndArray() + + gen.writeNumberField("moduleCount", info.moduleCount) + + gen.writeEndObject() + } +} + +/** + * Helper data class for module set hierarchy analysis. + */ +private data class ModuleSetHierarchyInfo( + val includes: List, + val includedBy: MutableList = mutableListOf(), + val moduleCount: Int +) + +/** + * Writes module usage index. + * For each module, provides complete information about where it's used and how to navigate to it. + * Replaces TypeScript findModuleUsages() function. + */ +private fun writeModuleUsageIndex( + gen: JsonGenerator, + allModuleSets: List, + products: List +) { + // Build comprehensive usage index + val usageIndex = mutableMapOf() + + // Collect from module sets + for ((moduleSet, location, sourceFile) in allModuleSets) { + for (module in moduleSet.modules) { + val info = usageIndex.getOrPut(module.name) { ModuleUsageInfo() } + info.moduleSets.add(ModuleSetReference(moduleSet.name, location, sourceFile)) + } + } + + // Collect from products + for (product in products) { + val contentSpec = product.contentSpec ?: continue + + // Collect all modules used by this product + val allModulesInProduct = mutableSetOf() + + // From module sets + for (msRef in contentSpec.moduleSets) { + val modulesFromSet = collectAllModuleNamesFromSet(allModuleSets.map { it.moduleSet }, msRef.moduleSet.name) + allModulesInProduct.addAll(modulesFromSet) + } + + // From additional modules + for (module in contentSpec.additionalModules) { + allModulesInProduct.add(module.name) + } + + // Add to index + for (moduleName in allModulesInProduct) { + val info = usageIndex.getOrPut(moduleName) { ModuleUsageInfo() } + info.products.add(ProductReference(product.name, product.sourceFile)) + } + } + + // Write JSON + gen.writeObjectFieldStart("modules") + for ((moduleName, info) in usageIndex.entries.sortedBy { it.key }) { + gen.writeObjectFieldStart(moduleName) + + // Module sets that include this module + gen.writeArrayFieldStart("moduleSets") + for (msRef in info.moduleSets.sortedBy { it.name }) { + gen.writeStartObject() + gen.writeStringField("name", msRef.name) + gen.writeStringField("location", msRef.location) + gen.writeStringField("sourceFile", msRef.sourceFile) + gen.writeEndObject() + } + gen.writeEndArray() + + // Products that use this module + gen.writeArrayFieldStart("products") + for (prodRef in info.products.sortedBy { it.name }) { + gen.writeStartObject() + gen.writeStringField("name", prodRef.name) + gen.writeStringField("sourceFile", prodRef.sourceFile) + gen.writeEndObject() + } + gen.writeEndArray() + + gen.writeEndObject() + } + gen.writeEndObject() +} + +/** + * Helper data classes for module usage index. + */ +private data class ModuleUsageInfo( + val moduleSets: MutableList = mutableListOf(), + val products: MutableList = mutableListOf() +) + +private data class ModuleSetReference( + val name: String, + val location: String, + val sourceFile: String +) + +private data class ProductReference( + val name: String, + val sourceFile: String +) + +/** + * Recursively collects all module names from a module set and its nested sets. + * Helper function for module distribution and usage analysis. + */ +private fun collectAllModuleNamesFromSet( + moduleSets: List, + setName: String, + visited: MutableSet = mutableSetOf() +): Set { + if (setName in visited) return emptySet() + visited.add(setName) + + val moduleSet = moduleSets.firstOrNull { it.name == setName } ?: return emptySet() + + val allModules = moduleSet.modules.map { it.name }.toMutableSet() + + // Recursively collect from nested sets + for (nestedSet in moduleSet.nestedSets) { + allModules.addAll(collectAllModuleNamesFromSet(moduleSets, nestedSet.name, visited)) + } + + return allModules +} + +/** + * Violation when a community product uses ultimate modules. + */ +private data class CommunityProductViolation( + val product: String, + val productFile: String, + val moduleSet: String, + val moduleSetFile: String, + val ultimateModules: List, + val communityModulesCount: Int, + val unknownModulesCount: Int, + val totalModulesCount: Int +) + +/** + * Violation when a module set is in the wrong location (community vs ultimate). + */ +private data class ModuleSetLocationViolation( + val moduleSet: String, + val file: String, + val issue: String, // "community_contains_ultimate" or "ultimate_contains_only_community" + val ultimateModules: List? = null, + val communityModules: List? = null, + val communityModulesCount: Int? = null, + val ultimateModulesCount: Int? = null, + val unknownModulesCount: Int, + val suggestion: String +) + +/** + * Similarity between two products based on module set overlap. + * Used for identifying merge candidates and refactoring opportunities. + */ +private data class ProductSimilarityPair( + val product1: String, + val product2: String, + val similarity: Double, + val moduleSetSimilarity: Double, + val sharedModuleSets: List, + val uniqueToProduct1: List, + val uniqueToProduct2: List +) + +/** + * Overlap between two module sets. + * Correctly identifies intentional nested set inclusions vs actual duplications. + * Intentional nesting (e.g., libraries includes libraries.core) is filtered out. + */ +private data class ModuleSetOverlap( + val moduleSet1: String, + val moduleSet2: String, + val location1: String, + val location2: String, + val relationship: String, // "overlap", "subset", "superset" + val overlapPercent: Int, + val sharedModules: Int, + val totalModules1: Int, + val totalModules2: Int, + val recommendation: String +) + +/** + * Suggestion for module set unification (merge, inline, factor, split). + * Generated by analyzing overlap, product similarity, and module set usage patterns. + */ +private data class UnificationSuggestion( + val priority: String, // "high", "medium", "low" + val strategy: String, // "merge", "inline", "factor", "split" + val type: String?, // For merge: "subset", "superset", "high-overlap" + val moduleSet: String?, // For inline/split: single module set + val moduleSet1: String?, // For merge: first module set + val moduleSet2: String?, // For merge: second module set + val products: List?, // For factor: products with shared sets + val sharedModuleSets: List?, // For factor: shared module sets + val reason: String, + val impact: Map +) + +/** + * Impact analysis result for merging, moving, or inlining module sets. + * Used to assess safety and predict consequences before refactoring. + */ +private data class MergeImpactResult( + val operation: String, // "merge", "move", "inline" + val sourceSet: String, + val targetSet: String?, + val productsUsingSource: List, + val productsUsingTarget: List, + val productsThatWouldChange: List, + val sizeImpact: Map, + val violations: List>, + val recommendation: String, + val safe: Boolean +) + +/** + * Validates that community products don't use ultimate modules. + * This enforces the architectural constraint that community products must only use community modules. + * + * @param products List of all products + * @param allModuleSets List of all module sets with metadata + * @param moduleLocations Map of module names to their locations (from .idea/modules.xml) + * @param projectRoot Project root path for constructing file paths + * @return List of violations + */ +private fun validateCommunityProducts( + products: List, + allModuleSets: List, + moduleLocations: Map, + projectRoot: Path +): List { + val violations = mutableListOf() + + for (product in products) { + if (product.pluginXmlPath == null || product.contentSpec == null) continue + + // Check if product is in community + val productFile = projectRoot.resolve(product.pluginXmlPath).toString() + val isCommunityProduct = productFile.contains("/community/") + + if (isCommunityProduct) { + // Check each module set used by this product + for (msRef in product.contentSpec.moduleSets) { + val setName = msRef.moduleSet.name + val msEntry = allModuleSets.firstOrNull { it.moduleSet.name == setName } ?: continue + + // Get ALL modules including those from nested sets (pre-calculated) + val allModules = collectAllModuleNamesFromSet(allModuleSets.map { it.moduleSet }, setName) + + // Find ultimate modules + val ultimateModules = mutableListOf() + var communityModulesCount = 0 + var unknownModulesCount = 0 + + for (moduleName in allModules) { + val locationInfo = moduleLocations[moduleName] + when (locationInfo?.location) { + "ultimate" -> ultimateModules.add(moduleName) + "community" -> communityModulesCount++ + else -> unknownModulesCount++ + } + } + + if (ultimateModules.isNotEmpty()) { + violations.add(CommunityProductViolation( + product = product.name, + productFile = productFile, + moduleSet = setName, + moduleSetFile = projectRoot.resolve(msEntry.sourceFile).toString(), + ultimateModules = ultimateModules, + communityModulesCount = communityModulesCount, + unknownModulesCount = unknownModulesCount, + totalModulesCount = allModules.size + )) + } + } + } + } + + return violations +} + +/** + * Validates that module sets are in correct locations (community vs ultimate). + * Reports violations where: + * - A module set in community/ contains ultimate modules + * - A module set in ultimate/ contains only community modules + * + * @param allModuleSets List of all module sets with metadata + * @param moduleLocations Map of module names to their locations (from .idea/modules.xml) + * @param projectRoot Project root path for constructing file paths + * @return List of violations + */ +private fun validateModuleSetLocations( + allModuleSets: List, + moduleLocations: Map, + projectRoot: Path +): List { + val violations = mutableListOf() + + for (msEntry in allModuleSets) { + val ms = msEntry.moduleSet + val setFile = projectRoot.resolve(msEntry.sourceFile).toString() + val isInCommunity = setFile.contains("/community/") + val isInUltimate = setFile.contains("/ultimate/") + + // Count community vs ultimate modules in this set + val ultimateModules = mutableListOf() + val communityModules = mutableListOf() + var unknownCount = 0 + + for (module in ms.modules) { + val locationInfo = moduleLocations[module.name] + when (locationInfo?.location) { + "ultimate" -> ultimateModules.add(module.name) + "community" -> communityModules.add(module.name) + else -> unknownCount++ + } + } + + // Violation: Module set in community/ contains ultimate modules + if (isInCommunity && ultimateModules.isNotEmpty()) { + violations.add(ModuleSetLocationViolation( + moduleSet = ms.name, + file = setFile, + issue = "community_contains_ultimate", + ultimateModules = ultimateModules, + communityModulesCount = communityModules.size, + unknownModulesCount = unknownCount, + suggestion = "Move to ultimate/platform-ultimate/resources/META-INF/" + )) + } + + // Warning: Module set in ultimate/ contains only community modules + if (isInUltimate && ultimateModules.isEmpty() && communityModules.isNotEmpty()) { + violations.add(ModuleSetLocationViolation( + moduleSet = ms.name, + file = setFile, + issue = "ultimate_contains_only_community", + ultimateModulesCount = 0, + communityModules = communityModules, + unknownModulesCount = unknownCount, + suggestion = "Consider if this should be in community/" + )) + } + } + + return violations +} + +/** + * Recursively collects all nested set names (direct + transitive) from a module set. + * + * For example, if essential includes libraries, and libraries includes libraries.core, + * this returns {"libraries", "libraries.core", ...} for essential. + * + * @param allModuleSets All module sets to search in + * @param startSetName The module set to start collecting from + * @param visited Set of already visited module sets to prevent infinite recursion + * @return Set of all nested set names (direct and transitive) + */ +private fun collectAllNestedSetNames( + allModuleSets: List, + startSetName: String, + visited: MutableSet = mutableSetOf() +): Set { + if (visited.contains(startSetName)) return emptySet() + visited.add(startSetName) + + val startSet = allModuleSets.firstOrNull { it.name == startSetName } ?: return emptySet() + val result = mutableSetOf() + + for (nestedSet in startSet.nestedSets) { + result.add(nestedSet.name) + // Recursively collect nested sets from this nested set + result.addAll(collectAllNestedSetNames(allModuleSets, nestedSet.name, visited)) + } + + return result +} + +/** + * Detects overlapping or redundant module sets. + * CRITICAL FIX: Filters out intentional nested set inclusions (e.g., libraries ⊃ libraries.core). + * ENHANCED: Now checks TRANSITIVE nested relationships (e.g., essential → libraries → libraries.core). + * Only reports actual duplications, not designed composition patterns. + * + * @param allModuleSets List of all module sets with metadata + * @param minOverlapPercent Minimum overlap percentage (0-100) to include in results + * @return List of overlapping module set pairs sorted by overlap percentage (descending) + */ +private fun detectModuleSetOverlap( + allModuleSets: List, + minOverlapPercent: Int = 50 +): List { + val overlaps = mutableListOf() + val moduleSetsList = allModuleSets.map { it.moduleSet } + + for (i in allModuleSets.indices) { + for (j in i + 1 until allModuleSets.size) { + val ms1 = allModuleSets[i] + val ms2 = allModuleSets[j] + + // ✅ CRITICAL FIX: Skip if one explicitly includes the other as a nested set + // This prevents false positives like "libraries overlaps with libraries.core" + // when libraries explicitly includes libraries.core by design + // + // ✅ ENHANCED: Now checks TRANSITIVE relationships too! + // Example: essential → libraries → libraries.core + // This prevents false positive for "essential overlaps with libraries.core" + val ms1AllNestedSetNames = collectAllNestedSetNames(moduleSetsList, ms1.moduleSet.name) + val ms2AllNestedSetNames = collectAllNestedSetNames(moduleSetsList, ms2.moduleSet.name) + + if (ms1AllNestedSetNames.contains(ms2.moduleSet.name) || + ms2AllNestedSetNames.contains(ms1.moduleSet.name)) { + continue // Intentional composition via nesting (direct or transitive), not duplication! + } + + // Calculate overlap based on direct modules only (not nested) + val modules1 = ms1.moduleSet.modules.map { it.name }.toSet() + val modules2 = ms2.moduleSet.modules.map { it.name }.toSet() + + val intersection = modules1.intersect(modules2) + if (intersection.isEmpty()) continue + + val union = modules1.union(modules2) + val overlapPercent = (intersection.size * 100) / union.size + + if (overlapPercent >= minOverlapPercent) { + val relationship = when { + intersection.size == modules1.size -> "subset" // ms1 ⊂ ms2 + intersection.size == modules2.size -> "superset" // ms1 ⊃ ms2 + else -> "overlap" + } + + overlaps.add(ModuleSetOverlap( + moduleSet1 = ms1.moduleSet.name, + moduleSet2 = ms2.moduleSet.name, + location1 = ms1.location, + location2 = ms2.location, + relationship = relationship, + overlapPercent = overlapPercent, + sharedModules = intersection.size, + totalModules1 = modules1.size, + totalModules2 = modules2.size, + recommendation = generateOverlapRecommendation(ms1, ms2, relationship, overlapPercent) + )) + } + } + } + + return overlaps.sortedByDescending { it.overlapPercent } +} + +/** + * Generates recommendation for overlapping module sets. + */ +private fun generateOverlapRecommendation( + ms1: ModuleSetMetadata, + ms2: ModuleSetMetadata, + relationship: String, + overlapPercent: Int +): String { + return when (relationship) { + "subset" -> "${ms1.moduleSet.name} is fully contained in ${ms2.moduleSet.name}. Consider removing ${ms1.moduleSet.name}." + "superset" -> "${ms2.moduleSet.name} is fully contained in ${ms1.moduleSet.name}. Consider removing ${ms2.moduleSet.name}." + else -> if (overlapPercent >= 80) { + "High overlap ($overlapPercent%). Review if modules should be reorganized." + } else { + "Moderate overlap ($overlapPercent%). Consider extracting shared modules." + } + } +} + +/** + * Analyzes similarity between products based on module set overlap. + * Used to identify products with similar compositions for potential refactoring. + * + * @param products List of all products + * @param similarityThreshold Minimum similarity (0.0 to 1.0) to include in results + * @return List of similar product pairs sorted by similarity (descending) + */ +private fun analyzeProductSimilarity( + products: List, + similarityThreshold: Double = 0.7 +): List { + val pairs = mutableListOf() + val productsWithContent = products.filter { it.contentSpec != null } + + for (i in productsWithContent.indices) { + for (j in i + 1 until productsWithContent.size) { + val p1 = productsWithContent[i] + val p2 = productsWithContent[j] + + val sets1 = p1.contentSpec!!.moduleSets.map { it.moduleSet.name }.toSet() + val sets2 = p2.contentSpec!!.moduleSets.map { it.moduleSet.name }.toSet() + + val shared = sets1.intersect(sets2) + val union = sets1.union(sets2) + val similarity = if (union.isNotEmpty()) shared.size.toDouble() / union.size else 0.0 + + if (similarity >= similarityThreshold) { + pairs.add(ProductSimilarityPair( + product1 = p1.name, + product2 = p2.name, + similarity = similarity, + moduleSetSimilarity = similarity, + sharedModuleSets = shared.toList().sorted(), + uniqueToProduct1 = sets1.minus(sets2).toList().sorted(), + uniqueToProduct2 = sets2.minus(sets1).toList().sorted() + )) + } + } + } + + return pairs.sortedByDescending { it.similarity } +} + +/** + * Suggests module set unification opportunities based on overlap, similarity, and usage patterns. + * + * Strategies: + * - merge: Combine overlapping module sets (especially subsets/supersets) + * - inline: Inline rarely-used small module sets directly into products + * - factor: Extract common patterns from similar products + * - split: Split oversized module sets for better maintainability + * + * @param allModuleSets All module sets with metadata + * @param products All products + * @param overlaps Pre-calculated module set overlaps + * @param similarityPairs Pre-calculated product similarity pairs + * @param maxSuggestions Maximum number of suggestions to return + * @param strategy Filter by strategy: "merge", "inline", "factor", "split", or "all" + * @return List of suggestions sorted by priority + */ +private fun suggestModuleSetUnification( + allModuleSets: List, + products: List, + overlaps: List, + similarityPairs: List, + maxSuggestions: Int = 10, + strategy: String = "all" +): List { + val suggestions = mutableListOf() + + // Strategy 1: Merge overlapping module sets + if (strategy == "merge" || strategy == "all") { + for (overlap in overlaps) { + if (overlap.relationship == "subset" || overlap.relationship == "superset") { + suggestions.add(UnificationSuggestion( + priority = "high", + strategy = "merge", + type = overlap.relationship, + moduleSet = null, + moduleSet1 = overlap.moduleSet1, + moduleSet2 = overlap.moduleSet2, + products = null, + sharedModuleSets = null, + reason = overlap.recommendation, + impact = mapOf( + "moduleSetsSaved" to 1, + "overlapPercent" to overlap.overlapPercent + ) + )) + } else if (overlap.overlapPercent >= 80) { + suggestions.add(UnificationSuggestion( + priority = "medium", + strategy = "merge", + type = "high-overlap", + moduleSet = null, + moduleSet1 = overlap.moduleSet1, + moduleSet2 = overlap.moduleSet2, + products = null, + sharedModuleSets = null, + reason = overlap.recommendation, + impact = mapOf("overlapPercent" to overlap.overlapPercent) + )) + } + } + } + + // Strategy 2: Find rarely-used module sets (inline candidates) + if (strategy == "inline" || strategy == "all") { + for (msEntry in allModuleSets) { + val usedByProducts = products.filter { p -> + p.contentSpec?.moduleSets?.any { it.moduleSet.name == msEntry.moduleSet.name } == true + } + + if (usedByProducts.size <= 1 && msEntry.moduleSet.modules.size <= 5) { + suggestions.add(UnificationSuggestion( + priority = "low", + strategy = "inline", + type = null, + moduleSet = msEntry.moduleSet.name, + moduleSet1 = null, + moduleSet2 = null, + products = null, + sharedModuleSets = null, + reason = "Used by only ${usedByProducts.size} product(s) and contains only ${msEntry.moduleSet.modules.size} modules. Consider inlining into the product directly.", + impact = mapOf( + "moduleSetsSaved" to 1, + "moduleCount" to msEntry.moduleSet.modules.size, + "affectedProducts" to usedByProducts.map { it.name } + ) + )) + } + } + } + + // Strategy 3: Find common patterns (factoring opportunities) + if (strategy == "factor" || strategy == "all") { + for (pair in similarityPairs) { + if (pair.sharedModuleSets.size >= 3) { + suggestions.add(UnificationSuggestion( + priority = "medium", + strategy = "factor", + type = null, + moduleSet = null, + moduleSet1 = null, + moduleSet2 = null, + products = listOf(pair.product1, pair.product2), + sharedModuleSets = pair.sharedModuleSets, + reason = "Products ${pair.product1} and ${pair.product2} share ${pair.sharedModuleSets.size} module sets (${(pair.similarity * 100).toInt()}% similarity). Consider creating a common base.", + impact = mapOf( + "similarity" to pair.similarity, + "sharedModuleSets" to pair.sharedModuleSets.size + ) + )) + } + } + } + + // Strategy 4: Split large module sets + if (strategy == "split" || strategy == "all") { + for (msEntry in allModuleSets) { + if (msEntry.moduleSet.modules.size > 200) { + suggestions.add(UnificationSuggestion( + priority = "low", + strategy = "split", + type = null, + moduleSet = msEntry.moduleSet.name, + moduleSet1 = null, + moduleSet2 = null, + products = null, + sharedModuleSets = null, + reason = "Module set contains ${msEntry.moduleSet.modules.size} modules. Consider splitting into smaller, more focused sets for better maintainability.", + impact = mapOf("moduleCount" to msEntry.moduleSet.modules.size) + )) + } + } + } + + // Remove duplicates and sort by priority + val uniqueSuggestions = mutableListOf() + val seen = mutableSetOf() + for (suggestion in suggestions) { + val key = listOf(suggestion.strategy, suggestion.moduleSet1, suggestion.moduleSet2, suggestion.moduleSet).toString() + if (!seen.contains(key)) { + seen.add(key) + uniqueSuggestions.add(suggestion) + } + } + + // Sort by priority: high > medium > low + val priorityOrder = mapOf("high" to 3, "medium" to 2, "low" to 1) + uniqueSuggestions.sortByDescending { priorityOrder[it.priority] ?: 0 } + + return uniqueSuggestions.take(maxSuggestions) +} + +/** + * Analyzes the impact of merging, moving, or inlining module sets. + * Checks for violations, calculates size impact, and provides recommendations. + * + * @param sourceSet Source module set name + * @param targetSet Target module set name (null for inline operation) + * @param operation Operation type: "merge", "move", or "inline" + * @param allModuleSets All module sets with metadata + * @param products All products + * @return Impact analysis result + */ +private fun analyzeMergeImpact( + sourceSet: String, + targetSet: String?, + operation: String, + allModuleSets: List, + products: List +): MergeImpactResult? { + // Find source module set + val sourceEntry = allModuleSets.firstOrNull { it.moduleSet.name == sourceSet } + if (sourceEntry == null) { + return null // Error: source not found + } + + // Find target module set (if applicable) + var targetEntry: ModuleSetMetadata? = null + if (targetSet != null) { + targetEntry = allModuleSets.firstOrNull { it.moduleSet.name == targetSet } + if (targetEntry == null) { + return null // Error: target not found + } + } + + // Find products using source + val productsUsingSource = products.filter { p -> + p.contentSpec?.moduleSets?.any { it.moduleSet.name == sourceSet } == true + } + + // Find products using target + val productsUsingTarget = if (targetSet != null) { + products.filter { p -> + p.contentSpec?.moduleSets?.any { it.moduleSet.name == targetSet } == true + } + } else { + emptyList() + } + + // Calculate module changes + val sourceModules = sourceEntry.moduleSet.modules.map { it.name }.toSet() + val targetModules = if (targetEntry != null) { + targetEntry.moduleSet.modules.map { it.name }.toSet() + } else { + emptySet() + } + + val newModules = sourceModules.minus(targetModules) + val duplicateModules = sourceModules.intersect(targetModules) + + // Check for community/ultimate violations + val violations = mutableListOf>() + if (operation == "merge" && targetEntry != null) { + val sourceLocation = sourceEntry.location + val targetLocation = targetEntry.location + + if (sourceLocation == "ultimate" && targetLocation == "community") { + violations.add(mapOf( + "type" to "location", + "severity" to "error", + "message" to "Cannot merge ultimate module set \"$sourceSet\" into community module set \"$targetSet\"", + "fix" to "Move \"$targetSet\" to ultimate directory, or extract community modules from \"$sourceSet\"" + )) + } + + // Check if any community products would gain ultimate modules + val communityProductsUsingTarget = productsUsingTarget.filter { p -> + val productSets = p.contentSpec?.moduleSets?.map { it.moduleSet.name } ?: emptyList() + !productSets.contains("commercialIdeBase") && !productSets.contains("ide.ultimate") + } + + if (sourceLocation == "ultimate" && communityProductsUsingTarget.isNotEmpty()) { + violations.add(mapOf( + "type" to "community-uses-ultimate", + "severity" to "error", + "message" to "Merging ultimate set \"$sourceSet\" into \"$targetSet\" would expose ultimate modules to ${communityProductsUsingTarget.size} community products", + "affectedProducts" to communityProductsUsingTarget.map { it.name }, + "fix" to "Remove \"$targetSet\" from community products, or split ultimate modules from \"$sourceSet\"" + )) + } + } + + // Calculate size impact + val sizeImpact = mapOf( + "sourceModuleCount" to sourceModules.size, + "targetModuleCount" to targetModules.size, + "newModulesToTarget" to newModules.size, + "duplicateModules" to duplicateModules.size, + "resultingModuleCount" to targetModules.size + newModules.size + ) + + // Generate recommendation + val recommendation = when { + violations.isNotEmpty() -> "NOT RECOMMENDED: Operation would introduce violations. See violations for details." + operation == "merge" && duplicateModules.isNotEmpty() -> + "CAUTION: ${duplicateModules.size} modules already exist in target. Merge would create no duplicates, but review if modules serve the same purpose." + operation == "merge" && newModules.isNotEmpty() -> + "SAFE TO MERGE: Would add ${newModules.size} new modules to \"$targetSet\". ${productsUsingTarget.size} products using target would gain these modules." + operation == "inline" -> + "SAFE TO INLINE: ${productsUsingSource.size} products using \"$sourceSet\" would directly include ${sourceModules.size} modules instead." + else -> "Operation appears safe based on current analysis." + } + + return MergeImpactResult( + operation = operation, + sourceSet = sourceSet, + targetSet = targetSet, + productsUsingSource = productsUsingSource.map { it.name }, + productsUsingTarget = productsUsingTarget.map { it.name }, + productsThatWouldChange = if (operation == "merge") { + productsUsingTarget.map { it.name } + } else { + productsUsingSource.map { it.name } + }, + sizeImpact = sizeImpact, + violations = violations, + recommendation = recommendation, + safe = violations.isEmpty() + ) +} + +/** + * Writes community product validation violations to JSON. + */ +private fun writeCommunityProductViolations( + gen: JsonGenerator, + violations: List +) { + gen.writeArrayFieldStart("violations") + for (violation in violations) { + gen.writeStartObject() + gen.writeStringField("product", violation.product) + gen.writeStringField("productFile", violation.productFile) + gen.writeStringField("moduleSet", violation.moduleSet) + gen.writeStringField("moduleSetFile", violation.moduleSetFile) + + gen.writeArrayFieldStart("ultimateModules") + for (module in violation.ultimateModules) { + gen.writeString(module) + } + gen.writeEndArray() + + gen.writeNumberField("communityModulesCount", violation.communityModulesCount) + gen.writeNumberField("unknownModulesCount", violation.unknownModulesCount) + gen.writeNumberField("totalModulesCount", violation.totalModulesCount) + gen.writeEndObject() + } + gen.writeEndArray() + + // Summary + gen.writeObjectFieldStart("summary") + gen.writeNumberField("totalViolations", violations.size) + + gen.writeArrayFieldStart("affectedProducts") + for (product in violations.map { it.product }.distinct().sorted()) { + gen.writeString(product) + } + gen.writeEndArray() + + gen.writeArrayFieldStart("affectedModuleSets") + for (moduleSet in violations.map { it.moduleSet }.distinct().sorted()) { + gen.writeString(moduleSet) + } + gen.writeEndArray() + + gen.writeEndObject() +} + +/** + * Writes module set location validation violations to JSON. + */ +private fun writeModuleSetLocationViolations( + gen: JsonGenerator, + violations: List +) { + gen.writeArrayFieldStart("violations") + for (violation in violations) { + gen.writeStartObject() + gen.writeStringField("moduleSet", violation.moduleSet) + gen.writeStringField("file", violation.file) + gen.writeStringField("issue", violation.issue) + + if (violation.ultimateModules != null) { + gen.writeArrayFieldStart("ultimateModules") + for (module in violation.ultimateModules) { + gen.writeString(module) + } + gen.writeEndArray() + } + + if (violation.communityModules != null) { + gen.writeArrayFieldStart("communityModules") + for (module in violation.communityModules) { + gen.writeString(module) + } + gen.writeEndArray() + } + + if (violation.communityModulesCount != null) { + gen.writeNumberField("communityModulesCount", violation.communityModulesCount) + } + + if (violation.ultimateModulesCount != null) { + gen.writeNumberField("ultimateModulesCount", violation.ultimateModulesCount) + } + + gen.writeNumberField("unknownModulesCount", violation.unknownModulesCount) + gen.writeStringField("suggestion", violation.suggestion) + gen.writeEndObject() + } + gen.writeEndArray() + + // Summary + gen.writeObjectFieldStart("summary") + gen.writeNumberField("totalViolations", violations.size) + gen.writeNumberField("communityContainsUltimate", violations.count { it.issue == "community_contains_ultimate" }) + gen.writeNumberField("ultimateContainsOnlyCommunity", violations.count { it.issue == "ultimate_contains_only_community" }) + gen.writeEndObject() +} + +/** + * Writes product similarity analysis to JSON. + * Includes similar product pairs and summary statistics. + */ +private fun writeProductSimilarityAnalysis( + gen: JsonGenerator, + pairs: List, + threshold: Double +) { + gen.writeArrayFieldStart("pairs") + for (pair in pairs) { + gen.writeStartObject() + gen.writeStringField("product1", pair.product1) + gen.writeStringField("product2", pair.product2) + gen.writeNumberField("similarity", pair.similarity) + gen.writeNumberField("moduleSetSimilarity", pair.moduleSetSimilarity) + + gen.writeArrayFieldStart("sharedModuleSets") + for (setName in pair.sharedModuleSets) { + gen.writeString(setName) + } + gen.writeEndArray() + + gen.writeArrayFieldStart("uniqueToProduct1") + for (setName in pair.uniqueToProduct1) { + gen.writeString(setName) + } + gen.writeEndArray() + + gen.writeArrayFieldStart("uniqueToProduct2") + for (setName in pair.uniqueToProduct2) { + gen.writeString(setName) + } + gen.writeEndArray() + + gen.writeEndObject() + } + gen.writeEndArray() + + gen.writeNumberField("totalPairs", pairs.size) + gen.writeNumberField("threshold", threshold) + gen.writeStringField("summary", "Found ${pairs.size} product pairs with ≥${(threshold * 100).toInt()}% similarity") +} + +/** + * Writes module set overlap analysis to JSON. + * Includes overlapping module set pairs and summary statistics. + * Note: Intentional nested set inclusions are already filtered out during analysis. + */ +private fun writeModuleSetOverlapAnalysis( + gen: JsonGenerator, + overlaps: List, + minPercent: Int +) { + gen.writeArrayFieldStart("overlaps") + for (overlap in overlaps) { + gen.writeStartObject() + gen.writeStringField("moduleSet1", overlap.moduleSet1) + gen.writeStringField("moduleSet2", overlap.moduleSet2) + gen.writeStringField("location1", overlap.location1) + gen.writeStringField("location2", overlap.location2) + gen.writeStringField("relationship", overlap.relationship) + gen.writeNumberField("overlapPercent", overlap.overlapPercent) + gen.writeNumberField("sharedModules", overlap.sharedModules) + gen.writeNumberField("totalModules1", overlap.totalModules1) + gen.writeNumberField("totalModules2", overlap.totalModules2) + gen.writeStringField("recommendation", overlap.recommendation) + gen.writeEndObject() + } + gen.writeEndArray() + + gen.writeNumberField("count", overlaps.size) + gen.writeStringField("summary", "Found ${overlaps.size} module set pairs with ≥$minPercent% overlap (excluding intentional nesting)") +} + +/** + * Writes module set unification suggestions to JSON. + * Includes suggestions for merge, inline, factor, and split strategies. + */ +private fun writeUnificationSuggestions( + gen: JsonGenerator, + suggestions: List +) { + gen.writeArrayFieldStart("suggestions") + for (suggestion in suggestions) { + gen.writeStartObject() + gen.writeStringField("priority", suggestion.priority) + gen.writeStringField("strategy", suggestion.strategy) + + if (suggestion.type != null) { + gen.writeStringField("type", suggestion.type) + } + if (suggestion.moduleSet != null) { + gen.writeStringField("moduleSet", suggestion.moduleSet) + } + if (suggestion.moduleSet1 != null) { + gen.writeStringField("moduleSet1", suggestion.moduleSet1) + } + if (suggestion.moduleSet2 != null) { + gen.writeStringField("moduleSet2", suggestion.moduleSet2) + } + if (suggestion.products != null) { + gen.writeArrayFieldStart("products") + for (product in suggestion.products) { + gen.writeString(product) + } + gen.writeEndArray() + } + if (suggestion.sharedModuleSets != null) { + gen.writeArrayFieldStart("sharedModuleSets") + for (setName in suggestion.sharedModuleSets) { + gen.writeString(setName) + } + gen.writeEndArray() + } + + gen.writeStringField("reason", suggestion.reason) + + gen.writeObjectFieldStart("impact") + for ((key, value) in suggestion.impact) { + when (value) { + is Number -> gen.writeNumberField(key, value.toDouble()) + is String -> gen.writeStringField(key, value) + is List<*> -> { + gen.writeArrayFieldStart(key) + for (item in value) { + gen.writeString(item.toString()) + } + gen.writeEndArray() + } + } + } + gen.writeEndObject() + + gen.writeEndObject() + } + gen.writeEndArray() + + gen.writeNumberField("totalSuggestions", suggestions.size) + gen.writeStringField("summary", "Found ${suggestions.size} unification opportunities") +} + +/** + * Writes merge impact analysis to JSON. + * Includes products affected, size impact, violations, and recommendation. + */ +private fun writeMergeImpactAnalysis( + gen: JsonGenerator, + impact: MergeImpactResult +) { + gen.writeStringField("operation", impact.operation) + gen.writeStringField("sourceSet", impact.sourceSet) + if (impact.targetSet != null) { + gen.writeStringField("targetSet", impact.targetSet) + } + + gen.writeArrayFieldStart("productsUsingSource") + for (product in impact.productsUsingSource) { + gen.writeString(product) + } + gen.writeEndArray() + + gen.writeArrayFieldStart("productsUsingTarget") + for (product in impact.productsUsingTarget) { + gen.writeString(product) + } + gen.writeEndArray() + + gen.writeArrayFieldStart("productsThatWouldChange") + for (product in impact.productsThatWouldChange) { + gen.writeString(product) + } + gen.writeEndArray() + + gen.writeObjectFieldStart("sizeImpact") + for ((key, value) in impact.sizeImpact) { + gen.writeNumberField(key, value) + } + gen.writeEndObject() + + gen.writeArrayFieldStart("violations") + for (violation in impact.violations) { + gen.writeStartObject() + for ((key, value) in violation) { + when (value) { + is String -> gen.writeStringField(key, value) + is Number -> gen.writeNumberField(key, value.toDouble()) + is List<*> -> { + gen.writeArrayFieldStart(key) + for (item in value) { + gen.writeString(item.toString()) + } + gen.writeEndArray() + } + } + } + gen.writeEndObject() + } + gen.writeEndArray() + + gen.writeStringField("recommendation", impact.recommendation) + gen.writeBooleanField("safe", impact.safe) +} diff --git a/platform/build-scripts/product-dsl/src/ModuleSetXmlRenderer.kt b/platform/build-scripts/product-dsl/src/ModuleSetXmlRenderer.kt new file mode 100644 index 000000000000..2444e1b503b0 --- /dev/null +++ b/platform/build-scripts/product-dsl/src/ModuleSetXmlRenderer.kt @@ -0,0 +1,182 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.intellij.build.productLayout + +import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule + +/** + * Determines if a module set needs to be inlined (cannot use xi:include). + * Inlining is required when: + * - The module set has loading overrides for its direct modules + * - The module set contains nested sets that are overridden at top level + * + * @return true if the module set must be inlined, false if xi:include can be used + */ +internal fun shouldInlineModuleSet( + moduleSet: ModuleSet, + overrides: Map, + overriddenModuleSetNames: Set +): Boolean { + return overrides.isNotEmpty() || containsOverriddenNestedSet(moduleSet, overriddenModuleSetNames) +} + +/** + * Appends inlined module set content with direct modules and nested set processing. + * Used when xi:include cannot be used due to overrides or nested set conflicts. + */ +internal fun StringBuilder.appendInlinedModuleSet( + moduleSet: ModuleSet, + overrides: Map, + contentBlocks: List, + overriddenModuleSetNames: Set +) { + val hasOverrides = overrides.isNotEmpty() + val directBlock = contentBlocks.find { it.source == moduleSet.name } + + // Append direct modules if present + if (directBlock != null && directBlock.modules.isNotEmpty()) { + val label = if (hasOverrides) moduleSet.name else "${moduleSet.name} (selective inline)" + withEditorFold(this, " ", label) { + append(" \n") + + // Add explanatory comment + if (hasOverrides) { + val overriddenModules = overrides.entries + .sortedBy { it.key } + .joinToString(", ") { "${it.key}=${it.value.name.lowercase().replace('_', '-')}" } + append(" \n") + } + else { + val overriddenNested = findOverriddenNestedSetNames(moduleSet, overriddenModuleSetNames) + val nestedNames = overriddenNested.joinToString(", ") { "'$it'" } + append(" \n") + } + + // Append modules with effective overrides + for (module in directBlock.modules) { + val effectiveLoading = overrides[module.name] ?: module.loading + appendModuleLine(ModuleWithLoading(module.name, effectiveLoading), " ") + } + + append(" \n") + } + } + + // Process nested sets recursively + appendNestedModuleSets(moduleSet.nestedSets, contentBlocks, overriddenModuleSetNames) +} + +/** + * Appends nested module sets with appropriate handling (inline or xi:include). + */ +internal fun StringBuilder.appendNestedModuleSets( + nestedSets: List, + contentBlocks: List, + overriddenModuleSetNames: Set +) { + for (nestedSet in nestedSets) { + val nestedSetName = ModuleSetName(nestedSet.name) + + when { + nestedSetName in overriddenModuleSetNames -> { + // Skip - will be processed separately at top level with its overrides + append(" \n") + } + containsOverriddenNestedSet(nestedSet, overriddenModuleSetNames) -> { + // Recursively inline - contains overridden nested sets + appendModuleSetXml(nestedSet, emptyMap(), contentBlocks, overriddenModuleSetNames) + } + else -> { + // Safe to use xi:include + appendModuleSetInclude(nestedSet.name) + } + } + } +} + +/** + * Appends a simple xi:include directive for a module set. + */ +internal fun StringBuilder.appendModuleSetInclude(moduleSetName: String) { + append(" \n") +} + +/** + * Recursively generates XML content for a module set, applying selective inlining when necessary. + * + * When a module set has overrides or contains overridden nested sets, it cannot use xi:include + * (which would lose the overrides). Instead, we inline the direct modules and generate + * xi:include directives for non-overridden nested sets. + * + * @param moduleSet The module set to generate content for + * @param overrides Loading rule overrides for direct modules + * @param contentBlocks Pre-computed content blocks containing modules with applied rules + * @param overriddenModuleSetNames Names of module sets that are overridden at top-level + */ +internal fun StringBuilder.appendModuleSetXml( + moduleSet: ModuleSet, + overrides: Map, + contentBlocks: List, + overriddenModuleSetNames: Set +) { + if (shouldInlineModuleSet(moduleSet, overrides, overriddenModuleSetNames)) { + appendInlinedModuleSet(moduleSet, overrides, contentBlocks, overriddenModuleSetNames) + } + else { + appendModuleSetInclude(moduleSet.name) + } +} + +/** + * Appends a single module XML element with optional loading attribute. + */ +internal fun StringBuilder.appendModuleLine(moduleWithLoading: ModuleWithLoading, indent: String = " ") { + append("$indent on-demand) + append(" loading=\"${moduleWithLoading.loading.name.lowercase().replace('_', '-')}\"") + } + append("/>\n") +} + +/** + * Appends a content block with modules wrapped in editor fold. + */ +internal fun StringBuilder.appendContentBlock( + blockSource: String, + modules: List, + indent: String = " ", +) { + withEditorFold(sb = this, indent = indent, description = blockSource) { + append("$indent\n") + for (module in modules) { + appendModuleLine(module, "$indent ") + } + append("$indent\n") + } +} + +/** + * Appends module set loading strategy comment when there are overrides. + */ +internal fun StringBuilder.appendModuleSetsStrategyComment( + spec: ProductModulesContentSpec, + overriddenModuleSetNames: Set +) { + if (overriddenModuleSetNames.isEmpty()) return + + append(" \n") +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ProductModulesContentSpec.kt b/platform/build-scripts/product-dsl/src/ProductModulesContentSpec.kt similarity index 52% rename from platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ProductModulesContentSpec.kt rename to platform/build-scripts/product-dsl/src/ProductModulesContentSpec.kt index 5a2b9b6c4eaa..b54818aa4ad1 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ProductModulesContentSpec.kt +++ b/platform/build-scripts/product-dsl/src/ProductModulesContentSpec.kt @@ -4,6 +4,14 @@ package org.jetbrains.intellij.build.productLayout import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule +import kotlinx.serialization.Serializable + +/** + * Marker annotation for the product DSL to prevent implicit receiver scope leakage. + * This ensures that methods from outer DSL scopes are not accidentally accessible in nested blocks. + */ +@DslMarker +annotation class ProductDslMarker /** * Represents an XML include directive that references a resource within a module. @@ -13,6 +21,7 @@ import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule * @param ultimateOnly If true, this include is only processed in Ultimate builds (skipped in Community builds) * @param optional If true, this include is always generated with xi:fallback and never inlined (safe for files that may not exist) */ +@Serializable data class DeprecatedXmlInclude( @JvmField val moduleName: String, @JvmField val resourcePath: String, @@ -20,6 +29,52 @@ data class DeprecatedXmlInclude( @JvmField val optional: Boolean = false, ) +/** + * Tracks how a product spec is composed (e.g., via include(), moduleSet(), etc.). + * This enables tracing module origins and detecting redundancies. + */ +@Serializable +data class SpecComposition( + /** Type of composition (inline spec, module set reference, or deprecated XML include) */ + @JvmField val type: CompositionType, + /** Reference name (module set name or function name) if applicable */ + @JvmField val reference: String? = null, + /** Breadcrumb trail showing the composition path */ + @JvmField val path: List = emptyList(), + /** Source location where this composition was added (file:line) */ + @JvmField val sourceLocation: String? = null, +) + +/** + * Type of composition operation that added content to a product spec. + */ +@Serializable +enum class CompositionType { + /** include(spec) - embedded content from another ProductModulesContentSpec */ + INLINE_SPEC, + /** moduleSet(...) - reference to a module set */ + MODULE_SET_REF, + /** deprecatedInclude(...) - reference to an XML file */ + DEPRECATED_XML, + /** module() / embeddedModule() / requiredModule() - direct module addition */ + DIRECT_MODULE, + /** alias() - module alias addition */ + ALIAS, +} + +/** + * Metadata about a product spec's origin for traceability. + */ +@Serializable +data class SpecMetadata( + /** Name of the product or function that created this spec */ + @JvmField val name: String? = null, + /** Source file path relative to project root */ + @JvmField val sourceFile: String? = null, + /** Function name that created this spec */ + @JvmField val sourceFunction: String? = null, +) + /** * Specification for programmatically defining product content modules. * This allows products to specify module sets, individual modules, and XML includes in Kotlin code @@ -32,6 +87,7 @@ data class DeprecatedXmlInclude( * * @see org.jetbrains.intellij.build.ProductProperties.getProductContentDescriptor */ +@Serializable class ProductModulesContentSpec( /** * Product module aliases for `` declarations (e.g., "com.jetbrains.gateway", "com.intellij.modules.idea"). @@ -48,10 +104,10 @@ class ProductModulesContentSpec( @JvmField val deprecatedXmlIncludes: List, /** - * Module sets to include. Each set contains a named collection of modules. - * Module sets are processed in order and can overlap (duplicates are handled). + * Module sets to include with optional loading overrides. + * When a module set has overrides, it will be inlined in XML instead of using xi:include. */ - @JvmField val moduleSets: List, + @JvmField val moduleSets: List, /** * Additional individual modules to include beyond those from module sets. @@ -66,23 +122,36 @@ class ProductModulesContentSpec( @JvmField val excludedModules: Set, /** - * Loading attribute overrides for specific modules. - * Map of module name to loading mode (e.g., [ModuleLoadingRule.EMBEDDED]). - * This allows changing the loading mode of modules from module sets. + * Composition graph tracking how this spec was assembled. + * Records all include(), moduleSet(), and other composition operations. + * Used for analysis, tracing, and redundancy detection. */ - @JvmField val moduleLoadingOverrides: Map, + @JvmField val compositionGraph: List = emptyList(), + + /** + * Metadata about this spec's origin (product name, source file, function). + * Useful for debugging and tracing where a spec was created. + */ + @JvmField val metadata: SpecMetadata? = null, ) /** * DSL builder for creating ProductModulesContentSpec with reduced boilerplate. */ +@ProductDslMarker class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { private val productModuleAliases = mutableListOf() private val xmlIncludes = mutableListOf() - private val moduleSets = mutableListOf() + private val moduleSets = mutableListOf() private val additionalModules = mutableListOf() private val excludedModules = mutableSetOf() - private val loadingOverrides = mutableMapOf() + + // Composition tracking + private val compositionGraph = mutableListOf() + private val pathStack = mutableListOf() // Current path for nested compositions + + // Metadata for this spec + internal var metadata: SpecMetadata? = null /** * Add a product module alias for `` declaration. @@ -92,6 +161,47 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { */ fun alias(value: String) { productModuleAliases.add(value) + compositionGraph.add(SpecComposition( + type = CompositionType.ALIAS, + reference = value, + path = pathStack.toList(), + sourceLocation = null // TODO: capture if needed + )) + } + + /** + * Include another ProductModulesContentSpec, merging all its contents into this builder. + * This enables composition of product spec fragments for reuse across products. + * + * Example: + * ``` + * override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules { + * include(commonCapabilityAliases()) // include spec fragment with common aliases + * include(platformCommonIncludes()) // include spec fragment with deprecatedIncludes + * moduleSet(commercialIdeBase()) // include module set + * } + * ``` + * + * @param spec The ProductModulesContentSpec to merge into this builder + */ + fun include(spec: ProductModulesContentSpec) { + // Record composition before flattening (for analysis) + compositionGraph.add(SpecComposition( + type = CompositionType.INLINE_SPEC, + reference = spec.metadata?.name ?: spec.metadata?.sourceFunction, + path = pathStack.toList(), + sourceLocation = spec.metadata?.sourceFile + )) + + // Flatten content (existing behavior for backward compatibility) + productModuleAliases.addAll(spec.productModuleAliases) + xmlIncludes.addAll(spec.deprecatedXmlIncludes) + moduleSets.addAll(spec.moduleSets) + additionalModules.addAll(spec.additionalModules) + excludedModules.addAll(spec.excludedModules) + + // Also preserve the nested spec's composition graph for deep analysis + compositionGraph.addAll(spec.compositionGraph) } /** @@ -116,13 +226,48 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { */ fun deprecatedInclude(moduleName: String, resourcePath: String, ultimateOnly: Boolean = false, optional: Boolean = false) { xmlIncludes.add(DeprecatedXmlInclude(moduleName, resourcePath, ultimateOnly, optional)) + compositionGraph.add(SpecComposition( + type = CompositionType.DEPRECATED_XML, + reference = "$moduleName:$resourcePath", + path = pathStack.toList(), + sourceLocation = null + )) } /** - * Add a module set. + * Add a module set without loading overrides. */ fun moduleSet(set: ModuleSet) { - moduleSets.add(set) + addModuleSet(set = set, overrides = emptyMap()) + } + + /** + * Add a module set with loading overrides for specific modules. + * When overrides are provided, the module set will be inlined in the product XML + * instead of being referenced via xi:include. + * + * Example: + * ``` + * moduleSet(UltimateModuleSets.commercialIdeBase()) { + * overrideAsEmbedded("intellij.rd.platform") + * overrideAsEmbedded("intellij.rd.ui") + * overrideAsRequired("intellij.some.module") + * } + * ``` + */ + inline fun moduleSet(set: ModuleSet, block: ModuleLoadingOverrideBuilder.() -> Unit) { + addModuleSet(set, ModuleLoadingOverrideBuilder().apply(block).build()) + } + + @PublishedApi + internal fun addModuleSet(set: ModuleSet, overrides: Map) { + moduleSets.add(ModuleSetWithOverrides(set, overrides)) + compositionGraph.add(SpecComposition( + type = CompositionType.MODULE_SET_REF, + reference = set.name, + path = pathStack.toList(), + sourceLocation = null + )) } /** @@ -130,6 +275,12 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { */ fun module(name: String, loading: ModuleLoadingRule? = null) { additionalModules.add(ContentModule(name, loading)) + compositionGraph.add(SpecComposition( + type = CompositionType.DIRECT_MODULE, + reference = name, + path = pathStack.toList(), + sourceLocation = null + )) } /** @@ -137,6 +288,12 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { */ fun embeddedModule(name: String) { additionalModules.add(ContentModule(name, ModuleLoadingRule.EMBEDDED)) + compositionGraph.add(SpecComposition( + type = CompositionType.DIRECT_MODULE, + reference = name, + path = pathStack.toList(), + sourceLocation = null + )) } /** @@ -144,6 +301,12 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { */ fun requiredModule(name: String) { additionalModules.add(ContentModule(name, ModuleLoadingRule.REQUIRED)) + compositionGraph.add(SpecComposition( + type = CompositionType.DIRECT_MODULE, + reference = name, + path = pathStack.toList(), + sourceLocation = null + )) } /** @@ -153,13 +316,6 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { excludedModules.add(moduleName) } - /** - * Override the loading mode for a specific module. - */ - fun override(moduleName: String, loading: ModuleLoadingRule) { - loadingOverrides.put(moduleName, loading) - } - @PublishedApi internal fun build(): ProductModulesContentSpec { return ProductModulesContentSpec( @@ -168,7 +324,8 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { moduleSets = java.util.List.copyOf(moduleSets), additionalModules = java.util.List.copyOf(additionalModules), excludedModules = java.util.Set.copyOf(excludedModules), - moduleLoadingOverrides = java.util.Map.copyOf(loadingOverrides), + compositionGraph = java.util.List.copyOf(compositionGraph), + metadata = metadata, ) } } @@ -191,9 +348,8 @@ class ProductModulesContentSpecBuilder @PublishedApi internal constructor() { * // Individual modules * embeddedModule("com.example.additional") * - * // Exclusions and overrides + * // Exclusions * exclude("unwanted.module") - * override("some.module", ModuleLoadingRule.OPTIONAL) * } * } * ``` diff --git a/platform/build-scripts/product-dsl/src/ProductXmlRenderer.kt b/platform/build-scripts/product-dsl/src/ProductXmlRenderer.kt new file mode 100644 index 000000000000..f26813716326 --- /dev/null +++ b/platform/build-scripts/product-dsl/src/ProductXmlRenderer.kt @@ -0,0 +1,118 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.intellij.build.productLayout + +import com.intellij.openapi.util.JDOMUtil +import org.jetbrains.intellij.build.ModuleOutputProvider +import org.jetbrains.intellij.build.findFileInModuleLibraryDependencies +import org.jetbrains.intellij.build.findFileInModuleSources +import org.jetbrains.intellij.build.isModuleNameLikeFilename + +/** + * Appends XML header comments. + */ +internal fun StringBuilder.appendXmlHeader(generatorCommand: String, productPropertiesClass: String) { + append(" \n") + append(" \n") + append(" \n") +} + +/** + * Appends the opening tag with optional xi:include namespace. + */ +internal fun StringBuilder.appendOpeningTag( + spec: ProductModulesContentSpec, + inlineXmlIncludes: Boolean, + inlineModuleSets: Boolean +) { + // Determine if xi:include namespace is needed + val hasXmlIncludes = !inlineXmlIncludes && spec.deprecatedXmlIncludes.isNotEmpty() + val hasModuleSetIncludes = !inlineModuleSets && spec.moduleSets.isNotEmpty() + val needsXiNamespace = hasXmlIncludes || hasModuleSetIncludes + + if (needsXiNamespace) { + append("\n") + } + else { + append("\n") + } + + // Add id and name as child tags if PlatformLangPlugin.xml is not included + val includesPlatformLang = spec.deprecatedXmlIncludes.any { + it.resourcePath == "META-INF/PlatformLangPlugin.xml" || + it.resourcePath == "META-INF/JavaIdePlugin.xml" || + it.resourcePath == "META-INF/pycharm-core.xml" + } + + if (!includesPlatformLang) { + append(" com.intellij\n") + append(" IDEA CORE\n") + } +} + +/** + * Generates xi:include directives or inline content for deprecated XML includes. + */ +internal fun generateXIncludes( + spec: ProductModulesContentSpec, + moduleOutputProvider: ModuleOutputProvider, + inlineXmlIncludes: Boolean, + sb: StringBuilder, + isUltimateBuild: Boolean, +) { + for (include in spec.deprecatedXmlIncludes) { + // When inlining: skip ultimate-only xi-includes in Community builds + if (inlineXmlIncludes && include.ultimateOnly && !isUltimateBuild) { + continue + } + + // Find the module and file + val module = moduleOutputProvider.findModule(include.moduleName) + val resourcePath = include.resourcePath + if (module == null) { + if (include.ultimateOnly) { + error("Ultimate-only module '${include.moduleName}' not found in Ultimate build - this is a configuration error (referenced in xi:include for '$resourcePath')") + } + error("Module '${include.moduleName}' not found (referenced in xi:include for '$resourcePath')") + } + + val data = findFileInModuleSources(module, resourcePath)?.let { JDOMUtil.load(it) } + ?: findFileInModuleLibraryDependencies(module = module, relativePath = resourcePath)?.let { JDOMUtil.load(it) } + ?: error("Resource '$resourcePath' not found in module '${module.name}' sources or libraries (referenced in xi:include)") + + if (inlineXmlIncludes && !include.optional) { + withEditorFold(sb, " ", "Inlined from ${include.moduleName}/$resourcePath") { + // Inline the actual XML content + for (element in data.children) { + sb.append(JDOMUtil.write(element).prependIndent(" ")) + sb.append("\n") + } + } + sb.append("\n") + } + else { + // Generate xi:include with absolute path (resources are in /META-INF/... in jars) + // Wrap ultimate-only and optional xi-includes with xi:fallback for graceful handling + if (include.ultimateOnly || include.optional) { + sb.append(""" """) + sb.append("\n") + sb.append(""" """) + sb.append("\n") + sb.append(""" """) + sb.append("\n") + } + else { + sb.append(""" """) + sb.append("\n") + } + } + } +} + +/** + * Converts a resource path to an xi:include href path. + */ +internal fun resourcePathToXIncludePath(resourcePath: String): String { + return if (isModuleNameLikeFilename(resourcePath)) resourcePath else "/$resourcePath" +} diff --git a/platform/build-scripts/product-dsl/src/ValidationUtils.kt b/platform/build-scripts/product-dsl/src/ValidationUtils.kt new file mode 100644 index 000000000000..d5838527680c --- /dev/null +++ b/platform/build-scripts/product-dsl/src/ValidationUtils.kt @@ -0,0 +1,201 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.intellij.build.productLayout + +/** + * Formats a validation error message with consistent structure. + * + * @param title Error title (will be prefixed with ❌) + * @param details List of detail lines to include in the message body + * @param hint Optional hint text (will be prefixed with 💡 Hint:) + * @return Formatted error message string + */ +internal fun formatValidationError( + title: String, + details: List, + hint: String? = null +): String = buildString { + appendLine("❌ $title") + appendLine() + details.forEach { appendLine(it) } + if (hint != null) { + appendLine() + appendLine("💡 Hint: $hint") + } +} + +/** + * Validates that all overridden modules exist as direct modules in their respective module sets. + * Throws an error if invalid overrides are found. + * + * @param moduleSetWithOverrides The module set with overrides to validate + * @param spec The product modules specification (for excluded modules) + */ +internal fun validateModuleSetOverrides( + moduleSetWithOverrides: ModuleSetWithOverrides, + spec: ProductModulesContentSpec +) { + if (moduleSetWithOverrides.loadingOverrides.isEmpty()) return + + val directModules = getDirectModules(moduleSetWithOverrides.moduleSet, spec.excludedModules).map { it.name }.toSet() + val invalidOverrides = moduleSetWithOverrides.loadingOverrides.keys.filter { it !in directModules } + + if (invalidOverrides.isNotEmpty()) { + val details = buildList { + add("The following ${invalidOverrides.size} module(s) are not direct modules of this set:") + invalidOverrides.sorted().forEach { add(" ✗ $it") } + add("") + add("Note: You cannot override nested set modules.") + add("") + add("Available direct modules in '${moduleSetWithOverrides.moduleSet.name}':") + directModules.sorted().take(10).forEach { add(" ✓ $it") } + if (directModules.size > 10) { + add(" ... and ${directModules.size - 10} more") + } + } + val hint = """You can only override direct modules, not modules from nested sets. + To override modules from nested sets, reference the nested set directly: + + moduleSet(YourNestedSet()) { + overrideAsEmbedded("module.name") + }""" + error(formatValidationError( + "Invalid loading overrides for module set '${moduleSetWithOverrides.moduleSet.name}'", + details, + hint + )) + } +} + +/** + * Validates that no module appears in multiple module sets. + * Throws an error if duplicates are found. + * + * @param moduleToSets Map from module name to list of module set names containing it + */ +internal fun validateNoDuplicateModules(moduleToSets: Map>) { + val duplicates = moduleToSets.filterValues { it.size > 1 } + + if (duplicates.isNotEmpty()) { + val details = buildList { + for ((moduleName, sets) in duplicates.toSortedMap()) { + add(" ✗ Module '$moduleName' appears in:") + sets.sorted().forEach { add(" - $it") } + add(" → Suggested fix: Remove from: ${sets.sorted().drop(1).joinToString(", ")}") + add("") + } + add("📋 Each module must belong to exactly one module set.") + add(" Fix the module set definitions in:") + add(" • community/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/CommunityModuleSets.kt") + add(" • platform/buildScripts/src/productLayout/UltimateModuleSets.kt") + } + error(formatValidationError( + "ERROR: Duplicate modules found across ${duplicates.size} module set(s)", + details, + hint = null + )) + } +} + +/** + * Validates that a module alias is unique and records it. + * Throws an error if the alias is already defined. + * + * @param alias The alias to validate + * @param source The source defining this alias (e.g., "product level", "module set 'foo'") + * @param aliasToSource Map tracking where each alias is defined + */ +internal fun validateAndRecordAlias( + alias: String, + source: String, + aliasToSource: MutableMap +) { + val existing = aliasToSource.put(alias, source) + if (existing != null) { + val hint = if (source == "product level") { + "Module aliases must be unique across the entire product. Remove the duplicate alias declaration from your product specification." + } else { + """Module aliases must be unique across all module sets. + Either: + • Remove the alias from $source + • Rename the alias to something unique + • Remove the alias from $existing""" + } + error(formatValidationError( + "Duplicate module alias detected: '$alias'", + listOf( + " Already defined in: $existing", + " Attempted to redefine at: $source" + ), + hint + )) + } +} + +/** + * Validates that products don't reference redundant module sets. + * A module set is redundant if it's already nested inside another module set the product uses + * and the product doesn't apply any overrides to it. + * + * This validation ensures product specifications are correct and maintainable. + * Redundant module set references can lead to confusion and maintenance issues. + * + * @param allModuleSets All available module sets + * @param productSpecs List of products with their specifications to validate + * @throws IllegalStateException if any product has redundant module set references + */ +fun validateNoRedundantModuleSets( + allModuleSets: List, + productSpecs: List> +) { + // Build map of module set name -> nested set names + val moduleSetToNested = allModuleSets.associate { moduleSet -> + moduleSet.name to moduleSet.nestedSets.map { it.name }.toSet() + } + + val errors = mutableListOf() + + for ((productName, contentSpec) in productSpecs) { + if (contentSpec == null || contentSpec.moduleSets.isEmpty()) continue + + // Get module set names the product uses + val usedSets = contentSpec.moduleSets.map { it.moduleSet.name } + + // Check each module set for redundancy + for (moduleSetWithOverrides in contentSpec.moduleSets) { + val setName = moduleSetWithOverrides.moduleSet.name + + // Skip if this set has overrides (overrides make it non-redundant) + if (moduleSetWithOverrides.loadingOverrides.isNotEmpty()) { + continue + } + + // Check if this set is nested in any other set the product uses + for (otherSetName in usedSets) { + if (otherSetName == setName) continue + + val nestedInOther = moduleSetToNested[otherSetName] + if (nestedInOther != null && setName in nestedInOther) { + errors.add(" ✗ Product '$productName': module set '$setName' is redundant (already nested in '$otherSetName')") + } + } + } + } + + if (errors.isNotEmpty()) { + val hint = """Remove redundant module sets from product's getProductContentDescriptor() method. + + Example fix: + override fun getProductContentDescriptor() = productModules { + // moduleSet(ssh()) // ← REMOVE (already in ide.ultimate) + // moduleSet(rd.common()) // ← REMOVE (already in ide.ultimate) + moduleSet(ideUltimate()) // ← KEEP (includes ssh and rd.common) + }""" + error(formatValidationError( + "Product specification errors: Redundant module set references detected", + errors, + hint + )) + } +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/XmlGenerationUtils.kt b/platform/build-scripts/product-dsl/src/XmlGenerationUtils.kt similarity index 64% rename from platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/XmlGenerationUtils.kt rename to platform/build-scripts/product-dsl/src/XmlGenerationUtils.kt index aab2dcb98200..f6f55c5d1367 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/XmlGenerationUtils.kt +++ b/platform/build-scripts/product-dsl/src/XmlGenerationUtils.kt @@ -27,6 +27,7 @@ internal fun visitAllModuleSets(sets: List, visitor: (ModuleSet) -> U /** * Recursively collects all aliases from a module set and its nested sets. + * Collects the single `alias` field (for module set's own capability). * * @param moduleSet The module set to collect aliases from * @param result Accumulator for collecting aliases (avoids intermediate allocations) @@ -67,6 +68,31 @@ internal inline fun withEditorFold(sb: StringBuilder, indent: String, descriptio sb.append("$indent\n") } +/** + * Recursively collects all module names from a module set and its nested sets. + * + * @param moduleSet The module set to collect modules from + * @param excludedModules Set of module names to exclude + * @return Set of all module names found in the module set hierarchy + */ +internal fun collectAllModuleNames(moduleSet: ModuleSet, excludedModules: Set = emptySet()): Set { + val result = HashSet() + + fun collect(set: ModuleSet) { + for (module in set.modules) { + if (module.name !in excludedModules) { + result.add(module.name) + } + } + for (nestedSet in set.nestedSets) { + collect(nestedSet) + } + } + + collect(moduleSet) + return result +} + /** * Gets direct modules from a module set (excluding modules from nested sets and excluded modules). * @@ -101,4 +127,39 @@ internal fun getDirectModules(moduleSet: ModuleSet, excludedModules: Set } } return directModules +} + +/** + * Checks if a module set contains (directly or transitively) any nested set whose name is in the given set. + * Used to detect when a parent module set contains overridden nested sets. + */ +internal fun containsOverriddenNestedSet(moduleSet: ModuleSet, overriddenNames: Set): Boolean { + // Check direct nested sets + for (nestedSet in moduleSet.nestedSets) { + if (ModuleSetName(nestedSet.name) in overriddenNames) { + return true + } + // Check recursively + if (containsOverriddenNestedSet(nestedSet, overriddenNames)) { + return true + } + } + return false +} + +/** + * Finds all nested set names (directly or transitively) that are in the given set of names. + * Used for generating descriptive comments about which nested sets are overridden. + */ +internal fun findOverriddenNestedSetNames(moduleSet: ModuleSet, overriddenNames: Set): List { + val result = mutableListOf() + for (nestedSet in moduleSet.nestedSets) { + val nestedSetName = ModuleSetName(nestedSet.name) + if (nestedSetName in overriddenNames) { + result.add(nestedSetName) + } + // Check recursively + result.addAll(findOverriddenNestedSetNames(nestedSet, overriddenNames)) + } + return result } \ No newline at end of file diff --git a/platform/build-scripts/product-dsl/src/generator.kt b/platform/build-scripts/product-dsl/src/generator.kt new file mode 100644 index 000000000000..92f5f4276388 --- /dev/null +++ b/platform/build-scripts/product-dsl/src/generator.kt @@ -0,0 +1,192 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.intellij.build.productLayout + +import org.jetbrains.intellij.build.ModuleOutputProvider +import java.nio.file.Files +import java.nio.file.Path + +/** + * Main entry point for generating XML files for both module sets and products. + * This file orchestrates the generation process by calling specialized components for: + * - Building content blocks (ContentBlockBuilder.kt) + * - Validating specifications (ValidationUtils.kt) + * - Rendering XML content (ModuleSetXmlRenderer.kt, ProductXmlRenderer.kt) + */ + +/** + * Generates an XML file for a module set. + * Used to maintain backward compatibility with XML-based module set loading. + * + * @param moduleSet The module set to generate XML for + * @param outputDir The directory where the XML file will be written + * @param label Description label ("community" or "ultimate") for header generation + * @return Result containing file status and statistics + */ +fun generateModuleSetXml(moduleSet: ModuleSet, outputDir: Path, label: String): ModuleSetFileResult { + val fileName = "${MODULE_SET_PREFIX}${moduleSet.name}.xml" + val outputPath = outputDir.resolve(fileName) + + val buildResult = buildModuleSetXml(moduleSet, label) + + // determine change status + val status = when { + !Files.exists(outputPath) -> FileChangeStatus.CREATED + Files.readString(outputPath) == buildResult.xml -> FileChangeStatus.UNCHANGED + else -> FileChangeStatus.MODIFIED + } + + // Only write if changed + if (status != FileChangeStatus.UNCHANGED) { + Files.writeString(outputPath, buildResult.xml) + } + + return ModuleSetFileResult(fileName, status, buildResult.directModuleCount) +} + +/** + * Generates complete product plugin.xml file from programmatic specification. + * + * @param isUltimateBuild Whether this is an Ultimate build (vs. Community build) + * @return Result containing file status and statistics + */ +fun generateProductXml( + pluginXmlPath: Path, + spec: ProductModulesContentSpec, + productName: String, + productPropertiesClass: String, + moduleOutputProvider: ModuleOutputProvider, + projectRoot: Path, + isUltimateBuild: Boolean, +): ProductFileResult { + // Determine which generator to recommend based on plugin.xml file location + // Community products are under community/ directory, Ultimate products are not + val generatorCommand = (if (pluginXmlPath.toString().contains("/community/")) "CommunityModuleSets" else "UltimateModuleSets") + GENERATOR_SUFFIX + + // Build complete plugin.xml file + val buildResult = buildProductContentXml( + spec = spec, + moduleOutputProvider = moduleOutputProvider, + inlineXmlIncludes = false, + inlineModuleSets = false, + productPropertiesClass = productPropertiesClass, + generatorCommand = generatorCommand, + isUltimateBuild = isUltimateBuild + ) + + // Compare with existing file if it exists + val originalContent = Files.readString(pluginXmlPath) + val status = if (originalContent == buildResult.xml) { + FileChangeStatus.UNCHANGED + } + else { + FileChangeStatus.MODIFIED + } + + // Only write if changed + if (status != FileChangeStatus.UNCHANGED) { + Files.writeString(pluginXmlPath, buildResult.xml) + } + + // Calculate statistics using the contentBlocks from generation + val totalModules = buildResult.contentBlocks.sumOf { it.modules.size } + val relativePath = projectRoot.relativize(pluginXmlPath).toString() + + return ProductFileResult( + productName = productName, + relativePath = relativePath, + status = status, + includeCount = spec.deprecatedXmlIncludes.size, + contentBlockCount = buildResult.contentBlocks.size, + totalModules = totalModules + ) +} + +/** + * Builds XML content for programmatic product modules. + * Generates module alias, xi:include directives (or inlined content), and `` blocks for each module set. + */ +fun buildProductContentXml( + spec: ProductModulesContentSpec, + moduleOutputProvider: ModuleOutputProvider, + inlineXmlIncludes: Boolean, + inlineModuleSets: Boolean, + productPropertiesClass: String, + generatorCommand: String, + isUltimateBuild: Boolean, +): ProductContentBuildResult { + // Build content blocks, chain mapping, and collect module set aliases in single traversal + val (contentBlocks, moduleToSetChainMapping, moduleSetAliases) = + buildContentBlocksAndChainMapping(spec, collectModuleSetAliases = inlineModuleSets) + + val xml = buildString { + appendXmlHeader(generatorCommand, productPropertiesClass) + appendOpeningTag(spec, inlineXmlIncludes, inlineModuleSets) + + // Collect and validate product-level aliases, checking for conflicts with module set aliases + val validatedAliases = collectAndValidateAliases(spec, moduleSetAliases) + val aliasXml = buildModuleAliasesXml(validatedAliases) + if (aliasXml.isNotEmpty()) { + append(aliasXml) + append("\n") + } + + // Generate xi:include directives or inline content + if (spec.deprecatedXmlIncludes.isNotEmpty()) { + generateXIncludes(spec = spec, moduleOutputProvider = moduleOutputProvider, inlineXmlIncludes = inlineXmlIncludes, sb = this, isUltimateBuild = isUltimateBuild) + } + + // Generate module sets as xi:includes or inline content blocks + if (spec.moduleSets.isNotEmpty()) { + if (inlineModuleSets) { + // Generate single content block with all module sets inlined + append(" \n") + for ((index, block) in contentBlocks.withIndex()) { + if (block.source == ADDITIONAL_MODULES_BLOCK) continue // Skip additional modules, handle separately + withEditorFold(this, " ", block.source) { + for (module in block.modules) { + appendModuleLine(module, " ") + } + } + // Add blank line between sections for readability (except after last block) + if (index < contentBlocks.size - 1) { + append("\n") + } + } + append(" \n") + } + else { + // Build set of module set names that are referenced at top-level WITH overrides + // These cannot be brought in via `xi:include` from parent sets (would lose overrides) + val overriddenModuleSetNames = spec.moduleSets + .filter { it.hasOverrides } + .map { ModuleSetName(it.moduleSet.name) } + .toSet() + + appendModuleSetsStrategyComment(spec, overriddenModuleSetNames) + + // Generate content for each top-level module set + for (moduleSetWithOverrides in spec.moduleSets) { + appendModuleSetXml( + moduleSetWithOverrides.moduleSet, + moduleSetWithOverrides.loadingOverrides, + contentBlocks, + overriddenModuleSetNames + ) + } + } + } + + // Handle additional modules separately (they don't have XML files, inline them) + val additionalBlock = contentBlocks.firstOrNull { it.source == ADDITIONAL_MODULES_BLOCK } + if (additionalBlock != null) { + appendContentBlock(additionalBlock.source, additionalBlock.modules) + } + + // Closing tag + append("\n") + } + + return ProductContentBuildResult(xml = xml, contentBlocks = contentBlocks, moduleToSetChainMapping = moduleToSetChainMapping) +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/generatorStats.kt b/platform/build-scripts/product-dsl/src/generatorStats.kt similarity index 100% rename from platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/generatorStats.kt rename to platform/build-scripts/product-dsl/src/generatorStats.kt diff --git a/platform/build-scripts/product-dsl/testSrc/ProductDslGeneratorTest.kt b/platform/build-scripts/product-dsl/testSrc/ProductDslGeneratorTest.kt new file mode 100644 index 000000000000..feca3034d522 --- /dev/null +++ b/platform/build-scripts/product-dsl/testSrc/ProductDslGeneratorTest.kt @@ -0,0 +1,161 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +/** + * Tests for generator.kt helper functions. + */ +class ProductDslGeneratorTest { + // Test fixtures + private fun createSimpleModuleSet(name: String, vararg moduleNames: String): ModuleSet { + return ModuleSet( + name = name, + modules = moduleNames.map { ContentModule(it) } + ) + } + + private fun createNestedModuleSet( + name: String, + moduleNames: List, + nestedSets: List, + ): ModuleSet { + return ModuleSet( + name = name, + modules = moduleNames.map { ContentModule(it) }, + nestedSets = nestedSets + ) + } + + // Tests for containsOverriddenNestedSet() + + @Test + fun `containsOverriddenNestedSet detects direct nested override`() { + val overriddenSet = createSimpleModuleSet("overridden", "mod.a") + val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(overriddenSet)) + + val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("overridden"))) + + assertThat(result).isTrue() + } + + @Test + fun `containsOverriddenNestedSet detects deeply nested override`() { + val deeplyNested = createSimpleModuleSet("deep", "mod.a") + val middleNested = createNestedModuleSet("middle", listOf("mod.b"), listOf(deeplyNested)) + val parentSet = createNestedModuleSet("parent", listOf("mod.c"), listOf(middleNested)) + + val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("deep"))) + + assertThat(result).isTrue() + } + + @Test + fun `containsOverriddenNestedSet returns false when no overrides`() { + val nestedSet = createSimpleModuleSet("nested", "mod.a") + val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet)) + + val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("someOtherSet"))) + + assertThat(result).isFalse() + } + + @Test + fun `containsOverriddenNestedSet returns false for empty override set`() { + val nestedSet = createSimpleModuleSet("nested", "mod.a") + val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet)) + + val result = containsOverriddenNestedSet(parentSet, emptySet()) + + assertThat(result).isFalse() + } + + @Test + fun `containsOverriddenNestedSet returns false for module set with no nested sets`() { + val parentSet = createSimpleModuleSet("parent", "mod.a", "mod.b") + + val result = containsOverriddenNestedSet(parentSet, setOf(ModuleSetName("someSet"))) + + assertThat(result).isFalse() + } + + // Tests for findOverriddenNestedSetNames() + + @Test + fun `findOverriddenNestedSetNames finds direct overridden set`() { + val overridden1 = createSimpleModuleSet("overridden1", "mod.a") + val overridden2 = createSimpleModuleSet("overridden2", "mod.b") + val notOverridden = createSimpleModuleSet("notOverridden", "mod.c") + val parentSet = createNestedModuleSet( + "parent", + listOf("mod.d"), + listOf(overridden1, notOverridden, overridden2) + ) + + val result = findOverriddenNestedSetNames(parentSet, setOf(ModuleSetName("overridden1"), ModuleSetName("overridden2"))) + + assertThat(result).containsExactlyInAnyOrder(ModuleSetName("overridden1"), ModuleSetName("overridden2")) + } + + @Test + fun `findOverriddenNestedSetNames finds all overridden sets recursively`() { + val deepOverridden = createSimpleModuleSet("deepOverridden", "mod.a") + val deepNormal = createSimpleModuleSet("deepNormal", "mod.b") + val middleOverridden = createNestedModuleSet( + "middleOverridden", + listOf("mod.c"), + listOf(deepOverridden, deepNormal) + ) + val middleNormal = createSimpleModuleSet("middleNormal", "mod.d") + val parentSet = createNestedModuleSet( + "parent", + listOf("mod.e"), + listOf(middleOverridden, middleNormal) + ) + + val result = findOverriddenNestedSetNames( + parentSet, + setOf(ModuleSetName("middleOverridden"), ModuleSetName("deepOverridden")) + ) + + assertThat(result).containsExactlyInAnyOrder(ModuleSetName("middleOverridden"), ModuleSetName("deepOverridden")) + } + + @Test + fun `findOverriddenNestedSetNames returns empty for no overrides`() { + val nestedSet = createSimpleModuleSet("nested", "mod.a") + val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet)) + + val result = findOverriddenNestedSetNames(parentSet, setOf(ModuleSetName("someOtherSet"))) + + assertThat(result).isEmpty() + } + + @Test + fun `findOverriddenNestedSetNames returns empty for empty override set`() { + val nestedSet = createSimpleModuleSet("nested", "mod.a") + val parentSet = createNestedModuleSet("parent", listOf("mod.b"), listOf(nestedSet)) + + val result = findOverriddenNestedSetNames(parentSet, emptySet()) + + assertThat(result).isEmpty() + } + + @Test + fun `findOverriddenNestedSetNames preserves order of discovery`() { + val nested1 = createSimpleModuleSet("nested1", "mod.a") + val nested2 = createSimpleModuleSet("nested2", "mod.b") + val nested3 = createSimpleModuleSet("nested3", "mod.c") + val parentSet = createNestedModuleSet( + "parent", + listOf("mod.d"), + listOf(nested1, nested2, nested3) + ) + + val result = findOverriddenNestedSetNames(parentSet, setOf(ModuleSetName("nested1"), ModuleSetName("nested2"), ModuleSetName("nested3"))) + + // Should be in order of traversal + assertThat(result).containsExactly(ModuleSetName("nested1"), ModuleSetName("nested2"), ModuleSetName("nested3")) + } +} diff --git a/platform/build-scripts/product-dsl/testSrc/ProductModulesContentSpecTest.kt b/platform/build-scripts/product-dsl/testSrc/ProductModulesContentSpecTest.kt new file mode 100644 index 000000000000..bf7ccfd65895 --- /dev/null +++ b/platform/build-scripts/product-dsl/testSrc/ProductModulesContentSpecTest.kt @@ -0,0 +1,453 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.jetbrains.intellij.build.ModuleOutputProvider +import org.jetbrains.jps.model.module.JpsModule +import org.junit.jupiter.api.Test +import java.nio.file.Path + +class ProductModulesContentSpecTest { + @Test + fun `test valid overrides for existing modules`() { + // Create a simple module set with some modules + val moduleSet = ModuleSet( + name = "testSet", + modules = listOf( + ContentModule("module.a"), + ContentModule("module.b"), + ContentModule("module.c") + ) + ) + + val spec = productModules { + moduleSet(moduleSet) { + overrideAsEmbedded("module.a") + overrideAsEmbedded("module.b") + } + } + + // This should not throw - all overridden modules exist + val result = buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + + assertThat(result.xml).contains("module.a") + assertThat(result.xml).contains("loading=\"embedded\"") + } + + @Test + fun `test invalid overrides for non-existent modules`() { + val moduleSet = ModuleSet( + name = "testSet", + modules = listOf( + ContentModule("module.a"), + ContentModule("module.b") + ) + ) + + val spec = productModules { + moduleSet(moduleSet) { + overrideAsEmbedded("module.a") + overrideAsEmbedded("module.nonexistent") + } + } + + // This should throw because module.nonexistent doesn't exist + assertThatThrownBy { + buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + } + .isInstanceOf(IllegalStateException::class.java) + .hasMessageContaining("Invalid loading overrides for module set 'testSet'") + .hasMessageContaining("module.nonexistent") + } + + @Test + fun `test cannot override modules from nested sets`() { + val nestedSet = ModuleSet( + name = "nested", + modules = listOf( + ContentModule("nested.module.a"), + ContentModule("nested.module.b") + ) + ) + + val parentSet = ModuleSet( + name = "parent", + modules = listOf( + ContentModule("parent.module.a") + ), + nestedSets = listOf(nestedSet) + ) + + val spec = productModules { + moduleSet(parentSet) { + // Trying to override nested module should fail + overrideAsEmbedded("parent.module.a") + overrideAsEmbedded("nested.module.a") + } + } + + // This should throw because nested.module.a is not a direct module of parent + assertThatThrownBy { + buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + } + .isInstanceOf(IllegalStateException::class.java) + .hasMessageContaining("Invalid loading overrides for module set 'parent'") + .hasMessageContaining("cannot override nested set modules") + .hasMessageContaining("nested.module.a") + } + + @Test + fun `test empty overrides work fine`() { + val moduleSet = ModuleSet( + name = "testSet", + modules = listOf( + ContentModule("module.a") + ) + ) + + val spec = productModules { + moduleSet(moduleSet) + } + + // This should not throw + val result = buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + + assertThat(result.xml).contains("module.a") + } + + @Test + fun `test multiple invalid overrides are all reported`() { + val moduleSet = ModuleSet( + name = "testSet", + modules = listOf( + ContentModule("module.a") + ) + ) + + val spec = productModules { + moduleSet(moduleSet) { + overrideAsEmbedded("module.nonexistent1") + overrideAsEmbedded("module.nonexistent2") + overrideAsEmbedded("module.a") + } + } + + // Should report both nonexistent modules + assertThatThrownBy { + buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + } + .isInstanceOf(IllegalStateException::class.java) + .hasMessageContaining("module.nonexistent1") + .hasMessageContaining("module.nonexistent2") + } + + @Test + fun `test overrides respect excluded modules`() { + val moduleSet = ModuleSet( + name = "testSet", + modules = listOf( + ContentModule("module.a"), + ContentModule("module.b") + ) + ) + + val spec = productModules { + moduleSet(moduleSet) { + overrideAsEmbedded("module.b") // module.b is excluded, so this override is invalid + } + exclude("module.b") + } + + // This should throw because module.b is excluded + assertThatThrownBy { + buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + } + .isInstanceOf(IllegalStateException::class.java) + .hasMessageContaining("Invalid loading overrides") + .hasMessageContaining("module.b") + } + + @Test + fun `test selective inlining with overrides generates correct XML`() { + val nestedSet = ModuleSet( + name = "nested", + modules = listOf( + ContentModule("nested.module.a"), + ContentModule("nested.module.b") + ) + ) + + val parentSet = ModuleSet( + name = "parent", + modules = listOf( + ContentModule("parent.module.a"), + ContentModule("parent.module.b") + ), + nestedSets = listOf(nestedSet) + ) + + val spec = productModules { + moduleSet(parentSet) { + overrideAsEmbedded("parent.module.a") + } + } + + // Test with inlineModuleSets = false (selective inlining mode) + val result = buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = false, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + + // Should inline parent modules with loading attributes + assertThat(result.xml).contains("parent.module.a") + assertThat(result.xml).contains("loading=\"embedded\"") + assertThat(result.xml).contains("parent.module.b") + + // Should generate xi:include for nested set + assertThat(result.xml).contains("") + } + + @Test + fun `test loading attributes are correctly applied in inlined mode`() { + val moduleSet = ModuleSet( + name = "testSet", + modules = listOf( + ContentModule("module.a"), + ContentModule("module.b"), + ContentModule("module.c") + ) + ) + + val spec = productModules { + moduleSet(moduleSet) { + overrideAsEmbedded("module.a") + overrideAsEmbedded("module.b") + } + } + + // Test with inlineModuleSets = true (full inlining mode) + val result = buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + + // Verify embedded modules have loading attribute + assertThat(result.xml).containsPattern("") + assertThat(result.xml).containsPattern("") + + // Verify non-overridden module does not have loading attribute + assertThat(result.xml).containsPattern("") + assertThat(result.xml).doesNotContain("module.c\" loading") + } + + @Test + fun `test nested set override prevents duplicate module entries`() { + // This test covers the real-world Rider scenario: + // Parent set (commercialIdeBase) contains nested set (rdCommon) + // Product also references rdCommon directly with overrides + // Expected: rdCommon modules appear ONCE with overrides, not twice + + val deeplyNestedSet = ModuleSet( + name = "rdCommon", + modules = listOf( + ContentModule("rd.module.a"), + ContentModule("rd.module.b"), + ContentModule("rd.module.c") + ) + ) + + val middleSet = ModuleSet( + name = "ideUltimate", + modules = listOf( + ContentModule("ide.module.a") + ), + nestedSets = listOf(deeplyNestedSet) + ) + + val parentSet = ModuleSet( + name = "commercialIdeBase", + modules = listOf( + ContentModule("commercial.module.a") + ), + nestedSets = listOf(middleSet) + ) + + val spec = productModules { + // Include parent (which contains rdCommon nested deeply) + moduleSet(parentSet) + + // Also reference rdCommon directly with overrides + moduleSet(deeplyNestedSet) { + overrideAsEmbedded("rd.module.a") + overrideAsEmbedded("rd.module.b") + } + } + + val result = buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = false, + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + + // Verify rdCommon modules appear exactly ONCE with loading attributes + // Use regex to count only actual module elements, not comments + val rdModuleACount = Regex("""") + assertThat(result.xml).contains("") + + // Verify parent and middle sets were selectively inlined (not xi:included) + assertThat(result.xml).contains("commercial.module.a") + assertThat(result.xml).contains("ide.module.a") + + // Verify no xi:include for sets containing overridden nested sets + assertThat(result.xml).doesNotContain("") + assertThat(result.xml).doesNotContain("") + } + + @Test + fun `test overrides preserved when module set nested and directly referenced with full inlining`() { + // This test covers the Rider ClassNotFoundException bug: + // When inlineModuleSets = true (full inlining mode), overrides are lost if: + // - A module set is nested inside another set + // - The same module set is also directly referenced with overrides + // Bug: Only the first encounter's overrides are kept due to processedSets deduplication + + val rdCommon = ModuleSet( + name = "rdCommon", + modules = listOf( + ContentModule("intellij.rd.platform"), + ContentModule("intellij.rd.ui") + ) + ) + + val commercialIdeBase = ModuleSet( + name = "commercialIdeBase", + modules = listOf( + ContentModule("commercial.module") + ), + nestedSets = listOf(rdCommon) + ) + + val spec = productModules { + moduleSet(commercialIdeBase) // Includes rdCommon nested (no overrides) + moduleSet(rdCommon) { // Also reference rdCommon with overrides + overrideAsEmbedded("intellij.rd.platform") + overrideAsEmbedded("intellij.rd.ui") + } + } + + // Test with inlineModuleSets = true (full inlining mode) - this is where the bug occurs + val result = buildProductContentXml( + spec = spec, + moduleOutputProvider = MockModuleOutputProvider(), + inlineXmlIncludes = false, + inlineModuleSets = true, // Full inlining mode + productPropertiesClass = "TestProperties", + generatorCommand = "test", + isUltimateBuild = false + ) + + // Verify rd modules have loading="embedded" attribute + assertThat(result.xml).contains("") + assertThat(result.xml).contains("") + + // Verify they don't appear without loading attribute (bug would cause this) + assertThat(result.xml).doesNotContainPattern("") + assertThat(result.xml).doesNotContainPattern("") + + // Verify modules appear exactly once + val rdPlatformCount = Regex(""" { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/CompilationContext.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/CompilationContext.kt index e365388a3c92..cd29a717e40b 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/CompilationContext.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/CompilationContext.kt @@ -5,7 +5,6 @@ import org.jetbrains.annotations.ApiStatus.Internal import org.jetbrains.intellij.build.dependencies.DependenciesProperties import org.jetbrains.intellij.build.impl.BundledRuntime import org.jetbrains.intellij.build.impl.CompilationTasksImpl -import org.jetbrains.intellij.build.impl.ModuleOutputProvider import org.jetbrains.intellij.build.moduleBased.OriginalModuleRepository import org.jetbrains.jps.model.JpsModel import org.jetbrains.jps.model.JpsProject diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/DuplicateIncludeAnalyzer.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/DuplicateIncludeAnalyzer.kt new file mode 100644 index 000000000000..a7494c3b2315 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/DuplicateIncludeAnalyzer.kt @@ -0,0 +1,142 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build + +import com.intellij.openapi.application.PathManager +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import org.jetbrains.intellij.build.productLayout.DuplicateIncludeDetector +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.isRegularFile +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.readText +import kotlin.io.path.walk + +/** + * Command-line tool to detect duplicate xi:include elements in product plugin.xml files. + * + * Usage: + * bazel run //community/platform/build-scripts:DuplicateIncludeAnalyzer + * + * Output: + * JSON report showing products with duplicate includes, where each duplicate comes from, + * and summary statistics. + */ +object DuplicateIncludeAnalyzer { + @JvmStatic + fun main(args: Array) { + runBlocking(Dispatchers.Default) { + val projectRoot = Path.of(PathManager.getHomePathFor(DuplicateIncludeAnalyzer::class.java)!!) + + // Discover all product plugin.xml files + val productFiles = discoverProductFiles(projectRoot) + + // Run detection + val report = DuplicateIncludeDetector.detectDuplicates(productFiles, projectRoot) + + // Output JSON + val json = Json { prettyPrint = true } + println(json.encodeToString(report)) + } + } + + /** + * Discovers all product plugin.xml files in the project. + * Searches in known product directories and looks for files ending with Plugin.xml or named plugin.xml. + */ + private fun discoverProductFiles(projectRoot: Path): List { + val productFiles = mutableListOf() + + // Known product directories to search + val productDirs = listOf( + projectRoot.resolve("community"), + projectRoot.resolve("ultimate"), + projectRoot.resolve("CIDR"), + projectRoot.resolve("goland"), + projectRoot.resolve("ruby"), + projectRoot.resolve("WebStorm"), + projectRoot.resolve("dbe"), + projectRoot.resolve("aqua"), + projectRoot.resolve("rider"), + projectRoot.resolve("python"), + projectRoot.resolve("plugins"), + ) + + for (dir in productDirs) { + if (!dir.exists() || !dir.isDirectory()) { + continue + } + + // Search for Plugin.xml files in resources/META-INF directories + dir.walk() + .filter { it.isRegularFile() } + .filter { it.parent?.fileName?.toString() == "META-INF" } + .filter { + val name = it.fileName.toString() + name.endsWith("Plugin.xml") || name == "plugin.xml" + } + .filter { isProductFile(it) } + .forEach { productFiles.add(it) } + } + + return productFiles.distinct() + } + + /** + * Checks if an XML file is a product descriptor (not a plugin descriptor). + * Products don't have their own tag or have com.intellij. + * Also filters out test files and non-product descriptors. + */ + private fun isProductFile(file: Path): Boolean { + // Skip test resources + if (file.toString().contains("/testResources/") || + file.toString().contains("/testSrc/") || + file.toString().contains("/test/")) { + return false + } + + // Skip toolbox + if (file.toString().contains("/toolbox/")) { + return false + } + + val fileName = file.fileName.toString() + + // Skip module descriptor files (have dots in the name like intellij.platform.jewel.detektPlugin.xml) + if (fileName != "plugin.xml" && fileName.contains(".") && !fileName.matches(Regex("^[A-Z][a-zA-Z]*Plugin\\.xml$"))) { + return false + } + + try { + val content = file.readText() + + // Check for tag that's NOT com.intellij + val idMatch = Regex("""([^<]+)""").find(content) + if (idMatch != null && idMatch.groupValues[1] != "com.intellij") { + // This is a plugin with its own ID, not a product + return false + } + + // Check if it has ApplicationInfo.xml nearby (strong indicator of a product) + val resourceRoot = file.parent?.parent // Go up from META-INF to resources + if (resourceRoot != null) { + val ideaDir = resourceRoot.resolve("idea") + if (ideaDir.exists() && ideaDir.isDirectory()) { + val hasAppInfo = ideaDir.listDirectoryEntries() + .any { it.fileName.toString().endsWith("ApplicationInfo.xml") } + if (hasAppInfo) { + return true + } + } + } + + // If no tag or has com.intellij, likely a product + return true + } + catch (e: Exception) { + return false + } + } +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/JarPackagerDependencyHelper.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/JarPackagerDependencyHelper.kt index fa66c9673002..f9e253b6d7a7 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/JarPackagerDependencyHelper.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/JarPackagerDependencyHelper.kt @@ -5,11 +5,8 @@ package org.jetbrains.intellij.build import com.intellij.util.xml.dom.XmlElement import com.intellij.util.xml.dom.readXmlAsModel -import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.impl.ModuleItem import org.jetbrains.intellij.build.impl.PluginLayout -import org.jetbrains.jps.model.java.JavaResourceRootType -import org.jetbrains.jps.model.java.JavaSourceRootType import org.jetbrains.jps.model.java.JpsJavaClasspathKind import org.jetbrains.jps.model.java.JpsJavaExtensionService import org.jetbrains.jps.model.module.JpsDependencyElement @@ -17,11 +14,8 @@ import org.jetbrains.jps.model.module.JpsLibraryDependency import org.jetbrains.jps.model.module.JpsModule import org.jetbrains.jps.model.module.JpsModuleDependency import org.jetbrains.jps.model.module.JpsModuleReference -import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap -internal val useTestSourceEnabled: Boolean = System.getProperty("idea.build.pack.test.source.enabled", "true").toBoolean() - // production-only - JpsJavaClasspathKind.PRODUCTION_RUNTIME internal class JarPackagerDependencyHelper(private val context: CompilationContext) { private val javaExtensionService = JpsJavaExtensionService.getInstance() @@ -74,7 +68,7 @@ internal class JarPackagerDependencyHelper(private val context: CompilationConte return moduleName.endsWith("._test") } - suspend fun getPluginIdByModule(pluginModule: JpsModule): String { + fun getPluginIdByModule(pluginModule: JpsModule): String { // it is ok to read the plugin descriptor with unresolved x-include as the ID should be specified at the root val root = readXmlAsModel(getUnprocessedPluginXmlContent(module = pluginModule, context = context)) val element = root.getChild("id") ?: root.getChild("name") ?: throw IllegalStateException("Cannot find attribute id or name (module=$pluginModule)") diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/LinuxDistributionCustomizer.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/LinuxDistributionCustomizer.kt index f1820c45fe01..763c135ef318 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/LinuxDistributionCustomizer.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/LinuxDistributionCustomizer.kt @@ -33,7 +33,7 @@ open class LinuxDistributionCustomizer { */ var extraExecutables: PersistentList = persistentListOf() - open fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture, targetLibcImpl: LibcImpl): Sequence { + open fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, targetLibcImpl: LibcImpl, context: BuildContext): Sequence { val basePatterns = sequenceOf( "bin/*.sh", "plugins/**/*.sh", @@ -77,13 +77,14 @@ open class LinuxDistributionCustomizer { /** * Name of the root directory inside the .tar.gz archive. */ - open fun getRootDirectoryName(appInfo: ApplicationInfoProperties, buildNumber: String): String = - "${appInfo.fullProductName}-${if (appInfo.isEAP) buildNumber else appInfo.fullVersion}" + open fun getRootDirectoryName(appInfo: ApplicationInfoProperties, buildNumber: String): String { + return "${appInfo.fullProductName}-${if (appInfo.isEAP) buildNumber else appInfo.fullVersion}" + } /** * Override this method to copy additional files to the Linux distribution of the product. */ - open suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path, arch: JvmArchitecture) { + open suspend fun copyAdditionalFiles(targetDir: Path, arch: JvmArchitecture, context: BuildContext) { RepairUtilityBuilder.bundle(context, OsFamily.LINUX, arch, targetDir) } } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/MacDistributionCustomizer.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/MacDistributionCustomizer.kt index 1005ce14c21c..11e9072f6589 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/MacDistributionCustomizer.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/MacDistributionCustomizer.kt @@ -6,7 +6,7 @@ import kotlinx.collections.immutable.persistentListOf import org.jetbrains.annotations.ApiStatus import org.jetbrains.intellij.build.impl.support.RepairUtilityBuilder import java.nio.file.Path -import java.util.* +import java.util.UUID import java.util.function.Predicate open class MacDistributionCustomizer { @@ -142,7 +142,7 @@ open class MacDistributionCustomizer { RepairUtilityBuilder.bundle(context, OsFamily.MACOS, arch, targetDir) } - open fun generateExecutableFilesPatterns(context: BuildContext, includeRuntime: Boolean, arch: JvmArchitecture): Sequence { + open fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, context: BuildContext): Sequence { val basePatterns = sequenceOf( "bin/*.sh", "plugins/**/*.sh", diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/ProductProperties.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/ProductProperties.kt index 3554ff930520..891be3933a7b 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/ProductProperties.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/ProductProperties.kt @@ -330,7 +330,7 @@ abstract class ProductProperties { /** * Override this method to copy additional files to distributions of all operating systems. */ - open suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) { } + open suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) { } /** * Override this method if the product has several editions to ensure that their artifacts won't be mixed up. @@ -408,7 +408,7 @@ abstract class ProductProperties { * Copies additional localization resources to the plugin-generated localization resources directory. */ @ApiStatus.Internal - open suspend fun copyAdditionalLocalizationResourcesToPlugin(context: BuildContext, lang: String, targetDir: Path) {} + open suspend fun copyAdditionalLocalizationResourcesToPlugin(lang: String, targetDir: Path, context: BuildContext) {} /** * Build steps which are always skipped for this product. diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/WindowsDistributionCustomizer.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/WindowsDistributionCustomizer.kt index 603c4f1cc897..7de2cffab8a8 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/WindowsDistributionCustomizer.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/WindowsDistributionCustomizer.kt @@ -75,7 +75,7 @@ open class WindowsDistributionCustomizer { /** * Override this method to copy additional files to the Windows distribution of the product. */ - open suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path, arch: JvmArchitecture) { + open suspend fun copyAdditionalFiles(targetDir: Path, arch: JvmArchitecture, context: BuildContext) { RepairUtilityBuilder.bundle(context, OsFamily.WINDOWS, arch, targetDir) } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/autoLayout.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/autoLayout.kt index 0e9b46412f68..788f6c3058b7 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/autoLayout.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/autoLayout.kt @@ -3,7 +3,6 @@ package org.jetbrains.intellij.build import com.intellij.openapi.util.JDOMUtil import com.intellij.util.xml.dom.readXmlAsModel -import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.impl.BUILT_IN_HELP_MODULE_NAME import org.jetbrains.intellij.build.impl.JarPackager import org.jetbrains.intellij.build.impl.ModuleItem diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/classpath.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/classpath.kt index 70f44d63bf30..41f4be6aff5c 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/classpath.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/classpath.kt @@ -11,6 +11,7 @@ import org.jetbrains.annotations.VisibleForTesting import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.OsFamily import org.jetbrains.intellij.build.PLATFORM_LOADER_JAR +import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.UTIL_8_JAR import org.jetbrains.intellij.build.UTIL_JAR import org.jetbrains.intellij.build.impl.DescriptorCacheContainer diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/contentModuleEmbedding.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/contentModuleEmbedding.kt index 2acb4bb1201c..d693d8cb3843 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/contentModuleEmbedding.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/classPath/contentModuleEmbedding.kt @@ -27,11 +27,11 @@ import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.CompilationContext import org.jetbrains.intellij.build.FrontendModuleFilter import org.jetbrains.intellij.build.JarPackagerDependencyHelper +import org.jetbrains.intellij.build.ModuleOutputProvider import org.jetbrains.intellij.build.findFileInModuleDependencies import org.jetbrains.intellij.build.findUnprocessedDescriptorContent import org.jetbrains.intellij.build.impl.BuildContextImpl import org.jetbrains.intellij.build.impl.DescriptorCacheContainer -import org.jetbrains.intellij.build.impl.ModuleOutputProvider import org.jetbrains.intellij.build.impl.PluginLayout import org.jetbrains.intellij.build.impl.ScopedCachedDescriptorContainer import org.jetbrains.intellij.build.impl.XIncludeElementResolver @@ -41,8 +41,6 @@ import org.jetbrains.intellij.build.impl.toLoadPath import java.io.IOException import java.nio.file.Files -internal const val PLUGIN_XML_RELATIVE_PATH = "META-INF/plugin.xml" - /** * Defines a search scope for resolving XInclude references in plugin descriptors. * diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/dev/IdeBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/IdeBuilder.kt index aa2321906472..bf20890d8566 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/dev/IdeBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/IdeBuilder.kt @@ -29,6 +29,7 @@ import org.jetbrains.intellij.build.JvmArchitecture import org.jetbrains.intellij.build.LibcImpl import org.jetbrains.intellij.build.LinuxDistributionCustomizer import org.jetbrains.intellij.build.MacDistributionCustomizer +import org.jetbrains.intellij.build.ModuleOutputProvider import org.jetbrains.intellij.build.OsFamily import org.jetbrains.intellij.build.ProductProperties import org.jetbrains.intellij.build.ProprietaryBuildTools @@ -44,7 +45,6 @@ import org.jetbrains.intellij.build.getDevModeOrTestBuildDateInSeconds import org.jetbrains.intellij.build.impl.BuildContextImpl import org.jetbrains.intellij.build.impl.CompilationContextImpl import org.jetbrains.intellij.build.impl.ModuleOutputPatcher -import org.jetbrains.intellij.build.impl.ModuleOutputProvider import org.jetbrains.intellij.build.impl.PLUGIN_CLASSPATH import org.jetbrains.intellij.build.impl.PlatformLayout import org.jetbrains.intellij.build.impl.asArchived diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt index 4a97eb1034f3..2012c235e2a6 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt @@ -209,7 +209,7 @@ private suspend fun layoutShared(context: BuildContext) { Files.createDirectories(to.parent) Files.copy(from, to, StandardCopyOption.REPLACE_EXISTING) } - context.productProperties.copyAdditionalFiles(context, context.paths.distAllDir) + context.productProperties.copyAdditionalFiles(context.paths.distAllDir, context) } } checkClassFiles(root = context.paths.distAllDir, isDistAll = true, context) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt index a2539b2bd254..6b2104edc3cf 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt @@ -107,7 +107,7 @@ class CompilationContextImpl private constructor( val global: JpsGlobal get() = model.global - private val moduleOutputProvider = ModuleOutputProvider.jps(project.modules) + private val moduleOutputProvider = jpsModuleOutputProvider(project.modules) override var classesOutputDirectory: Path get() = Path.of(JpsPathUtil.urlToPath(JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(project).outputUrl)) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JpsModuleOutpuProvider.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JpsModuleOutpuProvider.kt new file mode 100644 index 000000000000..4c06c03520e3 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JpsModuleOutpuProvider.kt @@ -0,0 +1,43 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.impl + +import org.jetbrains.intellij.build.ModuleOutputProvider +import org.jetbrains.jps.model.java.JpsJavaExtensionService +import org.jetbrains.jps.model.module.JpsModule +import org.jetbrains.jps.util.JpsPathUtil +import java.nio.file.Files +import java.nio.file.NoSuchFileException +import java.nio.file.Path + +internal fun jpsModuleOutputProvider(modules: List): ModuleOutputProvider { + return object : ModuleOutputProvider { + private val nameToModule = modules.associateByTo(HashMap(modules.size)) { it.name } + override fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean): ByteArray? { + val outputRoots = getModuleOutputRoots(module, forTests) + val outputDir = outputRoots.singleOrNull() ?: error("More than one output root for module '${module.name}': ${outputRoots.joinToString()}") + val file = outputDir.resolve(relativePath) + try { + return Files.readAllBytes(file) + } + catch (_: NoSuchFileException) { + return null + } + } + + override fun findModule(name: String): JpsModule? = nameToModule.get(name.removeSuffix("._test")) + + override fun findRequiredModule(name: String): JpsModule { + return checkNotNull(findModule(name)) { + "Cannot find required module '$name' in the project" + } + } + + override fun getModuleOutputRoots(module: JpsModule, forTests: Boolean): List { + val url = JpsJavaExtensionService.getInstance().getOutputUrl(/* module = */ module, /* forTests = */ forTests) + requireNotNull(url) { + "Output directory for ${module.name} isn't set" + } + return listOf(Path.of(JpsPathUtil.urlToPath(url))) + } + } +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt index a75edeeeeb5a..876cc3a65d8b 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt @@ -91,12 +91,12 @@ class LinuxDistributionBuilder( createFrontendContextForLaunchers(context)?.let { clientContext -> writeLinuxVmOptions(distBinDir, clientContext) generateLauncherScript( - distBinDir, arch, ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, clientContext, targetLibcImpl + distBinDir, arch, ADDITIONAL_EMBEDDED_CLIENT_VM_OPTIONS, targetLibcImpl, clientContext ) } generateReadme(targetPath) generateVersionMarker(targetPath, context) - customizer.copyAdditionalFiles(context, targetPath, arch) + customizer.copyAdditionalFiles(targetPath, arch, context) } } } @@ -137,12 +137,12 @@ class LinuxDistributionBuilder( "linux_tar_gz_${arch.name}" ) { _ -> val suffix = suffix(arch, targetLibcImpl) - buildTarGz(arch, runtimeDir, osAndArchSpecificDistPath, suffix) + buildTarGz(arch = arch, runtimeDir = runtimeDir, unixDistPath = osAndArchSpecificDistPath, suffix = suffix) } if (targetLibcImpl != LinuxLibcImpl.MUSL) { launch(Dispatchers.IO + CoroutineName("build Snap package")) { - buildSnapPackage(runtimeDir, osAndArchSpecificDistPath, arch, targetLibcImpl) + buildSnapPackage(runtimeDir = runtimeDir, unixDistPath = osAndArchSpecificDistPath, arch = arch, targetLibcImpl = targetLibcImpl) } } @@ -179,8 +179,9 @@ class LinuxDistributionBuilder( ) } - override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence = - customizer.generateExecutableFilesPatterns(context, includeRuntime, arch, libc) + override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence { + return customizer.generateExecutableFilesPatterns(includeRuntime = includeRuntime, arch = arch, targetLibcImpl = libc, context = context) + } private val rootDirectoryName: String get() = customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber) @@ -202,7 +203,7 @@ class LinuxDistributionBuilder( } val productJsonDir = context.paths.tempDir.resolve("linux.dist.product-info.json${suffix}") - val productJsonFile = writeProductJsonFile(productJsonDir, arch, withRuntime = runtimeDir != null) + val productJsonFile = writeProductJsonFile(targetDir = productJsonDir, arch = arch, withRuntime = runtimeDir != null) dirs.add(productJsonDir) spanBuilder("build Linux tar.gz") @@ -216,7 +217,13 @@ class LinuxDistributionBuilder( context.notifyArtifactBuilt(tarProductInfoJsonPath) context.notifyArtifactBuilt(tarPath) - checkExecutablePermissions(tarPath, rootDirectoryName, includeRuntime = runtimeDir != null, arch, this@LinuxDistributionBuilder.targetLibcImpl) + checkExecutablePermissions( + distribution = tarPath, + root = rootDirectoryName, + includeRuntime = runtimeDir != null, + arch = arch, + libc = this@LinuxDistributionBuilder.targetLibcImpl, + ) } tarPath } @@ -227,11 +234,6 @@ class LinuxDistributionBuilder( "${appInfo.majorVersion}.${appInfo.minorVersion}${if (versionSuffix.isEmpty()) "" else "-${versionSuffix}"}" } - private fun getSnapArchName(arch: JvmArchitecture) = when (arch) { - JvmArchitecture.x64 -> "amd64" - JvmArchitecture.aarch64 -> "arm64" - } - private fun getSnapArtifactName(snapName: String, arch: JvmArchitecture): String = "${snapName}_${snapVersion}_${getSnapArchName(arch)}.snap" private suspend fun buildSnapPackage(runtimeDir: Path, unixDistPath: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl) { @@ -244,7 +246,7 @@ class LinuxDistributionBuilder( } buildSnapPackage(snapName, runtimeDir, unixDistPath, arch, targetLibcImpl) customizer.snapLegacyAliases.forEach { - buildSnapPackage(it, runtimeDir, unixDistPath, arch, targetLibcImpl) + buildSnapPackage(snapName = it, runtimeDir = runtimeDir, unixDistPath = unixDistPath, arch = arch, targetLibcImpl = targetLibcImpl) } } @@ -279,9 +281,7 @@ class LinuxDistributionBuilder( ) ) copyFile(iconPngPath, snapDir.resolve("$snapName.png")) - val snapcraftTemplate = context.paths.communityHomeDir.resolve( - "platform/build-scripts/resources/linux/snap/snapcraft-template.yaml" - ) + val snapcraftTemplate = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/snap/snapcraft-template.yaml") val snapcraftConfig = snapDir.resolve("snapcraft.yaml") substituteTemplatePlaceholders( inputFile = snapcraftTemplate, @@ -306,7 +306,7 @@ class LinuxDistributionBuilder( val productJsonDir = context.paths.tempDir.resolve("linux.dist.snap.$snapName.product-info.json.$architecture") val productJsonFile = writeProductJsonFile(productJsonDir, arch) val installationDirectories = listOf(context.paths.distAllDir, unixDistPath, runtimeDir) - validateProductJson(jsonText = productJsonFile.readText(), installationDirectories, installationArchives = emptyList(), context) + validateProductJson(jsonText = productJsonFile.readText(), installationDirectories = installationDirectories, installationArchives = emptyList(), context = context) val resultDir = snapDir.resolve("result") Files.createDirectories(resultDir) @@ -385,101 +385,112 @@ class LinuxDistributionBuilder( writeProductInfoJson(file, json, context) return file } +} - private fun generateVersionMarker(unixDistPath: Path, context: BuildContext) { - val targetDir = unixDistPath.resolve("lib") - Files.createDirectories(targetDir) - Files.writeString(targetDir.resolve("build-marker-" + context.fullBuildNumber), context.fullBuildNumber) - } +private fun generateVersionMarker(unixDistPath: Path, context: BuildContext) { + val targetDir = unixDistPath.resolve("lib") + Files.createDirectories(targetDir) + Files.writeString(targetDir.resolve("build-marker-${context.fullBuildNumber}"), context.fullBuildNumber) +} - private fun generateScripts(distBinDir: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl, context: BuildContext) { - Files.createDirectories(distBinDir) +private fun generateScripts(distBinDir: Path, arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl, context: BuildContext) { + Files.createDirectories(distBinDir) - val sourceScriptDir = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts") - Files.newDirectoryStream(sourceScriptDir).use { - for (file in it) { - val fileName = file.fileName.toString() - if (fileName != EXECUTABLE_TEMPLATE_NAME) { - copyScript(file, distBinDir.resolve(fileName), additionalTemplateValues = emptyList(), context) - } + val sourceScriptDir = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts") + Files.newDirectoryStream(sourceScriptDir).use { + for (file in it) { + val fileName = file.fileName.toString() + if (fileName != EXECUTABLE_TEMPLATE_NAME) { + copyScript(sourceFile = file, targetFile = distBinDir.resolve(fileName), additionalTemplateValues = emptyList(), context = context) } } - - copyInspectScript(context, distBinDir) - - generateLauncherScript(distBinDir, arch, nonCustomizableJvmArgs = emptyList(), context, targetLibcImpl) } - private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture, context: BuildContext) { - val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch) - copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName)) - copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html")) - } + copyInspectScript(context, distBinDir) - private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List, context: BuildContext, targetLibcImpl: LinuxLibcImpl) { - val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") + generateLauncherScript(distBinDir = distBinDir, arch = arch, nonCustomizableJvmArgs = emptyList(), targetLibcImpl = targetLibcImpl, context = context) +} - val defaultXmxParameter = try { - Files.readAllLines(vmOptionsPath).firstOrNull { it.startsWith("-Xmx") } - ?: throw IllegalStateException("-Xmx was not found in '$vmOptionsPath'") - } - catch (e: NoSuchFileException) { - throw IllegalStateException("File '$vmOptionsPath' should be already generated at this point", e) - } +private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture, context: BuildContext) { + val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch) + copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName)) + copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html")) +} - val classPathJars = context.bootClassPathJarNames - var classPath = $$"CLASS_PATH=\"$IDE_HOME/lib/$${classPathJars[0]}\"" - for (i in 1 until classPathJars.size) { - classPath += $$"\nCLASS_PATH=\"$CLASS_PATH:$IDE_HOME/lib/$${classPathJars[i]}\"" - } +private fun generateLauncherScript(distBinDir: Path, arch: JvmArchitecture, nonCustomizableJvmArgs: List, targetLibcImpl: LinuxLibcImpl, context: BuildContext) { + val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") - val additionalJvmArguments = mutableListOf() - // https://youtrack.jetbrains.com/issue/IDEA-304440 - // "-Djdk.lang.Process.launchMechanism=vfork" - if (targetLibcImpl == LinuxLibcImpl.MUSL) { - additionalJvmArguments.add("-Djdk.lang.Process.launchMechanism=vfork") - } - additionalJvmArguments.addAll(context.getAdditionalJvmArguments(OsFamily.LINUX, arch, isScript = true) + nonCustomizableJvmArgs) + val defaultXmxParameter = try { + Files.readAllLines(vmOptionsPath).firstOrNull { it.startsWith("-Xmx") } + ?: throw IllegalStateException("-Xmx was not found in '$vmOptionsPath'") + } + catch (e: NoSuchFileException) { + throw IllegalStateException("File '$vmOptionsPath' should be already generated at this point", e) + } - val additionalTemplateValues = listOf( - Pair("vm_options", context.productProperties.baseFileName), - Pair("system_selector", context.systemSelector), - Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")), - Pair("ide_default_xmx", defaultXmxParameter.trim()), - Pair("class_path", classPath), - Pair("main_class_name", context.ideMainClassName), - ) + val classPathJars = context.bootClassPathJarNames + var classPath = $$"CLASS_PATH=\"$IDE_HOME/lib/$${classPathJars[0]}\"" + for (i in 1 until classPathJars.size) { + classPath += $$"\nCLASS_PATH=\"$CLASS_PATH:$IDE_HOME/lib/$${classPathJars[i]}\"" + } - val template = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts/${EXECUTABLE_TEMPLATE_NAME}") - copyScript(template, distBinDir.resolve("${context.productProperties.baseFileName}.sh"), additionalTemplateValues, context) + val additionalJvmArguments = mutableListOf() + // https://youtrack.jetbrains.com/issue/IDEA-304440 + // "-Djdk.lang.Process.launchMechanism=vfork" + if (targetLibcImpl == LinuxLibcImpl.MUSL) { + additionalJvmArguments.add("-Djdk.lang.Process.launchMechanism=vfork") } + additionalJvmArguments.addAll(context.getAdditionalJvmArguments(os = OsFamily.LINUX, arch = arch, isScript = true) + nonCustomizableJvmArgs) + + val additionalTemplateValues = listOf( + Pair("vm_options", context.productProperties.baseFileName), + Pair("system_selector", context.systemSelector), + Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")), + Pair("ide_default_xmx", defaultXmxParameter.trim()), + Pair("class_path", classPath), + Pair("main_class_name", context.ideMainClassName), + ) - private fun copyScript(sourceFile: Path, targetFile: Path, additionalTemplateValues: List>, context: BuildContext) { - // Until CR (\r) will be removed from the repository checkout, we need to filter it out from Unix-style scripts - // https://youtrack.jetbrains.com/issue/IJI-526/Force-git-to-use-LF-line-endings-in-working-copy-of-via-gitattri - substituteTemplatePlaceholders( - inputFile = sourceFile, - outputFile = targetFile, - placeholder = "__", - values = listOf( - Pair("product_full", context.applicationInfo.fullProductName), - Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)), - Pair("product_vendor", context.applicationInfo.shortCompanyName), - Pair("product_code", context.applicationInfo.productCode), - Pair("script_name", "${context.productProperties.baseFileName}.sh"), - ) + additionalTemplateValues, - mustUseAllPlaceholders = false, - convertToUnixLineEndings = true, - ) - } + val template = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/linux/scripts/${EXECUTABLE_TEMPLATE_NAME}") + copyScript( + sourceFile = template, + targetFile = distBinDir.resolve("${context.productProperties.baseFileName}.sh"), + additionalTemplateValues = additionalTemplateValues, + context = context, + ) +} + +private fun copyScript(sourceFile: Path, targetFile: Path, additionalTemplateValues: List>, context: BuildContext) { + // Until CR (\r) will be removed from the repository checkout, we need to filter it out from Unix-style scripts + // https://youtrack.jetbrains.com/issue/IJI-526/Force-git-to-use-LF-line-endings-in-working-copy-of-via-gitattri + substituteTemplatePlaceholders( + inputFile = sourceFile, + outputFile = targetFile, + placeholder = "__", + values = listOf( + Pair("product_full", context.applicationInfo.fullProductName), + Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)), + Pair("product_vendor", context.applicationInfo.shortCompanyName), + Pair("product_code", context.applicationInfo.productCode), + Pair("script_name", "${context.productProperties.baseFileName}.sh"), + ) + additionalTemplateValues, + mustUseAllPlaceholders = false, + convertToUnixLineEndings = true, + ) +} + +private fun writeLinuxVmOptions(distBinDir: Path, context: BuildContext): Path { + val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") + val vmOptions = generateVmOptions(context).asSequence() + sequenceOf("-Dsun.tools.attach.tmp.only=true", "-Dawt.lock.fair=true") + writeVmOptions(file = vmOptionsPath, vmOptions = vmOptions, separator = "\n") + return vmOptionsPath +} - private fun writeLinuxVmOptions(distBinDir: Path, context: BuildContext): Path { - val vmOptionsPath = distBinDir.resolve("${context.productProperties.baseFileName}64.vmoptions") - val vmOptions = VmOptionsGenerator.generate(context).asSequence() + sequenceOf("-Dsun.tools.attach.tmp.only=true", "-Dawt.lock.fair=true") - VmOptionsGenerator.writeVmOptions(vmOptionsPath, vmOptions, separator = "\n") - return vmOptionsPath - } +private fun suffix(arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl): String { + return suffix(arch) + if (targetLibcImpl == LinuxLibcImpl.MUSL) "-musl" else "" +} - private fun suffix(arch: JvmArchitecture, targetLibcImpl: LinuxLibcImpl): String = - suffix(arch) + if (targetLibcImpl == LinuxLibcImpl.MUSL) "-musl" else "" +private fun getSnapArchName(arch: JvmArchitecture) = when (arch) { + JvmArchitecture.x64 -> "amd64" + JvmArchitecture.aarch64 -> "arm64" } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt index 81ea43d9a0d9..d162e7bd66d7 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MacDistributionBuilder.kt @@ -118,7 +118,7 @@ class MacDistributionBuilder( override suspend fun copyFilesForOsDistribution(targetPath: Path, arch: JvmArchitecture) { withContext(Dispatchers.IO) { - doCopyExtraFiles(targetPath, arch, copyDistFiles = true) + doCopyExtraFiles(macDistDir = targetPath, arch = arch, copyDistFiles = true) } } @@ -327,7 +327,7 @@ class MacDistributionBuilder( } override fun generateExecutableFilesPatterns(includeRuntime: Boolean, arch: JvmArchitecture, libc: LibcImpl): Sequence = - customizer.generateExecutableFilesPatterns(context, includeRuntime, arch) + customizer.generateExecutableFilesPatterns(includeRuntime, arch, context) private suspend fun buildForArch( arch: JvmArchitecture, @@ -490,15 +490,15 @@ class MacDistributionBuilder( } } - validateProductJson(targetFile, pathInArchive = "${zipRoot}/Resources", macDistributionBuilder.context) + validateProductJson(archiveFile = targetFile, pathInArchive = "${zipRoot}/Resources", context = macDistributionBuilder.context) } } private fun writeMacOsVmOptions(distBinDir: Path, context: BuildContext): Path { val executable = context.productProperties.baseFileName - val vmOptions = VmOptionsGenerator.generate(context).asSequence() + sequenceOf("-Dapple.awt.application.appearance=system") + val vmOptions = generateVmOptions(context).asSequence() + sequenceOf("-Dapple.awt.application.appearance=system") val vmOptionsPath = distBinDir.resolve("${executable}.vmoptions") - VmOptionsGenerator.writeVmOptions(vmOptionsPath, vmOptions, separator = "\n") + writeVmOptions(vmOptionsPath, vmOptions, separator = "\n") return vmOptionsPath } @@ -567,8 +567,9 @@ class MacDistributionBuilder( } } - private fun getMacZipRoot(customizer: MacDistributionCustomizer, context: BuildContext): String = - "${customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)}/Contents" + private fun getMacZipRoot(customizer: MacDistributionCustomizer, context: BuildContext): String { + return "${customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber)}/Contents" + } private val publishSitArchive: Boolean get() = !context.isStepSkipped(BuildOptions.MAC_SIT_PUBLICATION_STEP) @@ -636,76 +637,75 @@ class MacDistributionBuilder( NioFiles.deleteRecursively(tempDir) } } +} - private fun prepareDmgBuildScripts(tempDir: Path, staple: Boolean, customizer: MacDistributionCustomizer, context: BuildContext): Path { - NioFiles.deleteRecursively(tempDir) - Files.createDirectories(tempDir) - val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png") - Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath), dmgImageCopy) - val scriptsDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts") - Files.copy(scriptsDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - NioFiles.setExecutable(tempDir.resolve("makedmg.sh")) - Files.copy(scriptsDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - Files.copy(scriptsDir.resolve("staple.sh"), tempDir.resolve("staple.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - val entrypoint = tempDir.resolve("build.sh") - Files.writeString( - entrypoint, - Files.readString(scriptsDir.resolve("build-template.sh")) - .resolveTemplateVar("staple", "$staple") - .resolveTemplateVar("appName", context.fullBuildNumber) - .resolveTemplateVar("contentSigned", "${context.isMacCodeSignEnabled}") - .resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}") - ) - NioFiles.setExecutable(entrypoint) - return entrypoint +private fun prepareDmgBuildScripts(tempDir: Path, staple: Boolean, customizer: MacDistributionCustomizer, context: BuildContext): Path { + NioFiles.deleteRecursively(tempDir) + Files.createDirectories(tempDir) + val dmgImageCopy = tempDir.resolve("${context.fullBuildNumber}.png") + Files.copy(Path.of((if (context.applicationInfo.isEAP) customizer.dmgImagePathForEAP else null) ?: customizer.dmgImagePath), dmgImageCopy) + val scriptsDir = context.paths.communityHomeDir.resolve("platform/build-scripts/tools/mac/scripts") + Files.copy(scriptsDir.resolve("makedmg.sh"), tempDir.resolve("makedmg.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + NioFiles.setExecutable(tempDir.resolve("makedmg.sh")) + Files.copy(scriptsDir.resolve("makedmg.py"), tempDir.resolve("makedmg.py"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + Files.copy(scriptsDir.resolve("staple.sh"), tempDir.resolve("staple.sh"), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + val entrypoint = tempDir.resolve("build.sh") + Files.writeString( + entrypoint, + Files.readString(scriptsDir.resolve("build-template.sh")) + .resolveTemplateVar("staple", "$staple") + .resolveTemplateVar("appName", context.fullBuildNumber) + .resolveTemplateVar("contentSigned", "${context.isMacCodeSignEnabled}") + .resolveTemplateVar("buildDateInSeconds", "${context.options.buildDateInSeconds}") + ) + NioFiles.setExecutable(entrypoint) + return entrypoint +} + +private fun String.resolveTemplateVar(variable: String, value: String): String { + val reference = "%$variable%" + check(contains(reference)) { "No $reference is found in:\n'$this'" } + return replace(reference, value) +} + +private fun publishDmgBuildScripts(entrypoint: Path, tempDir: Path, context: BuildContext) { + val artifactDir = context.paths.artifactDir.resolve("macos-dmg-build") + artifactDir.createDirectories() + synchronized("$artifactDir".intern()) { + tempDir.listDirectoryEntries().forEach { + Files.copy(it, artifactDir.resolve(it.name), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) + } + val message = """ + To build .dmg(s): + 1. transfer .sit(s) to macOS host; + 2. transfer ${artifactDir.name}/ content to the same folder; + 3. execute ${entrypoint.name} from Terminal. + .dmg(s) will be built in the same folder. + """.trimIndent() + artifactDir.resolve("README.txt").writeText(message) + context.messages.info(message) + context.notifyArtifactBuilt(artifactDir) + } +} + +private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, arch: JvmArchitecture, context: BuildContext) { + if (context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) { + return } - - private fun String.resolveTemplateVar(variable: String, value: String): String { - val reference = "%$variable%" - check(contains(reference)) { "No $reference is found in:\n'$this'" } - return replace(reference, value) + val tempSit = Files.createTempDirectory(context.paths.tempDir, "sit-") + try { + spanBuilder("extracting ${sitFile.name}").use(Dispatchers.IO) { + Decompressor.Zip(sitFile) + .withZipExtensions() + .extract(tempSit) + } + RepairUtilityBuilder.generateManifest(context, tempSit.resolve(sitRoot), OsFamily.MACOS, arch) } - - private fun publishDmgBuildScripts(entrypoint: Path, tempDir: Path, context: BuildContext) { - val artifactDir = context.paths.artifactDir.resolve("macos-dmg-build") - artifactDir.createDirectories() - synchronized("$artifactDir".intern()) { - tempDir.listDirectoryEntries().forEach { - Files.copy(it, artifactDir.resolve(it.name), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) - } - val message = """ - To build .dmg(s): - 1. transfer .sit(s) to macOS host; - 2. transfer ${artifactDir.name}/ content to the same folder; - 3. execute ${entrypoint.name} from Terminal. - .dmg(s) will be built in the same folder. - """.trimIndent() - artifactDir.resolve("README.txt").writeText(message) - context.messages.info(message) - context.notifyArtifactBuilt(artifactDir) - } - } - - private suspend fun generateIntegrityManifest(sitFile: Path, sitRoot: String, arch: JvmArchitecture, context: BuildContext) { - if (context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) { - return - } - - val tempSit = Files.createTempDirectory(context.paths.tempDir, "sit-") - try { - spanBuilder("extracting ${sitFile.name}").use(Dispatchers.IO) { - Decompressor.Zip(sitFile) - .withZipExtensions() - .extract(tempSit) - } - RepairUtilityBuilder.generateManifest(context, tempSit.resolve(sitRoot), OsFamily.MACOS, arch) - } - finally { - withContext(Dispatchers.IO + NonCancellable) { - @OptIn(ExperimentalPathApi::class) - tempSit.deleteRecursively() - } + finally { + withContext(Dispatchers.IO + NonCancellable) { + @OptIn(ExperimentalPathApi::class) + tempSit.deleteRecursively() } } } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ModuleOutputProvider.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ModuleOutputProvider.kt deleted file mode 100644 index ca38b2fe3ada..000000000000 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ModuleOutputProvider.kt +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -@file:Suppress("ReplaceGetOrSet") - -package org.jetbrains.intellij.build.impl - -import org.jetbrains.jps.model.java.JpsJavaExtensionService -import org.jetbrains.jps.model.module.JpsModule -import org.jetbrains.jps.util.JpsPathUtil -import java.nio.file.Files -import java.nio.file.NoSuchFileException -import java.nio.file.Path - -interface ModuleOutputProvider { - companion object { - fun jps(modules: List): ModuleOutputProvider { - return object : ModuleOutputProvider { - private val nameToModule = modules.associateByTo(HashMap(modules.size)) { it.name } - override fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean): ByteArray? { - val outputRoots = getModuleOutputRoots(module, forTests) - val outputDir = outputRoots.singleOrNull() ?: error("More than one output root for module '${module.name}': ${outputRoots.joinToString()}") - val file = outputDir.resolve(relativePath) - try { - return Files.readAllBytes(file) - } - catch (_: NoSuchFileException) { - return null - } - } - - override fun findModule(name: String): JpsModule? = nameToModule.get(name.removeSuffix("._test")) - - override fun findRequiredModule(name: String): JpsModule { - return checkNotNull(findModule(name)) { - "Cannot find required module '$name' in the project" - } - } - - override fun getModuleOutputRoots(module: JpsModule, forTests: Boolean): List { - val url = JpsJavaExtensionService.getInstance().getOutputUrl(/* module = */ module, /* forTests = */ forTests) - requireNotNull(url) { - "Output directory for ${module.name} isn't set" - } - return listOf(Path.of(JpsPathUtil.urlToPath(url))) - } - } - } - } - - fun readFileContentFromModuleOutput(module: JpsModule, relativePath: String, forTests: Boolean = false): ByteArray? - - fun findModule(name: String): JpsModule? - - fun findRequiredModule(name: String): JpsModule - - fun getModuleOutputRoots(module: JpsModule, forTests: Boolean = false): List -} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt index 61202a42df93..517ba788b290 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt @@ -9,8 +9,8 @@ import org.jdom.Element import org.jetbrains.annotations.TestOnly import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.CompatibleBuildRange +import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.classPath.DescriptorSearchScope -import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.classPath.XIncludeElementResolverImpl import org.jetbrains.intellij.build.classPath.embedContentModule import org.jetbrains.intellij.build.getUnprocessedPluginXmlContent diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt index 1c192cc06739..3d654232dbc2 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/TestingTasksImpl.kt @@ -633,7 +633,7 @@ internal class TestingTasksImpl(context: CompilationContext, private val options val customMemoryOptions = options.jvmMemoryOptions?.trim()?.split(Regex("\\s+"))?.takeIf { it.isNotEmpty() } jvmArgs.addAll( index = 0, - elements = VmOptionsGenerator.generate( + elements = generateVmOptions( isEAP = true, customVmMemoryOptions = if (customMemoryOptions == null) mapOf("-Xms" to "750m", "-Xmx" to "1024m") else emptyMap(), additionalVmOptions = customMemoryOptions ?: emptyList(), diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt index a0829ef7d9a1..ea338d5addb0 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/VmOptionsGenerator.kt @@ -9,92 +9,92 @@ import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.name -object VmOptionsGenerator { - private const val DEFAULT_MIN_HEAP = "128m" - private const val DEFAULT_MAX_HEAP = "2048m" +/** duplicates RepositoryHelper.CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY */ +private const val CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY = "intellij.plugins.custom.built.in.repository.url" - private val COMMON_VM_OPTIONS: List = listOf( - "-XX:JbrShrinkingGcMaxHeapFreeRatio=40", // IJPL-181469. Used in a couple with AppIdleMemoryCleaner.runGc() - "-XX:ReservedCodeCacheSize=512m", - "-XX:+HeapDumpOnOutOfMemoryError", - "-XX:-OmitStackTraceInFastThrow", - "-XX:CICompilerCount=2", - "-XX:+IgnoreUnrecognizedVMOptions", // allowing the JVM to start even with outdated options stuck in user configs - "-XX:+UnlockDiagnosticVMOptions", - "-XX:TieredOldPercentage=100000", - "-ea", - "-Dsun.io.useCanonCaches=false", - "-Dsun.java2d.metal=true", - "-Djbr.catch.SIGABRT=true", - "-Djdk.http.auth.tunneling.disabledSchemes=\"\"", - "-Djdk.attach.allowAttachSelf=true", - "-Djdk.module.illegalAccess.silent=true", - "-Djdk.nio.maxCachedBufferSize=2097152", - "-Djava.util.zip.use.nio.for.zip.file.access=true", // IJPL-149160 - "-Dkotlinx.coroutines.debug=off", - ) +private const val DEFAULT_MIN_HEAP = "128m" +private const val DEFAULT_MAX_HEAP = "2048m" - /** duplicates RepositoryHelper.CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY */ - private const val CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY = "intellij.plugins.custom.built.in.repository.url" +private val COMMON_VM_OPTIONS: List = listOf( + "-XX:JbrShrinkingGcMaxHeapFreeRatio=40", // IJPL-181469. Used in a couple with AppIdleMemoryCleaner.runGc() + "-XX:ReservedCodeCacheSize=512m", + "-XX:+HeapDumpOnOutOfMemoryError", + "-XX:-OmitStackTraceInFastThrow", + "-XX:CICompilerCount=2", + "-XX:+IgnoreUnrecognizedVMOptions", // allowing the JVM to start even with outdated options stuck in user configs + "-XX:+UnlockDiagnosticVMOptions", + "-XX:TieredOldPercentage=100000", + "-ea", + "-Dsun.io.useCanonCaches=false", + "-Dsun.java2d.metal=true", + "-Djbr.catch.SIGABRT=true", + "-Djdk.http.auth.tunneling.disabledSchemes=\"\"", + "-Djdk.attach.allowAttachSelf=true", + "-Djdk.module.illegalAccess.silent=true", + "-Djdk.nio.maxCachedBufferSize=2097152", + "-Djava.util.zip.use.nio.for.zip.file.access=true", // IJPL-149160 + "-Dkotlinx.coroutines.debug=off", +) - fun generate(context: BuildContext): List = generate( - context.applicationInfo.isEAP, - context.productProperties.customJvmMemoryOptions, - context.productProperties.additionalVmOptions.let { +fun generateVmOptions(context: BuildContext): List { + return generateVmOptions( + isEAP = context.applicationInfo.isEAP, + customVmMemoryOptions = context.productProperties.customJvmMemoryOptions, + additionalVmOptions = context.productProperties.additionalVmOptions.let { val url = computeCustomPluginRepositoryUrl(context) if (url == null) it else it + "-D${CUSTOM_BUILT_IN_PLUGIN_REPOSITORY_PROPERTY}=${url}" }, - context.productProperties.platformPrefix, + platformPrefix = context.productProperties.platformPrefix, ) +} + +internal fun generateVmOptions( + isEAP: Boolean, + customVmMemoryOptions: Map, + additionalVmOptions: List, + platformPrefix: String?, +): List { + val result = ArrayList() + + val memory = LinkedHashMap(customVmMemoryOptions) + memory.putIfAbsent("-Xms", DEFAULT_MIN_HEAP) + memory.putIfAbsent("-Xmx", DEFAULT_MAX_HEAP) + for ((k, v) in memory) { + result.add(k + v) + } - internal fun generate( - isEAP: Boolean, - customVmMemoryOptions: Map, - additionalVmOptions: List, - platformPrefix: String?, - ): List { - val result = ArrayList() - - val memory = LinkedHashMap(customVmMemoryOptions) - memory.putIfAbsent("-Xms", DEFAULT_MIN_HEAP) - memory.putIfAbsent("-Xmx", DEFAULT_MAX_HEAP) - for ((k, v) in memory) { - result += k + v - } + result.addAll(COMMON_VM_OPTIONS) - result += COMMON_VM_OPTIONS + if (isMultiRoutingFileSystemEnabledForProduct(platformPrefix)) { + result.addAll(MULTI_ROUTING_FILE_SYSTEM_VMOPTIONS) + } - if (isMultiRoutingFileSystemEnabledForProduct(platformPrefix)) { - result.addAll(MULTI_ROUTING_FILE_SYSTEM_VMOPTIONS) - } + result.addAll(additionalVmOptions) - result += additionalVmOptions + if (isEAP) { + var index = result.indexOf("-ea") + if (index < 0) index = result.indexOfFirst { it.startsWith("-D") } + if (index < 0) index = result.size + result.add(index, "-XX:MaxJavaStackTraceDepth=10000") // must be consistent with `ConfigImportHelper#updateVMOptions` + } - if (isEAP) { - var index = result.indexOf("-ea") - if (index < 0) index = result.indexOfFirst { it.startsWith("-D") } - if (index < 0) index = result.size - result.add(index, "-XX:MaxJavaStackTraceDepth=10000") // must be consistent with `ConfigImportHelper#updateVMOptions` - } - - return result - } + return result +} - private fun computeCustomPluginRepositoryUrl(context: BuildContext): String? { - val artifactsServer = context.proprietaryBuildTools.artifactsServer - if (artifactsServer != null && context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins) { - val builtinPluginsRepoUrl = artifactsServer.urlToArtifact(context, "${context.nonBundledPlugins.name}/plugins.xml") - if (builtinPluginsRepoUrl != null) { - if (builtinPluginsRepoUrl.startsWith("http:")) { - context.messages.logErrorAndThrow("Insecure artifact server: ${builtinPluginsRepoUrl}") - } - return builtinPluginsRepoUrl +private fun computeCustomPluginRepositoryUrl(context: BuildContext): String? { + val artifactsServer = context.proprietaryBuildTools.artifactsServer + if (artifactsServer != null && context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins) { + val builtinPluginsRepoUrl = artifactsServer.urlToArtifact(context, "${context.nonBundledPlugins.name}/plugins.xml") + if (builtinPluginsRepoUrl != null) { + if (builtinPluginsRepoUrl.startsWith("http:")) { + context.messages.logErrorAndThrow("Insecure artifact server: ${builtinPluginsRepoUrl}") } + return builtinPluginsRepoUrl } - return null } + return null +} - internal fun writeVmOptions(file: Path, vmOptions: Sequence, separator: String) { - Files.writeString(file, vmOptions.joinToString(separator, postfix = separator), StandardCharsets.US_ASCII) - } +internal fun writeVmOptions(file: Path, vmOptions: Sequence, separator: String) { + Files.writeString(file, vmOptions.joinToString(separator, postfix = separator), StandardCharsets.US_ASCII) } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt index 007ca16059aa..e1bf91e6b8c3 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.kt @@ -84,31 +84,31 @@ internal class WindowsDistributionBuilder( copyDir(sourceBinDir.resolve(arch.dirName), distBinDir) - copyDir(sourceBinDir, distBinDir, dirFilter = { it == sourceBinDir }) + copyDir(sourceDir = sourceBinDir, targetDir = distBinDir, dirFilter = { it == sourceBinDir }) copyFileToDir(NativeBinaryDownloader.getRestarter(context, OsFamily.WINDOWS, arch), distBinDir) generateBuildTxt(context, targetPath) - copyDistFiles(context, targetPath, OsFamily.WINDOWS, arch, WindowsLibcImpl.DEFAULT) + copyDistFiles(context = context, newDir = targetPath, os = OsFamily.WINDOWS, arch = arch, libcImpl = WindowsLibcImpl.DEFAULT) Files.writeString(distBinDir.resolve(PROPERTIES_FILE_NAME), StringUtilRt.convertLineSeparators(ideaProperties!!, "\r\n")) Files.copy(computeIcoPath(context), distBinDir.resolve("${context.productProperties.baseFileName}.ico"), StandardCopyOption.REPLACE_EXISTING) if (customizer.includeBatchLaunchers) { - generateScripts(distBinDir, arch) + generateScripts(distBinDir, arch, context) } writeVmOptions(distBinDir) - buildWinLauncher(targetPath, arch, context, copyLicense = true) + buildWinLauncher(winDistPath = targetPath, arch = arch, copyLicense = true, context = context) createFrontendContextForLaunchers(context)?.let { clientContext -> writeWindowsVmOptions(distBinDir, clientContext) - buildWinLauncher(targetPath, arch, clientContext, copyLicense = false) + buildWinLauncher(winDistPath = targetPath, arch = arch, copyLicense = false, context = clientContext) } - customizer.copyAdditionalFiles(context, targetPath, arch) + customizer.copyAdditionalFiles(targetPath, arch, context) } context.executeStep(spanBuilder("sign windows"), BuildOptions.WIN_SIGN_STEP) { @@ -184,129 +184,14 @@ internal class WindowsDistributionBuilder( Span.current().addEvent("comparing .zip and .exe is not supported on ${SystemInfoRt.OS_NAME}") } else { - checkThatExeInstallerAndZipWithJbrAreTheSame(zipWithJbrPath, exePath, arch, context.paths.tempDir) + checkThatExeInstallerAndZipWithJbrAreTheSame(zipPath = zipWithJbrPath, exePath = exePath, arch = arch, tempDir = context.paths.tempDir, context = context) } } } override suspend fun writeProductInfoFile(targetDir: Path, arch: JvmArchitecture): Path = writeProductJsonFile(targetDir, arch, context) - private fun generateScripts(distBinDir: Path, arch: JvmArchitecture) { - val fullName = context.applicationInfo.fullProductName - val baseName = context.productProperties.baseFileName - val scriptName = "${baseName}.bat" - val vmOptionsFileName = "${baseName}64.exe" - - val classPathJars = context.bootClassPathJarNames - var classPath = "SET \"CLASS_PATH=%IDE_HOME%\\lib\\${classPathJars[0]}\"" - for (i in 1 until classPathJars.size) { - classPath += "\nSET \"CLASS_PATH=%CLASS_PATH%;%IDE_HOME%\\lib\\${classPathJars[i]}\"" - } - - val additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch, isScript = true) - val winScripts = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/win/scripts") - val actualScriptNames = Files.newDirectoryStream(winScripts).use { dirStream -> dirStream.map { it.fileName.toString() }.sorted() } - - val expectedScriptNames = listOf("executable-template.bat", "format.bat", "inspect.bat", "ltedit.bat") - check(actualScriptNames == expectedScriptNames) { - "Expected script names '${expectedScriptNames.joinToString(separator = " ")}', " + - "but got '${actualScriptNames.joinToString(separator = " ")}' " + - "in ${winScripts}. Please review ${WindowsDistributionBuilder::class.java.name} and update accordingly" - } - - substituteTemplatePlaceholders( - inputFile = winScripts.resolve("executable-template.bat"), - outputFile = distBinDir.resolve(scriptName), - placeholder = "@@", - values = listOf( - Pair("product_full", fullName), - Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)), - Pair("product_vendor", context.applicationInfo.shortCompanyName), - Pair("vm_options", vmOptionsFileName), - Pair("system_selector", context.systemSelector), - Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")), - Pair("class_path", classPath), - Pair("base_name", baseName), - Pair("main_class_name", context.ideMainClassName), - ) - ) - - val inspectScript = context.productProperties.inspectCommandName - for (fileName in listOf("format.bat", "inspect.bat", "ltedit.bat")) { - substituteTemplatePlaceholders( - inputFile = winScripts.resolve(fileName), - outputFile = distBinDir.resolve(fileName), - placeholder = "@@", - values = listOf( - Pair("product_full", fullName), - Pair("script_name", scriptName), - ) - ) - } - - if (inspectScript != "inspect") { - val targetPath = distBinDir.resolve("${inspectScript}.bat") - Files.move(distBinDir.resolve("inspect.bat"), targetPath) - context.patchInspectScript(targetPath) - } - - FileSet(distBinDir) - .include("*.bat") - .enumerate() - .forEach { file -> - transformFile(file) { target -> - Files.writeString(target, toDosLineEndings(Files.readString(file))) - } - } - } - - override fun writeVmOptions(distBinDir: Path) : Path = - writeWindowsVmOptions(distBinDir, context) - - private suspend fun createBuildWinZipTask( - runtimeDir: Path?, - zipNameSuffix: String, - winDistPath: Path, - arch: JvmArchitecture, - customizer: WindowsDistributionCustomizer, - context: BuildContext - ): Path { - val baseName = context.productProperties.getBaseArtifactName(context) - val targetFile = context.paths.artifactDir.resolve("${baseName}${zipNameSuffix}.zip") - val targetFileProductInfoJson = targetFile.resolveProductInfoJsonSibling() - - spanBuilder("build Windows ${zipNameSuffix}.zip distribution") - .setAttribute("targetFile", targetFile.toString()) - .setAttribute("arch", arch.dirName) - .use { - val dirs = mutableListOf(context.paths.distAllDir, winDistPath) - - if (runtimeDir != null) { - dirs.add(runtimeDir) - } - - val productJsonDir = context.paths.tempDir.resolve("win.dist.product-info.json.zip${zipNameSuffix}") - val productJsonFile = writeProductJsonFile(targetDir = productJsonDir, arch = arch, context = context, withRuntime = runtimeDir != null) - dirs.add(productJsonDir) - copyFile(productJsonFile, targetFileProductInfoJson) - - val zipPrefix = customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber) - - val dirMap = dirs.associateWithTo(LinkedHashMap(dirs.size)) { zipPrefix } - if (context.options.compressZipFiles) { - zipWithCompression(targetFile, dirMap) - } - else { - zip(targetFile, dirMap, AddDirEntriesMode.NONE) - } - validateProductJson(targetFile, zipPrefix, context) - context.notifyArtifactBuilt(targetFile) - context.notifyArtifactBuilt(targetFileProductInfoJson) - targetFile - } - - return targetFile - } + override fun writeVmOptions(distBinDir: Path) : Path = writeWindowsVmOptions(distBinDir, context) override fun distributionFilesBuilt(arch: JvmArchitecture): List { val archSuffix = suffix(arch) @@ -322,182 +207,302 @@ internal class WindowsDistributionBuilder( } override fun isRuntimeBundled(file: Path): Boolean = !file.name.contains(customizer.zipArchiveWithoutBundledJreSuffix) +} - private suspend fun buildWinLauncher(winDistPath: Path, arch: JvmArchitecture, context: BuildContext, copyLicense: Boolean) { - spanBuilder("build Windows executable").use { - val communityHome = context.paths.communityHomeDir - val appInfo = context.applicationInfo - val executableBaseName = "${context.productProperties.baseFileName}64" - val launcherPropertiesPath = context.paths.tempDir.resolve("launcher-${arch.dirName}.properties") - val icoFile = computeIcoPath(context) +private fun generateScripts(distBinDir: Path, arch: JvmArchitecture, context: BuildContext) { + val fullName = context.applicationInfo.fullProductName + val baseName = context.productProperties.baseFileName + val scriptName = "${baseName}.bat" + val vmOptionsFileName = "${baseName}64.exe" - val productVersion = context.buildNumber.replace(".SNAPSHOT", ".0") + ".0".repeat(3 - context.buildNumber.count { it == '.' }) - val launcherProperties = listOf( - "CompanyName" to appInfo.companyName, - "LegalCopyright" to "Copyright 2000-${LocalDate.now().year} ${appInfo.companyName}", - "FileDescription" to appInfo.productNameWithEdition, - "ProductName" to appInfo.productNameWithEdition, - "ProductVersion" to "$productVersion-${appInfo.productCode}", // "242.1234.56.0-IU" + val classPathJars = context.bootClassPathJarNames + var classPath = "SET \"CLASS_PATH=%IDE_HOME%\\lib\\${classPathJars[0]}\"" + for (i in 1 until classPathJars.size) { + classPath += "\nSET \"CLASS_PATH=%CLASS_PATH%;%IDE_HOME%\\lib\\${classPathJars[i]}\"" + } + + val additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch, isScript = true) + val winScripts = context.paths.communityHomeDir.resolve("platform/build-scripts/resources/win/scripts") + val actualScriptNames = Files.newDirectoryStream(winScripts).use { dirStream -> dirStream.map { it.fileName.toString() }.sorted() } + + val expectedScriptNames = listOf("executable-template.bat", "format.bat", "inspect.bat", "ltedit.bat") + check(actualScriptNames == expectedScriptNames) { + "Expected script names '${expectedScriptNames.joinToString(separator = " ")}', " + + "but got '${actualScriptNames.joinToString(separator = " ")}' " + + "in ${winScripts}. Please review ${WindowsDistributionBuilder::class.java.name} and update accordingly" + } + + substituteTemplatePlaceholders( + inputFile = winScripts.resolve("executable-template.bat"), + outputFile = distBinDir.resolve(scriptName), + placeholder = "@@", + values = listOf( + Pair("product_full", fullName), + Pair("product_uc", context.productProperties.getEnvironmentVariableBaseName(context.applicationInfo)), + Pair("product_vendor", context.applicationInfo.shortCompanyName), + Pair("vm_options", vmOptionsFileName), + Pair("system_selector", context.systemSelector), + Pair("ide_jvm_args", additionalJvmArguments.joinToString(separator = " ")), + Pair("class_path", classPath), + Pair("base_name", baseName), + Pair("main_class_name", context.ideMainClassName), + ) + ) + + val inspectScript = context.productProperties.inspectCommandName + for (fileName in listOf("format.bat", "inspect.bat", "ltedit.bat")) { + substituteTemplatePlaceholders( + inputFile = winScripts.resolve(fileName), + outputFile = distBinDir.resolve(fileName), + placeholder = "@@", + values = listOf( + Pair("product_full", fullName), + Pair("script_name", scriptName), ) - Files.writeString(launcherPropertiesPath, launcherProperties.joinToString(separator = System.lineSeparator()) { (k, v) -> "${k}=${v}" }) + ) + } - val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.WINDOWS, arch) - val outputPath = winDistPath.resolve("bin/${executableBaseName}.exe") + if (inspectScript != "inspect") { + val targetPath = distBinDir.resolve("${inspectScript}.bat") + Files.move(distBinDir.resolve("inspect.bat"), targetPath) + context.patchInspectScript(targetPath) + } - if (copyLicense) { - copyFile(licensePath, winDistPath.resolve("license/launcher-third-party-libraries.html")) + FileSet(distBinDir) + .include("*.bat") + .enumerate() + .forEach { file -> + transformFile(file) { target -> + Files.writeString(target, toDosLineEndings(Files.readString(file))) + } + } +} + +private suspend fun createBuildWinZipTask( + runtimeDir: Path?, + zipNameSuffix: String, + winDistPath: Path, + arch: JvmArchitecture, + customizer: WindowsDistributionCustomizer, + context: BuildContext +): Path { + val baseName = context.productProperties.getBaseArtifactName(context) + val targetFile = context.paths.artifactDir.resolve("${baseName}${zipNameSuffix}.zip") + val targetFileProductInfoJson = targetFile.resolveProductInfoJsonSibling() + + spanBuilder("build Windows ${zipNameSuffix}.zip distribution") + .setAttribute("targetFile", targetFile.toString()) + .setAttribute("arch", arch.dirName) + .use { + val dirs = mutableListOf(context.paths.distAllDir, winDistPath) + + if (runtimeDir != null) { + dirs.add(runtimeDir) } - val generatorModule = context.findRequiredModule("intellij.tools.launcherGenerator") - runJava( - mainClass = "com.pme.launcher.LauncherGeneratorMain", - args = listOf( - execPath.absolutePathString(), - "${communityHome}/native/XPlatLauncher/resources/windows/resource.h", - launcherPropertiesPath.absolutePathString(), - icoFile.absolutePathString(), - outputPath.absolutePathString(), - ), - jvmArgs = listOf("-Djava.awt.headless=true"), - context.getModuleRuntimeClasspath(generatorModule, forTests = false), - context.stableJavaExecutable - ) + val productJsonDir = context.paths.tempDir.resolve("win.dist.product-info.json.zip${zipNameSuffix}") + val productJsonFile = writeProductJsonFile(targetDir = productJsonDir, arch = arch, context = context, withRuntime = runtimeDir != null) + dirs.add(productJsonDir) + copyFile(productJsonFile, targetFileProductInfoJson) + + val zipPrefix = customizer.getRootDirectoryName(context.applicationInfo, context.buildNumber) + + val dirMap = dirs.associateWithTo(LinkedHashMap(dirs.size)) { zipPrefix } + if (context.options.compressZipFiles) { + zipWithCompression(targetFile, dirMap) + } + else { + zip(targetFile, dirMap, AddDirEntriesMode.NONE) + } + validateProductJson(targetFile, zipPrefix, context) + context.notifyArtifactBuilt(targetFile) + context.notifyArtifactBuilt(targetFileProductInfoJson) + targetFile } - } - private fun computeIcoPath(context: BuildContext): Path { - val customizer = context.windowsDistributionCustomizer!! - val icoPath = (if (context.applicationInfo.isEAP) customizer.icoPathForEAP else null) ?: customizer.icoPath - requireNotNull(icoPath) { "`WindowsDistributionCustomizer#icoPath` must be set" } - return Path.of(icoPath) - } + return targetFile +} - private suspend fun checkThatExeInstallerAndZipWithJbrAreTheSame(zipPath: Path, exePath: Path, arch: JvmArchitecture, tempDir: Path) = CompareDistributionsSemaphore.withPermit { - fun compareStreams(stream1: InputStream, stream2: InputStream): Boolean { - val b1 = ByteArray(DEFAULT_BUFFER_SIZE) - val b2 = ByteArray(DEFAULT_BUFFER_SIZE) - stream1.use { s1 -> - stream2.use { s2 -> - while (true) { - val l1 = s1.readNBytes(b1, 0, b1.size) - val l2 = s2.readNBytes(b2, 0, b2.size) - if (l1 != l2) return false - if (l1 <= 0) return true - if (!Arrays.equals(b1, 0, l1, b2, 0, l2)) return false - } +private suspend fun buildWinLauncher(winDistPath: Path, arch: JvmArchitecture, copyLicense: Boolean, context: BuildContext) { + spanBuilder("build Windows executable").use { + val communityHome = context.paths.communityHomeDir + val appInfo = context.applicationInfo + val executableBaseName = "${context.productProperties.baseFileName}64" + val launcherPropertiesPath = context.paths.tempDir.resolve("launcher-${arch.dirName}.properties") + val icoFile = computeIcoPath(context) + + val productVersion = context.buildNumber.replace(".SNAPSHOT", ".0") + ".0".repeat(3 - context.buildNumber.count { it == '.' }) + val launcherProperties = listOf( + "CompanyName" to appInfo.companyName, + "LegalCopyright" to "Copyright 2000-${LocalDate.now().year} ${appInfo.companyName}", + "FileDescription" to appInfo.productNameWithEdition, + "ProductName" to appInfo.productNameWithEdition, + "ProductVersion" to "$productVersion-${appInfo.productCode}", // "242.1234.56.0-IU" + ) + Files.writeString(launcherPropertiesPath, launcherProperties.joinToString(separator = System.lineSeparator()) { (k, v) -> "${k}=${v}" }) + + val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.WINDOWS, arch) + val outputPath = winDistPath.resolve("bin/${executableBaseName}.exe") + + if (copyLicense) { + copyFile(licensePath, winDistPath.resolve("license/launcher-third-party-libraries.html")) + } + + val generatorModule = context.findRequiredModule("intellij.tools.launcherGenerator") + runJava( + mainClass = "com.pme.launcher.LauncherGeneratorMain", + args = listOf( + execPath.absolutePathString(), + "${communityHome}/native/XPlatLauncher/resources/windows/resource.h", + launcherPropertiesPath.absolutePathString(), + icoFile.absolutePathString(), + outputPath.absolutePathString(), + ), + jvmArgs = listOf("-Djava.awt.headless=true"), + context.getModuleRuntimeClasspath(generatorModule, forTests = false), + context.stableJavaExecutable + ) + } +} + +private suspend fun checkThatExeInstallerAndZipWithJbrAreTheSame( + zipPath: Path, + exePath: Path, + arch: JvmArchitecture, + tempDir: Path, + context: BuildContext, +) = CompareDistributionsSemaphore.withPermit { + fun compareStreams(stream1: InputStream, stream2: InputStream): Boolean { + val b1 = ByteArray(DEFAULT_BUFFER_SIZE) + val b2 = ByteArray(DEFAULT_BUFFER_SIZE) + stream1.use { s1 -> + stream2.use { s2 -> + while (true) { + val l1 = s1.readNBytes(b1, 0, b1.size) + val l2 = s2.readNBytes(b2, 0, b2.size) + if (l1 != l2) return false + if (l1 <= 0) return true + if (!Arrays.equals(b1, 0, l1, b2, 0, l2)) return false } } } + } - val tempExe = withContext(Dispatchers.IO) { Files.createTempDirectory(tempDir, "exe-${arch.dirName}") } - try { - withContext(Dispatchers.IO) { - spanBuilder("compare zip and exe contents") - .setAttribute("zipPath", zipPath.toString()) - .setAttribute("exePath", exePath.toString()) - .use { - runProcess(args = listOf("7z", "x", "-bd", exePath.toString()), workingDir = tempExe) - // deleting NSIS-related files that appear after manual unpacking of .exe installer and do not belong to its contents - @Suppress("SpellCheckingInspection") - NioFiles.deleteRecursively(tempExe.resolve($$"$PLUGINSDIR")) - Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe.nsis")) - Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe")) + val tempExe = withContext(Dispatchers.IO) { Files.createTempDirectory(tempDir, "exe-${arch.dirName}") } + try { + withContext(Dispatchers.IO) { + spanBuilder("compare zip and exe contents") + .setAttribute("zipPath", zipPath.toString()) + .setAttribute("exePath", exePath.toString()) + .use { + runProcess(args = listOf("7z", "x", "-bd", exePath.toString()), workingDir = tempExe) + // deleting NSIS-related files that appear after manual unpacking of .exe installer and do not belong to its contents + @Suppress("SpellCheckingInspection") + NioFiles.deleteRecursively(tempExe.resolve($$"$PLUGINSDIR")) + Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe.nsis")) + Files.deleteIfExists(tempExe.resolve("bin/Uninstall.exe")) - val extraInZip = ArrayList() - val differ = ArrayList() - ZipFile.Builder().setSeekableByteChannel(Files.newByteChannel(zipPath)).get().use { zipFile -> - zipFile.entries.asSequence() - .filter { !it.isDirectory }.toList() - .mapConcurrent(Runtime.getRuntime().availableProcessors().coerceAtLeast(4)) { entry -> - val entryPath = Path.of(entry.name) - val fileInExe = tempExe.resolve(entryPath) - if (!fileInExe.exists()) { - extraInZip.add(entryPath.toString()) + val extraInZip = ArrayList() + val differ = ArrayList() + ZipFile.Builder().setSeekableByteChannel(Files.newByteChannel(zipPath)).get().use { zipFile -> + zipFile.entries.asSequence() + .filter { !it.isDirectory }.toList() + .mapConcurrent(Runtime.getRuntime().availableProcessors().coerceAtLeast(4)) { entry -> + val entryPath = Path.of(entry.name) + val fileInExe = tempExe.resolve(entryPath) + if (!fileInExe.exists()) { + extraInZip.add(entryPath.toString()) + } + else { + if (fileInExe.fileSize() != entry.size) { + differ.add(entryPath.toString()) } - else { - if (fileInExe.fileSize() != entry.size) { + else if (entry.size < 2 * FileUtilRt.MEGABYTE) { + if (!fileInExe.readBytes().contentEquals(zipFile.getInputStream(entry).readAllBytes())) { differ.add(entryPath.toString()) } - else if (entry.size < 2 * FileUtilRt.MEGABYTE) { - if (!fileInExe.readBytes().contentEquals(zipFile.getInputStream(entry).readAllBytes())) { - differ.add(entryPath.toString()) - } - } - else if (!compareStreams(fileInExe.inputStream().buffered(FileUtilRt.MEGABYTE), zipFile.getInputStream(entry).buffered(FileUtilRt.MEGABYTE))) { - differ.add(entryPath.toString()) - } - NioFiles.deleteRecursively(fileInExe) } + else if (!compareStreams(fileInExe.inputStream().buffered(FileUtilRt.MEGABYTE), zipFile.getInputStream(entry).buffered(FileUtilRt.MEGABYTE))) { + differ.add(entryPath.toString()) + } + NioFiles.deleteRecursively(fileInExe) } - } - - val extraInExe = Files.walk(tempExe) - .filter { Files.isRegularFile(it) } - .map { tempExe.relativize(it).toString() } - .toList() - - if (extraInExe.isNotEmpty() || extraInZip.isNotEmpty() || differ.isNotEmpty()) { - error(buildString { - if (extraInZip.isNotEmpty()) { - append("Files present only in ZIP:\n") - extraInZip.forEach { append(" ").append(it).append('\n') } - } - if (extraInExe.isNotEmpty()) { - append("Files present only in EXE:\n") - extraInExe.forEach { append(" ").append(it).append('\n') } - } - if (differ.isNotEmpty()) { - append("Files with different content:\n") - differ.forEach { append(" ").append(it).append('\n') } - } - }) - } + } } - } - if (!context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) { - RepairUtilityBuilder.generateManifest(context, tempExe, OsFamily.WINDOWS, arch) - } + + val extraInExe = Files.walk(tempExe) + .filter { Files.isRegularFile(it) } + .map { tempExe.relativize(it).toString() } + .toList() + + if (extraInExe.isNotEmpty() || extraInZip.isNotEmpty() || differ.isNotEmpty()) { + error(buildString { + if (extraInZip.isNotEmpty()) { + append("Files present only in ZIP:\n") + extraInZip.forEach { append(" ").append(it).append('\n') } + } + if (extraInExe.isNotEmpty()) { + append("Files present only in EXE:\n") + extraInExe.forEach { append(" ").append(it).append('\n') } + } + if (differ.isNotEmpty()) { + append("Files with different content:\n") + differ.forEach { append(" ").append(it).append('\n') } + } + }) + } + } } - finally { - withContext(Dispatchers.IO + NonCancellable) { - NioFiles.deleteRecursively(tempExe) - } + if (!context.options.buildStepsToSkip.contains(BuildOptions.REPAIR_UTILITY_BUNDLE_STEP)) { + RepairUtilityBuilder.generateManifest(context, tempExe, OsFamily.WINDOWS, arch) } } - - private fun writeWindowsVmOptions(distBinDir: Path, context: BuildContext): Path { - val vmOptionsFile = distBinDir.resolve("${context.productProperties.baseFileName}64.exe.vmoptions") - val vmOptions = VmOptionsGenerator.generate(context).asSequence() - VmOptionsGenerator.writeVmOptions(vmOptionsFile, vmOptions, separator = "\r\n") - return vmOptionsFile - } - - private suspend fun writeProductJsonFile(targetDir: Path, arch: JvmArchitecture, context: BuildContext, withRuntime: Boolean = true): Path { - val embeddedFrontendLaunchData = generateEmbeddedFrontendLaunchData(arch, OsFamily.WINDOWS, context) { - "bin/${it.productProperties.baseFileName}64.exe.vmoptions" + finally { + withContext(Dispatchers.IO + NonCancellable) { + NioFiles.deleteRecursively(tempExe) } - val qodanaCustomLaunchData = generateQodanaLaunchData(context, arch, OsFamily.WINDOWS) - val json = generateProductInfoJson( - relativePathToBin = "bin", - builtinModules = context.builtinModule, - launch = listOf( - ProductInfoLaunchData.create( - os = OsFamily.WINDOWS.osName, - arch = arch.dirName, - launcherPath = "bin/${context.productProperties.baseFileName}64.exe", - javaExecutablePath = if (withRuntime) "jbr/bin/java.exe" else null, - vmOptionsFilePath = "bin/${context.productProperties.baseFileName}64.exe.vmoptions", - bootClassPathJarNames = context.bootClassPathJarNames, - additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch), - mainClass = context.ideMainClassName, - customCommands = listOfNotNull(embeddedFrontendLaunchData, qodanaCustomLaunchData), - ) - ), - context) - val file = targetDir.resolve(PRODUCT_INFO_FILE_NAME) - writeProductInfoJson(file, json, context) - return file } - - private fun toDosLineEndings(x: String): String = x.replace("\r", "").replace("\n", "\r\n") } + +private fun computeIcoPath(context: BuildContext): Path { + val customizer = context.windowsDistributionCustomizer!! + val icoPath = (if (context.applicationInfo.isEAP) customizer.icoPathForEAP else null) ?: customizer.icoPath + requireNotNull(icoPath) { "`WindowsDistributionCustomizer#icoPath` must be set" } + return Path.of(icoPath) +} + +private fun writeWindowsVmOptions(distBinDir: Path, context: BuildContext): Path { + val vmOptionsFile = distBinDir.resolve("${context.productProperties.baseFileName}64.exe.vmoptions") + val vmOptions = generateVmOptions(context).asSequence() + writeVmOptions(file = vmOptionsFile, vmOptions = vmOptions, separator = "\r\n") + return vmOptionsFile +} + +private suspend fun writeProductJsonFile(targetDir: Path, arch: JvmArchitecture, context: BuildContext, withRuntime: Boolean = true): Path { + val embeddedFrontendLaunchData = generateEmbeddedFrontendLaunchData(arch, OsFamily.WINDOWS, context) { + "bin/${it.productProperties.baseFileName}64.exe.vmoptions" + } + val qodanaCustomLaunchData = generateQodanaLaunchData(context, arch, OsFamily.WINDOWS) + val json = generateProductInfoJson( + relativePathToBin = "bin", + builtinModules = context.builtinModule, + launch = listOf( + ProductInfoLaunchData.create( + os = OsFamily.WINDOWS.osName, + arch = arch.dirName, + launcherPath = "bin/${context.productProperties.baseFileName}64.exe", + javaExecutablePath = if (withRuntime) "jbr/bin/java.exe" else null, + vmOptionsFilePath = "bin/${context.productProperties.baseFileName}64.exe.vmoptions", + bootClassPathJarNames = context.bootClassPathJarNames, + additionalJvmArguments = context.getAdditionalJvmArguments(OsFamily.WINDOWS, arch), + mainClass = context.ideMainClassName, + customCommands = listOfNotNull(embeddedFrontendLaunchData, qodanaCustomLaunchData), + ) + ), + context) + val file = targetDir.resolve(PRODUCT_INFO_FILE_NAME) + writeProductInfoJson(file, json, context) + return file +} + +private fun toDosLineEndings(x: String): String = x.replace("\r", "").replace("\n", "\r\n") diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt index 8084e0ead838..5d94db65d8e8 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt @@ -7,8 +7,8 @@ import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.MAVEN_REPO +import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.classPath.DescriptorSearchScope -import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.classPath.PluginBuildDescriptor import org.jetbrains.intellij.build.classPath.XIncludeElementResolverImpl import org.jetbrains.intellij.build.getUnprocessedPluginXmlContent diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt index c2f89608dd83..f4fc449b730e 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt @@ -22,8 +22,8 @@ import kotlinx.coroutines.launch import org.apache.commons.compress.archivers.zip.Zip64Mode import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.BuildOptions +import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.SearchableOptionSetDescriptor -import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.classPath.PluginBuildDescriptor import org.jetbrains.intellij.build.executeStep import org.jetbrains.intellij.build.getUnprocessedPluginXmlContent diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/productModuleLayout.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/productModuleLayout.kt index 043031c69bb8..788a045dbf43 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/productModuleLayout.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/productModuleLayout.kt @@ -10,8 +10,8 @@ import org.jdom.Element import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.ContentModuleFilter import org.jetbrains.intellij.build.FrontendModuleFilter +import org.jetbrains.intellij.build.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.classPath.DescriptorSearchScope -import org.jetbrains.intellij.build.classPath.PLUGIN_XML_RELATIVE_PATH import org.jetbrains.intellij.build.classPath.XIncludeElementResolverImpl import org.jetbrains.intellij.build.classPath.resolveAndEmbedContentModuleDescriptor import org.jetbrains.intellij.build.impl.PlatformJarNames.PRODUCT_BACKEND_JAR diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/CommunityModuleSets.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/CommunityModuleSets.kt index 17adfe8a0fc3..da99fbbda2c0 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/CommunityModuleSets.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/CommunityModuleSets.kt @@ -2,7 +2,6 @@ package org.jetbrains.intellij.build.productLayout import com.intellij.openapi.application.PathManager -import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule import org.jetbrains.intellij.build.BuildPaths import java.nio.file.Path @@ -460,15 +459,5 @@ object CommunityModuleSets : ModuleSetProvider { // Reason: Rider uses custom module loading mode due to early backend startup requirements. // Products that need rd.common include it explicitly in their product files. } -} -/** - * Represents a content module with optional loading attribute. - * - * @param name Module name - * @param loading Optional loading mode (e.g., ModuleLoadingRule.EMBEDDED) - */ -data class ContentModule( - @JvmField val name: String, - @JvmField val loading: ModuleLoadingRule? = null, -) \ No newline at end of file +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ModuleSetRunner.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ModuleSetRunner.kt new file mode 100644 index 000000000000..8841fbd76651 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ModuleSetRunner.kt @@ -0,0 +1,95 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import java.nio.file.Path + +/** + * Parses JSON argument from command line in the format --json or --json='{"filter":"...","value":"..."}'. + * Returns null for full JSON output, or JsonFilter for filtered output. + * + * @param arg The command line argument (e.g., "--json" or "--json={...}") + * @return JsonFilter if filter is specified, null for full JSON output + */ +fun parseJsonArgument(arg: String): JsonFilter? { + return if (arg.contains("=")) { + val filterJson = arg.substringAfter("=") + try { + Json.decodeFromString(filterJson) + } + catch (e: Exception) { + System.err.println("Failed to parse JSON filter: $filterJson") + System.err.println("Error: ${e.message}") + null + } + } + else { + null // Full JSON output + } +} + +/** + * Generic main runner for module set generation and analysis. + * Supports two modes: + * 1. JSON mode (--json): Outputs comprehensive analysis as JSON to stdout + * 2. Default mode: Generates XML files for module sets and products + * + * This is the generic orchestration logic that was previously duplicated in UltimateModuleSets.kt. + * Now module set providers (community, ultimate) can call this function with their specific data. + * + * @param args Command line arguments + * @param communityModuleSets Module sets from community + * @param ultimateModuleSets Module sets from ultimate (or empty for community-only) + * @param communitySourceFile Source file path for community module sets + * @param ultimateSourceFile Source file path for ultimate module sets (or null for community-only) + * @param projectRoot Project root path + * @param generateXmlImpl Lambda to generate XML files (default mode implementation) + */ +fun runModuleSetMain( + args: Array, + communityModuleSets: List, + ultimateModuleSets: List, + communitySourceFile: String, + ultimateSourceFile: String?, + projectRoot: Path, + generateXmlImpl: suspend () -> Unit +) { + // Parse --json arg with optional filter + val jsonArg = args.firstOrNull { it.startsWith("--json") } + + when { + jsonArg != null -> { + runBlocking(Dispatchers.Default) { + // Prepare all module sets with metadata + val communityModuleSetsWithMeta = communityModuleSets.map { + ModuleSetMetadata(it, "community", communitySourceFile) + } + val ultimateModuleSetsWithMeta = if (ultimateSourceFile != null) { + ultimateModuleSets.map { + ModuleSetMetadata(it, "ultimate", ultimateSourceFile) + } + } else { + emptyList() + } + val allModuleSets = communityModuleSetsWithMeta + ultimateModuleSetsWithMeta + + // Discover all products (reuse existing logic from productXmlFileGenerator) + val products = discoverAllProductsForJson(projectRoot) + + // Parse filter from --json='{"filter":"..."}' format + val filter = parseJsonArgument(jsonArg) + + // Stream JSON to stdout + streamModuleAnalysisJson(allModuleSets, products, projectRoot, filter) + } + } + else -> { + // Default mode: Generate XML files + runBlocking(Dispatchers.Default) { + generateXmlImpl() + } + } + } +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ProductModulesHelpers.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ProductModulesHelpers.kt new file mode 100644 index 000000000000..3cdd206293f6 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/ProductModulesHelpers.kt @@ -0,0 +1,106 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +/** + * Product spec helper functions for reusable product content fragments. + * These functions return ProductModulesContentSpec instances that can be composed using include(). + * + * Use these helpers to reduce duplication across product specifications while keeping + * ProductModulesContentSpec immutable and declarative (suitable for future YAML representation). + */ + +/** + * Common capability aliases shared across most ultimate products. + * Bundles frequently repeated capability declarations to reduce duplication. + * + * Includes: + * - Run targets support + * - Microservices capabilities + * - ML inline completion + * - IDE provisioner + * - Marketplace integration + * + * Usage: + * ``` + * override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules { + * include(commonCapabilityAliases()) + * // ... + * } + * ``` + */ +fun commonCapabilityAliases(): ProductModulesContentSpec = productModules { + alias("com.intellij.modules.run.targets") + alias("com.intellij.modules.microservices") + alias("com.intellij.ml.inline.completion") + alias("com.intellij.platform.ide.provisioner") + alias("com.intellij.marketplace") +} + +/** + * Python capability aliases for non-PyCharm products. + * Enables Python support in multi-language IDEs like GoLand, DataGrip, CLion, RustRover, etc. + * + * Includes: + * - Python core capabilities + * - Python in mini-IDE capabilities + * - Python in non-PyCharm IDE capabilities + * + * Usage: + * ``` + * override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules { + * include(pythonMiniIdeCapabilities()) + * // ... + * } + * ``` + */ +fun pythonMiniIdeCapabilities(): ProductModulesContentSpec = productModules { + alias("com.intellij.modules.python-core-capable") + alias("com.intellij.modules.python-in-mini-ide-capable") + alias("com.intellij.modules.python-in-non-pycharm-ide-capable") +} + +/** + * Common platform includes repeated across most ultimate products. + * Bundles deprecatedInclude statements for legacy XML resource inclusion. + * + * Includes: + * - Platform lang plugin resources + * - Structural search resources + * - Remote servers implementation + * - Ultimate edition resources + * + * Usage: + * ``` + * override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules { + * include(platformCommonIncludes()) + * // ... + * } + * ``` + */ +fun platformCommonIncludes(): ProductModulesContentSpec = productModules { + deprecatedInclude("intellij.platform.resources", "META-INF/PlatformLangPlugin.xml") + deprecatedInclude("intellij.platform.structuralSearch", "META-INF/structuralsearch.xml") + deprecatedInclude("intellij.platform.remoteServers.impl", "intellij.platform.remoteServers.impl.xml") + deprecatedInclude("intellij.platform.commercial", "META-INF/ultimate.xml") +} + +/** + * Extensions for native development IDEs (CLion, GoLand, RustRover). + * Combines process elevation support with native debugger capability. + * + * Includes: + * - Process elevation module set (for operations requiring elevated privileges) + * - Native debugger plugin capability alias + * + * Usage: + * ``` + * override fun getProductContentDescriptor(): ProductModulesContentSpec = productModules { + * include(nativeDevExtensions()) + * // ... + * } + * ``` + */ +fun nativeDevExtensions(): ProductModulesContentSpec = productModules { + moduleSet(CommunityModuleSets.elevation()) + alias("com.intellij.modules.nativeDebug-plugin-capable") +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/generator.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/generator.kt deleted file mode 100644 index e83a9609617b..000000000000 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/generator.kt +++ /dev/null @@ -1,498 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -@file:Suppress("ReplaceGetOrSet") - -package org.jetbrains.intellij.build.productLayout - -import com.intellij.openapi.util.JDOMUtil -import com.intellij.platform.plugins.parser.impl.elements.ModuleLoadingRule -import kotlinx.serialization.json.Json -import org.jetbrains.intellij.build.BuildPaths -import org.jetbrains.intellij.build.dev.createProductProperties -import org.jetbrains.intellij.build.findFileInModuleLibraryDependencies -import org.jetbrains.intellij.build.findFileInModuleSources -import org.jetbrains.intellij.build.impl.ModuleOutputProvider -import org.jetbrains.intellij.build.isModuleNameLikeFilename -import org.jetbrains.jps.model.serialization.JpsMavenSettings -import org.jetbrains.jps.model.serialization.JpsSerializationManager -import java.nio.file.Files -import java.nio.file.Path - -/** - * Generates an XML file for a module set. - * Used to maintain backward compatibility with XML-based module set loading. - * - * @param moduleSet The module set to generate XML for - * @param outputDir The directory where the XML file will be written - * @param label Description label ("community" or "ultimate") for header generation - * @return Result containing file status and statistics - */ -fun generateModuleSetXml(moduleSet: ModuleSet, outputDir: Path, label: String): ModuleSetFileResult { - val fileName = "intellij.moduleSets.${moduleSet.name}.xml" - val outputPath = outputDir.resolve(fileName) - - val buildResult = buildModuleSetXml(moduleSet, label) - - // determine change status - val status = if (Files.exists(outputPath)) { - val existingContent = Files.readString(outputPath) - if (existingContent == buildResult.xml) FileChangeStatus.UNCHANGED else FileChangeStatus.MODIFIED - } - else { - FileChangeStatus.CREATED - } - - // Only write if changed - if (status != FileChangeStatus.UNCHANGED) { - Files.writeString(outputPath, buildResult.xml) - } - - return ModuleSetFileResult(fileName, status, buildResult.directModuleCount) -} - -/** - * Represents a single content block in a product plugin.xml. - * Each block corresponds to a module set or additional modules section. - */ -data class ContentBlock( - /** Source identifier for the block (e.g., "essential", "vcs", "additional") */ - val source: String, - /** List of modules with their effective loading modes */ - val modules: List, -) - -/** - * A module with its effective loading mode after applying overrides and exclusions. - */ -data class ModuleWithLoading( - /** Module name */ - val name: String, - /** Effective loading mode (null means default/no attribute) */ - val loading: ModuleLoadingRule?, -) - -/** - * Result of building product content XML. - * Contains the generated XML string, content blocks, and module-to-set chain mapping. - */ -data class ProductContentBuildResult( - /** Generated XML content as string */ - val xml: String, - /** List of content blocks generated from the spec */ - val contentBlocks: List, - /** Mapping from module name to its module set chain as list (e.g., ["parent", "child"]) */ - val moduleToSetChainMapping: Map>, -) - -/** - * Generates complete product plugin.xml file from programmatic specification. - * - * @param isUltimateBuild Whether this is an Ultimate build (vs. Community build) - * @return Result containing file status and statistics - */ -fun generateProductXml( - pluginXmlPath: Path, - spec: ProductModulesContentSpec, - productName: String, - productPropertiesClass: String, - moduleOutputProvider: ModuleOutputProvider, - projectRoot: Path, - isUltimateBuild: Boolean, -): ProductFileResult { - // Determine which generator to recommend based on plugin.xml file location - // Community products are under community/ directory, Ultimate products are not - val generatorCommand = (if (pluginXmlPath.toString().contains("/community/")) "CommunityModuleSets" else "UltimateModuleSets") + ".main()" - - // Build complete plugin.xml file - val buildResult = buildProductContentXml( - spec = spec, - moduleOutputProvider = moduleOutputProvider, - inlineXmlIncludes = false, - inlineModuleSets = false, - productPropertiesClass = productPropertiesClass, - generatorCommand = generatorCommand, - isUltimateBuild = isUltimateBuild - ) - - // Compare with existing file if it exists - val originalContent = Files.readString(pluginXmlPath) - val status = if (originalContent == buildResult.xml) { - FileChangeStatus.UNCHANGED - } - else { - FileChangeStatus.MODIFIED - } - - // Only write if changed - if (status != FileChangeStatus.UNCHANGED) { - Files.writeString(pluginXmlPath, buildResult.xml) - } - - // Calculate statistics using the contentBlocks from generation - val totalModules = buildResult.contentBlocks.sumOf { it.modules.size } - val relativePath = projectRoot.relativize(pluginXmlPath).toString() - - return ProductFileResult( - productName = productName, - relativePath = relativePath, - status = status, - includeCount = spec.deprecatedXmlIncludes.size, - contentBlockCount = buildResult.contentBlocks.size, - totalModules = totalModules - ) -} - -/** - * Generates product XMLs for all products using programmatic content. - * Discovers products from dev-build.json and generates complete plugin.xml files. - * - * @param projectRoot The project root path - * @return Result containing generation statistics or null if dev-build.json doesn't exist - */ -suspend fun generateAllProductXmlFiles(projectRoot: Path): ProductGenerationResult { - val jsonContent = Files.readString(projectRoot.resolve(PRODUCT_REGISTRY_PATH)) - val productToConfiguration = Json.decodeFromString(jsonContent).products - - val project = JpsSerializationManager.getInstance().loadProject(projectRoot.toString(), mapOf("MAVEN_REPOSITORY" to JpsMavenSettings.getMavenRepositoryPath()), false) - val moduleOutputProvider = ModuleOutputProvider.jps(project.modules) - - // Detect if this is an Ultimate build (projectRoot != communityRoot) - val isUltimateBuild = projectRoot != BuildPaths.COMMUNITY_ROOT.communityRoot - - val productResults = mutableListOf() - for ((productName, productConfig) in productToConfiguration) { - // Skip products without pluginXmlPath configured - val pluginXmlRelativePath = productConfig.pluginXmlPath ?: continue - - val productProperties = createProductProperties( - productConfiguration = productConfig, - moduleOutputProvider = moduleOutputProvider, - projectDir = projectRoot, - platformPrefix = null - ) - val spec = productProperties.getProductContentDescriptor() ?: continue - - val pluginXmlPath = projectRoot.resolve(pluginXmlRelativePath) - val result = generateProductXml( - pluginXmlPath = pluginXmlPath, - spec = spec, - productName = productName, - moduleOutputProvider = moduleOutputProvider, - productPropertiesClass = productProperties::class.java.name, - projectRoot = projectRoot, - isUltimateBuild = isUltimateBuild, - ) - productResults.add(result) - } - - return ProductGenerationResult(productResults) -} - - -/** - * Generates content blocks and module-to-set chain mapping in a single hierarchical traversal. - * This optimized version eliminates redundant tree walking by computing both results simultaneously. - * - * @param spec The product modules specification - * @return Pair of (content blocks, module-to-set chain mapping) - */ -private fun generateContentBlocksWithChainMapping( - spec: ProductModulesContentSpec, -): Pair, Map>> { - val contentBlocks = mutableListOf() - val moduleToChain = mutableMapOf>() - val moduleToSets = mutableMapOf>() - val processedSets = HashSet() - - fun traverse(moduleSet: ModuleSet, chain: List) { - val setName = "intellij.moduleSets.${moduleSet.name}" - val currentChain = chain + setName - - // Skip if already processed - if (!processedSets.add(setName)) { - return - } - - // Get direct modules for this set - val directModules = getDirectModules(moduleSet, spec.excludedModules) - - // Build content block and track chains/duplicates in single pass - val modulesWithLoading = mutableListOf() - for (module in directModules) { - // Track for duplicate detection - moduleToSets.computeIfAbsent(module.name) { mutableListOf() }.add(moduleSet.name) - // Track chain - moduleToChain[module.name] = currentChain - // Build loading info - val effectiveLoading = spec.moduleLoadingOverrides[module.name] ?: module.loading - modulesWithLoading.add(ModuleWithLoading(module.name, effectiveLoading)) - } - - if (modulesWithLoading.isNotEmpty()) { - contentBlocks.add(ContentBlock(moduleSet.name, modulesWithLoading)) - } - - // Recursively process nested sets - for (nestedSet in moduleSet.nestedSets) { - traverse(nestedSet, currentChain) - } - } - - // Process all top-level module sets - for (moduleSet in spec.moduleSets) { - traverse(moduleSet, emptyList()) - } - - // Check for duplicates and FAIL if found - val duplicates = moduleToSets.filter { it.value.size > 1 } - if (duplicates.isNotEmpty()) { - val errorMessage = buildString { - appendLine("ERROR: Duplicate modules found across module sets:") - for ((moduleName, sets) in duplicates.toSortedMap()) { - appendLine(" - Module '$moduleName' appears in: ${sets.sorted().joinToString(", ")}") - } - appendLine() - appendLine("Each module must belong to exactly one module set.") - appendLine("Fix the module set definitions in CommunityModuleSets.kt or UltimateModuleSets.kt") - } - error(errorMessage) - } - - // Add additional modules if any - val additionalModulesWithLoading = mutableListOf() - for (module in spec.additionalModules) { - if (module.name !in spec.excludedModules) { - val effectiveLoading = spec.moduleLoadingOverrides[module.name] ?: module.loading - additionalModulesWithLoading.add(ModuleWithLoading(module.name, effectiveLoading)) - } - } - - if (additionalModulesWithLoading.isNotEmpty()) { - contentBlocks.add(ContentBlock("additional", additionalModulesWithLoading)) - } - - return Pair(contentBlocks, moduleToChain) -} - -/** - * Appends a single module XML element with optional loading attribute. - */ -private fun StringBuilder.appendModuleLine(moduleWithLoading: ModuleWithLoading, indent: String = " ") { - append("$indent on-demand) - append(" loading=\"${moduleWithLoading.loading.name.lowercase().replace('_', '-')}\"") - } - append("/>\n") -} - -/** - * Appends a content block with modules wrapped in editor fold. - */ -private fun StringBuilder.appendContentBlock( - blockSource: String, - modules: List, - indent: String = " ", -) { - append("$indent\n") - withEditorFold(this, "$indent ", blockSource) { - for (module in modules) { - appendModuleLine(module, "$indent ") - } - } - append("$indent\n") -} - -/** - * Collects and validates module aliases from the spec. - * Checks for duplicates and fails if any are found. - * - * @param spec The product modules specification - * @param inlineModuleSets Whether module sets are being inlined - * @return List of validated unique aliases - */ -private fun collectAndValidateAliases(spec: ProductModulesContentSpec, inlineModuleSets: Boolean): List { - val aliasToSource = HashMap() - - // Collect product-level aliases - for (alias in spec.productModuleAliases) { - val existing = aliasToSource.put(alias, "product level") - if (existing != null) { - error("Duplicate alias '$alias' found at product level (already defined in: $existing)") - } - } - - // When inlining module sets, also collect their aliases - if (inlineModuleSets) { - visitAllModuleSets(spec.moduleSets) { moduleSet -> - if (moduleSet.alias != null) { - val existing = aliasToSource.put(moduleSet.alias, "module set '${moduleSet.name}'") - if (existing != null) { - error("Duplicate alias '${moduleSet.alias}' in module set '${moduleSet.name}' (already defined in: $existing)") - } - } - } - } - - return aliasToSource.keys.sorted() -} - -/** - * Builds XML content for programmatic product modules. - * Generates module alias, xi:include directives (or inlined content), and `` blocks for each module set. - */ -internal fun buildProductContentXml( - spec: ProductModulesContentSpec, - moduleOutputProvider: ModuleOutputProvider, - inlineXmlIncludes: Boolean, - inlineModuleSets: Boolean, - productPropertiesClass: String, - generatorCommand: String, - isUltimateBuild: Boolean, -): ProductContentBuildResult { - // Generate content blocks and module-to-set chain mapping in single pass - val (contentBlocks, moduleToSetChainMapping) = generateContentBlocksWithChainMapping(spec) - - val xml = buildString { - // Header comments - append(" \n") - append(" \n") - append(" \n") - - // Opening tag with optional XInclude namespace - val needsXiNamespace = (!inlineXmlIncludes && spec.deprecatedXmlIncludes.isNotEmpty()) || - (!inlineModuleSets && spec.moduleSets.isNotEmpty()) - - // Check if PlatformLangPlugin.xml is included - if NOT, we need explicit id/name tags - // Products that include PlatformLangPlugin.xml inherit id/name from it - // Products without it (like Git Client) need explicit child tags - val includesPlatformLang = spec.deprecatedXmlIncludes.any { - it.resourcePath == "META-INF/PlatformLangPlugin.xml" || - it.resourcePath == "META-INF/JavaIdePlugin.xml" || - it.resourcePath == "META-INF/pycharm-core.xml" - } - - if (needsXiNamespace) { - append("\n") - } - else { - append("\n") - } - - // Add id and name as child tags if PlatformLangPlugin.xml is not included - if (!includesPlatformLang) { - append(" com.intellij\n") - append(" IDEA CORE\n") - } - - // Collect and validate aliases in a single pass - val validatedAliases = collectAndValidateAliases(spec, inlineModuleSets) - val aliasXml = buildModuleAliasesXml(validatedAliases) - if (aliasXml.isNotEmpty()) { - append(aliasXml) - append("\n") - } - - // Generate xi:include directives or inline content - if (spec.deprecatedXmlIncludes.isNotEmpty()) { - generateXIncludes(spec = spec, moduleOutputProvider = moduleOutputProvider, inlineXmlIncludes = inlineXmlIncludes, sb = this, isUltimateBuild = isUltimateBuild) - } - - // Generate module sets as xi:includes or inline content blocks - if (spec.moduleSets.isNotEmpty()) { - if (inlineModuleSets) { - // Generate single content block with all module sets inlined - append(" \n") - for ((index, block) in contentBlocks.withIndex()) { - if (block.source == "additional") continue // Skip additional modules, handle separately - withEditorFold(this, " ", block.source) { - for (module in block.modules) { - appendModuleLine(module, " ") - } - } - // Add blank line between sections for readability (except after last block) - if (index < contentBlocks.size - 1) { - append("\n") - } - } - append(" \n") - } - else { - // Generate xi:include directives for top-level module sets only (nested sets are resolved via parent includes) - for (moduleSet in spec.moduleSets) { - append(" \n") - } - } - } - - // Handle additional modules separately (they don't have XML files, inline them) - val additionalBlock = contentBlocks.firstOrNull { it.source == "additional" } - if (additionalBlock != null) { - appendContentBlock(additionalBlock.source, additionalBlock.modules) - } - - // Closing tag - append("\n") - } - - return ProductContentBuildResult(xml = xml, contentBlocks = contentBlocks, moduleToSetChainMapping = moduleToSetChainMapping) -} - -private fun generateXIncludes( - spec: ProductModulesContentSpec, - moduleOutputProvider: ModuleOutputProvider, - inlineXmlIncludes: Boolean, - sb: StringBuilder, - isUltimateBuild: Boolean, -) { - for (include in spec.deprecatedXmlIncludes) { - // When inlining: skip ultimate-only xi-includes in Community builds - if (inlineXmlIncludes && include.ultimateOnly && !isUltimateBuild) { - continue - } - - // Find the module and file - val module = moduleOutputProvider.findModule(include.moduleName) - val resourcePath = include.resourcePath - if (module == null) { - if (include.ultimateOnly) { - error("Ultimate-only module '${include.moduleName}' not found in Ultimate build - this is a configuration error (referenced in xi:include for '$resourcePath')") - } - error("Module '${include.moduleName}' not found (referenced in xi:include for '$resourcePath')") - } - - val data = findFileInModuleSources(module, resourcePath)?.let { JDOMUtil.load(it) } - ?: findFileInModuleLibraryDependencies(module = module, relativePath = resourcePath)?.let { JDOMUtil.load(it) } - ?: error("Resource '$resourcePath' not found in module '${module.name}' sources or libraries (referenced in xi:include)") - - if (inlineXmlIncludes && !include.optional) { - withEditorFold(sb, " ", "Inlined from ${include.moduleName}/$resourcePath") { - // Inline the actual XML content - for (element in data.children) { - sb.append(JDOMUtil.write(element).prependIndent(" ")) - sb.append("\n") - } - } - sb.append("\n") - } - else { - // Generate xi:include with absolute path (resources are in /META-INF/... in jars) - // Wrap ultimate-only and optional xi-includes with xi:fallback for graceful handling - if (include.ultimateOnly || include.optional) { - sb.append(""" """) - sb.append("\n") - sb.append(""" """) - sb.append("\n") - sb.append(""" """) - sb.append("\n") - } - else { - sb.append(""" """) - sb.append("\n") - } - } - } -} - -private fun resourcePathToXIncludePath(resourcePath: String): String { - return if (isModuleNameLikeFilename(resourcePath)) resourcePath else "/$resourcePath" -} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/module-sets.md b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/module-sets.md deleted file mode 100644 index 28d13f51dcab..000000000000 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/module-sets.md +++ /dev/null @@ -1,132 +0,0 @@ -# Module Sets - -Module sets are collections of modules that can be referenced as a single entity in product configurations. -All module set files follow the naming pattern: `intellij.moduleSets...xml` - -## Creating a New Module Set - -See `/create-module-set` slash command for detailed instructions on creating a new module set. - -## IDE Module Sets - -### [intellij.moduleSets.ide.common.xml](intellij.moduleSets.ide.common.xml) - -A set of product modules for regular IDE. For example, for WebStorm, but not for Fleet backend. - -This set includes [intellij.moduleSets.essential](#intellijmodulesetsessentialxml) and [intellij.moduleSets.vcs](#intellijmodulesetsvcsxml) sets. - -### [intellij.moduleSets.essential.xml](intellij.moduleSets.essential.xml) - -A set of product modules that are essential for any IDE based on IJ Platform. - -Includes [intellij.moduleSets.libraries](#intellijmodulesetslibrariesxml). - -Included in the [ide.common](#intellijmodulesetsidecommonxml) set. - -### [intellij.moduleSets.ide.ultimate.xml](intellij.moduleSets.ide.ultimate.xml) - -A set of modules common to all ultimate IDEs. - -Includes observability features (coverage, profiling), IDE infrastructure (new UI onboarding, import settings, DAP, tips, registry cloud), and Language Server Protocol support. - -Includes the [commercial](#intellijmodulesetscommercialxml) module set. - -This set is used by WebStorm, GoLand, RustRover, RubyMine, PhpStorm, CLion, DataGrip, Rider, PyCharm Pro, Aqua, and Ultimate. - -### [intellij.moduleSets.commercial.xml](intellij.moduleSets.commercial.xml) - -A set of commercial platform modules required by all JetBrains commercial IDEs. - -Includes the commercial platform module and licensing functionality. - -This set is included in the [ide.ultimate](#intellijmodulesetsideultimatexml) set. - -### [intellij.moduleSets.ide.trial.xml](intellij.moduleSets.ide.trial.xml) - -A set of trial and monetization modules for commercial IDEs without a free tier. - -Includes trial promotion and trace consent modules. - -This set is used by WebStorm, GoLand, RustRover, RubyMine, PhpStorm, CLion, and DataGrip. - -## VCS Module Sets - -### [intellij.moduleSets.vcs.xml](intellij.moduleSets.vcs.xml) - -A set of product modules for regular IDE with VCS support. -This is a separate set because, for instance, `intellij.platform.smRunner.vcs` should not be included in Rider, -yet we still want to avoid duplicating the list of VCS modules. - -This set is included in the [ide.common](#intellijmodulesetsidecommonxml) set. - -### [intellij.moduleSets.vcs.shared.xml](intellij.moduleSets.vcs.shared.xml) - -A set of VCS modules shared between different product variants. - -### [intellij.moduleSets.vcs.frontend.xml](intellij.moduleSets.vcs.frontend.xml) - -A set of VCS modules specific to frontend/client implementations. - -## Library Module Sets - -### [intellij.moduleSets.libraries.xml](intellij.moduleSets.libraries.xml) - -A set that aggregates all library module sets. This is the main entry point for including all platform libraries. - -Includes [libraries.core](#intellijmodulesetslibrariescorexml), [libraries.ktor](#intellijmodulesetslibrariesktrxml), [libraries.misc](#intellijmodulesetslibrariesmiscxml), and [libraries.temporaryBundled](#intellijmodulesetslibrariestemporarybundledxml). - -### [intellij.moduleSets.libraries.core.xml](intellij.moduleSets.libraries.core.xml) - -A set of library modules that are embedded into Core and bundled to all IDEs based on IJ Platform. - -All library modules in this file must have `loading="embedded"`. - -### [intellij.moduleSets.libraries.misc.xml](intellij.moduleSets.libraries.misc.xml) - -A set of library modules that must NOT be embedded into Core. -Plugins that require these libraries should bundle them individually. - -All libs here must not be embedded. If a library should be embedded, it should be moved to [libraries.core](#intellijmodulesetslibrariescorexml). - -### [intellij.moduleSets.libraries.ktor.xml](intellij.moduleSets.libraries.ktor.xml) - -A set of Ktor networking library modules that are embedded with the platform. -Includes ktor-io, ktor-utils, ktor-network-tls, ktor-client, and related modules. - -### [intellij.moduleSets.libraries.temporaryBundled.xml](intellij.moduleSets.libraries.temporaryBundled.xml) - -A set of library modules that are temporarily bundled with the platform but should eventually be moved elsewhere. - -## Other Module Sets - -### [intellij.moduleSets.xml.xml](intellij.moduleSets.xml.xml) - -A set of modules providing XML language support and related functionality. - -### [intellij.moduleSets.rd.common.xml](intellij.moduleSets.rd.common.xml) - -A set of common Remote Development modules. - -### [intellij.moduleSets.grid.core.xml](intellij.moduleSets.grid.core.xml) - -A set of core grid-related modules. - -### [intellij.moduleSets.elevation.xml](intellij.moduleSets.elevation.xml) - -A set of modules related to privilege elevation functionality. - -### [intellij.moduleSets.debugger.streams.xml](intellij.moduleSets.debugger.streams.xml) - -A set of debugger stream tracing modules for Ultimate Edition IDEs. - -Includes core stream tracing functionality, shared utilities, and backend integration. - -This set is used by Rider, Aqua, and Ultimate. - -### [intellij.moduleSets.ssh.xml](intellij.moduleSets.ssh.xml) - -A set of SSH-related modules for remote development and deployment features. - -Currently includes SSH UI components (`intellij.platform.ssh.ui`). - -This set is used by all commercial IDEs, PyCharm Pro, Gateway, JetBrains Client, and Ultimate editions. \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/productXmlFileGenerator.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/productXmlFileGenerator.kt new file mode 100644 index 000000000000..0fe607c4b67e --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/productLayout/productXmlFileGenerator.kt @@ -0,0 +1,209 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.productLayout + +import kotlinx.serialization.json.Json +import org.jetbrains.intellij.build.BuildPaths +import org.jetbrains.intellij.build.ProductProperties +import org.jetbrains.intellij.build.dev.createProductProperties +import org.jetbrains.intellij.build.impl.jpsModuleOutputProvider +import org.jetbrains.jps.model.serialization.JpsMavenSettings +import org.jetbrains.jps.model.serialization.JpsSerializationManager +import java.nio.file.Files +import java.nio.file.Path + +/** + * Represents a discovered product with all its metadata. + * Used by both XML and JSON generators to avoid code duplication. + * Test products have properties = null (they don't have ProductProperties classes). + */ +internal data class DiscoveredProduct( + val name: String, + val config: ProductConfiguration, + val properties: ProductProperties?, + val spec: ProductModulesContentSpec?, + val pluginXmlPath: String? +) + +/** + * Converts a DiscoveredProduct to ProductSpec for JSON output. + * Extension function that has access to ProductProperties types. + * Includes full ProductModulesContentSpec for complete DSL representation. + */ +internal fun DiscoveredProduct.toProductSpec(projectRoot: Path): ProductSpec { + // For test products (properties = null), use "test-product" as source file + val sourceFile = if (properties != null) { + getProductPropertiesSourceFile(properties.javaClass, projectRoot) + } else { + "test-product" + } + + return ProductSpec( + name = name, + className = config.className, + sourceFile = sourceFile, + pluginXmlPath = pluginXmlPath, + contentSpec = spec, // Pass full ProductModulesContentSpec for complete DSL serialization + buildModules = config.modules + ) +} + +/** + * Discovers all products from dev-build.json registry (internal representation). + * Returns DiscoveredProduct instances for internal use by XML generator. + * + * @param projectRoot The project root path + * @return List of discovered products with all metadata + */ +private suspend fun discoverAllProductsInternal(projectRoot: Path): List { + val jsonContent = Files.readString(projectRoot.resolve(PRODUCT_REGISTRY_PATH)) + val productToConfiguration = Json.decodeFromString(jsonContent).products + + val project = JpsSerializationManager.getInstance().loadProject( + projectRoot.toString(), + mapOf("MAVEN_REPOSITORY" to JpsMavenSettings.getMavenRepositoryPath()), + false + ) + val moduleOutputProvider = jpsModuleOutputProvider(project.modules) + + val products = mutableListOf() + for ((productName, productConfig) in productToConfiguration) { + val productProperties = createProductProperties( + productConfiguration = productConfig, + moduleOutputProvider = moduleOutputProvider, + projectDir = projectRoot, + platformPrefix = null + ) + val spec = productProperties.getProductContentDescriptor() + + products.add( + DiscoveredProduct( + name = productName, + config = productConfig, + properties = productProperties, + spec = spec, + pluginXmlPath = productConfig.pluginXmlPath + ) + ) + } + + return products +} + +/** + * Discovers test products from known locations. + * These are products used for testing that shouldn't be in dev-build.json. + * Retrieves contentSpec from module set registries (UltimateModuleSets). + * + * @param projectRoot The project root path + * @return List of test products as DiscoveredProduct with contentSpec + */ +private fun discoverTestProducts(projectRoot: Path): List { + // Build a registry of test product name -> (xmlPath, contentSpec) + val testProductRegistry = mutableMapOf>() + + // Get ultimate test product specs (only if ultimate directory exists) + if (Files.exists(projectRoot.resolve("ultimate"))) { + try { + // Load UltimateModuleSets via reflection to avoid hard dependency + val ultimateModuleSetsClass = Class.forName("com.intellij.platform.commercial.buildScripts.productLayout.UltimateModuleSets") + val instanceField = ultimateModuleSetsClass.getDeclaredField("INSTANCE") + val ultimateModuleSetsInstance = instanceField.get(null) + val getTestProductSpecsMethod = ultimateModuleSetsClass.getDeclaredMethod("getTestProductSpecs") + @Suppress("UNCHECKED_CAST") + val ultimateSpecs = getTestProductSpecsMethod.invoke(ultimateModuleSetsInstance) as List> + + for ((name, spec) in ultimateSpecs) { + val xmlPath = "ultimate/platform-ultimate/testResources/META-INF/${name}Plugin.xml" + testProductRegistry[name] = xmlPath to spec + } + } catch (e: ClassNotFoundException) { + // Ultimate module not available, skip ultimate test products + } + } + + // Build DiscoveredProduct instances for test products that exist on disk + return testProductRegistry.mapNotNull { (name, pair) -> + val (xmlPath, spec) = pair + val xmlFile = projectRoot.resolve(xmlPath) + if (!Files.exists(xmlFile)) return@mapNotNull null + + DiscoveredProduct( + name = name, + config = ProductConfiguration( + className = "test-product", // test products don't have Properties class + modules = emptyList(), + pluginXmlPath = xmlPath + ), + properties = null, // test products don't have ProductProperties + spec = spec, + pluginXmlPath = xmlPath + ) + } +} + +/** + * Discovers all products from dev-build.json registry and converts to ProductSpec for JSON output. + * Public function that can be called from other modules (like ultimate buildScripts). + * + * @param projectRoot The project root path + * @return List of products as ProductSpec (simple data class with no internal dependencies) + */ +suspend fun discoverAllProductsForJson(projectRoot: Path): List { + val regularProducts = discoverAllProductsInternal(projectRoot) + val testProducts = discoverTestProducts(projectRoot) + val allProducts = regularProducts + testProducts + return allProducts.map { it.toProductSpec(projectRoot) } +} + +/** + * Discovers all products for validation purposes. + * Returns list of (productName, ProductModulesContentSpec) pairs. + * + * @param projectRoot The project root path + * @return List of product name and spec pairs for validation + */ +suspend fun discoverAllProductsForValidation(projectRoot: Path): List> { + val discovered = discoverAllProductsInternal(projectRoot) + return discovered.map { it.name to it.spec } +} + +/** + * Generates product XMLs for all products using programmatic content. + * Discovers products from dev-build.json and test products, then generates complete plugin.xml files. + * + * @param projectRoot The project root path + * @return Result containing generation statistics + */ +suspend fun generateAllProductXmlFiles(projectRoot: Path): ProductGenerationResult { + val regularProducts = discoverAllProductsInternal(projectRoot) + val testProducts = discoverTestProducts(projectRoot) + val products = regularProducts + testProducts + + // Detect if this is an Ultimate build (projectRoot != communityRoot) + val isUltimateBuild = projectRoot != BuildPaths.COMMUNITY_ROOT.communityRoot + + val productResults = products.mapNotNull { discovered -> + // Skip products without pluginXmlPath or spec configured + val pluginXmlRelativePath = discovered.pluginXmlPath ?: return@mapNotNull null + val spec = discovered.spec ?: return@mapNotNull null + + val pluginXmlPath = projectRoot.resolve(pluginXmlRelativePath) + val moduleOutputProvider = jpsModuleOutputProvider( + JpsSerializationManager.getInstance() + .loadProject(projectRoot.toString(), mapOf("MAVEN_REPOSITORY" to JpsMavenSettings.getMavenRepositoryPath()), false) + .modules + ) + + generateProductXml( + pluginXmlPath = pluginXmlPath, + spec = spec, + productName = discovered.name, + moduleOutputProvider = moduleOutputProvider, + productPropertiesClass = discovered.properties?.javaClass?.name ?: "test-product", + projectRoot = projectRoot, + isUltimateBuild = isUltimateBuild, + ) + } + + return ProductGenerationResult(productResults) +} \ No newline at end of file diff --git a/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt b/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt index 72d188f5b197..26d343939a82 100644 --- a/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt +++ b/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt @@ -74,8 +74,8 @@ open class PyCharmCommunityProperties(protected val communityHome: Path) : PyCha deprecatedInclude("intellij.pycharm.community", "META-INF/pycharm-core-customization.xml") } - override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) { - super.copyAdditionalFiles(context, targetDir) + override suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) { + super.copyAdditionalFiles(targetDir, context) val licenseTargetDir = targetDir.resolve("license") copyFileToDir(context.paths.communityHomeDir.resolve("LICENSE.txt"), licenseTargetDir) @@ -98,8 +98,8 @@ open class PyCharmCommunityProperties(protected val communityHome: Path) : PyCha override fun getFullNameIncludingEdition(appInfo: ApplicationInfoProperties) = "PyCharm Community Edition" - override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path, arch: JvmArchitecture) { - super.copyAdditionalFiles(context, targetDir, arch) + override suspend fun copyAdditionalFiles(targetDir: Path, arch: JvmArchitecture, context: BuildContext) { + super.copyAdditionalFiles(targetDir, arch, context) PyCharmBuildUtils.copySkeletons(context, targetDir, "skeletons-win*.zip") } diff --git a/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.kt b/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.kt index 3ab24eb5dc62..a959df6a0b45 100644 --- a/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.kt +++ b/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmPropertiesBase.kt @@ -32,7 +32,7 @@ abstract class PyCharmPropertiesBase(enlargeWelcomeScreen: Boolean) : JetBrainsP )) } - override suspend fun copyAdditionalFiles(context: BuildContext, targetDir: Path) { + override suspend fun copyAdditionalFiles(targetDir: Path, context: BuildContext) { zipSourcesOfModules( modules = listOf("intellij.python.community", "intellij.python.psi"), targetFile = Path.of("$targetDir/lib/src/pycharm-openapi-src.zip"), diff --git a/python/ide-common/resources/META-INF/PyCharmCorePlugin.xml b/python/ide-common/resources/META-INF/PyCharmCorePlugin.xml index 6e4c4912c929..63a0098fa445 100644 --- a/python/ide-common/resources/META-INF/PyCharmCorePlugin.xml +++ b/python/ide-common/resources/META-INF/PyCharmCorePlugin.xml @@ -13,11 +13,11 @@ + - - +