From eca7a4442e1d2afca2e358657954c178a4aefa01 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Wed, 15 Feb 2023 15:41:00 +0100 Subject: [PATCH] projectLibrariesUsedByPlugins - compute only based on a bundled and plugins to publish lists, non-bundled compatible plugins are not taken in an account by intention GitOrigin-RevId: 71e49a02f815bffbc6b50989825016a5a319cf03 --- .../intellij/build/ProductModulesLayout.kt | 20 +-- .../intellij/build/impl/BuildTasksImpl.kt | 117 +++++++++++++----- .../build/impl/DistributionBuilderState.kt | 11 -- .../build/impl/JpsCompilationRunner.kt | 29 +++-- .../build/impl/MavenArtifactsBuilder.kt | 94 +++++++------- .../intellij/build/impl/PlatformModules.kt | 50 +++++--- .../pycharm/PyCharmCommunityProperties.kt | 4 +- 7 files changed, 189 insertions(+), 136 deletions(-) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/ProductModulesLayout.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/ProductModulesLayout.kt index 2f070b0257c7..c437b1e00349 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/ProductModulesLayout.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/ProductModulesLayout.kt @@ -36,15 +36,11 @@ class ProductModulesLayout { var bundledPluginModules: MutableList = DEFAULT_BUNDLED_PLUGINS.toMutableList() /** - * Names of the main modules (containing META-INF/plugin.xml) of the plugins which aren't bundled with the product but may be installed + * Main module names (containing META-INF/plugin.xml) of the plugins which aren't bundled with the product but may be installed * into it. Zip archives of these plugins will be built and placed under "<product-code>-plugins" directory in the build artifacts. * Layouts of the plugins are specified in {@link [pluginLayouts]} list. */ - var pluginModulesToPublish: Collection = LinkedHashSet() - get() = java.util.Set.copyOf(field) - set(value) { - field = LinkedHashSet(value) - } + var pluginModulesToPublish: PersistentSet = persistentSetOf() /** * Describes layout of non-trivial plugins which may be included into the product. The actual list of the plugins need to be bundled @@ -134,18 +130,6 @@ class ProductModulesLayout { * This API is experimental, use with care */ var excludedModuleNames: PersistentSet = persistentSetOf() - - /** - * @return list of all modules which output is included into the plugin's JARs - */ - fun getIncludedPluginModules(enabledPluginModules: Collection): Collection { - val result = LinkedHashSet() - result.addAll(enabledPluginModules) - pluginLayouts.asSequence() - .filter { enabledPluginModules.contains(it.mainModule) } - .flatMapTo(result) { it.includedModules.map { it.moduleName }.distinct() } - return result - } } internal fun createPluginLayoutSet(expectedSize: Int): MutableSet { diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt index d76e20d6373b..910cbf0ea459 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt @@ -81,16 +81,27 @@ class BuildTasksImpl(context: BuildContext) : BuildTasks { checkPluginModules(mainPluginModules, "mainPluginModules", context) copyDependenciesFile(context) val pluginsToPublish = getPluginLayoutsByJpsModuleNames(mainPluginModules, context.productProperties.productLayout) - val state = compilePlatformAndPluginModules( - pluginsToPublish = pluginsToPublish, - context = context, - extraModules = listOf("intellij.idea.community.build.tasks", "intellij.platform.images.build", "intellij.tools.launcherGenerator"), + val distState = createDistributionBuilderState(pluginsToPublish = pluginsToPublish, + context = context) + val compilationTasks = CompilationTasks.create(context = context) + compilationTasks.compileModules( + moduleNames = distState.getModulesForPluginsToPublish() + listOf("intellij.idea.community.build.tasks", + "intellij.platform.images.build", + "intellij.tools.launcherGenerator"), ) - buildSearchableOptions(state.platform, context) + + buildProjectArtifacts( + platform = distState.platform, + enabledPluginModules = getEnabledPluginModules(pluginsToPublish = distState.pluginsToPublish, + productProperties = context.productProperties), + compilationTasks = compilationTasks, + context = context, + ) + buildSearchableOptions(distState.platform, context) buildNonBundledPlugins(pluginsToPublish = pluginsToPublish, compressPluginArchive = context.options.compressZipFiles, buildPlatformLibJob = null, - state = state, + state = distState, context = context) } @@ -491,24 +502,12 @@ private inline fun filterSourceFilesOnly(name: String, context: BuildContext, co return sourceFiles } -private suspend fun compilePlatformAndPluginModules(pluginsToPublish: Set, - context: BuildContext, - extraModules: List = emptyList()): DistributionBuilderState { - val distState = createDistributionBuilderState(pluginsToPublish, context) - val compilationTasks = CompilationTasks.create(context) - compilationTasks.compileModules( - moduleNames = distState.getModulesForPluginsToPublish() + extraModules, - ) - compilationTasks.buildProjectArtifacts(distState.getIncludedProjectArtifacts()) - return distState -} - private suspend fun compileModulesForDistribution(context: BuildContext): DistributionBuilderState { val productProperties = context.productProperties val mavenArtifacts = productProperties.mavenArtifacts val toCompile = LinkedHashSet() - toCompile.addAll(getModulesToCompile(context)) + collectModulesToCompile(context = context, result = toCompile) context.proprietaryBuildTools.scrambleTool?.let { toCompile.addAll(it.additionalModulesToCompile) } @@ -526,6 +525,15 @@ private suspend fun compileModulesForDistribution(context: BuildContext): Distri productLayout = context.productProperties.productLayout) filterPluginsToPublish(pluginsToPublish, context) + var enabledPluginModules = getEnabledPluginModules(pluginsToPublish = pluginsToPublish, productProperties = context.productProperties) + // computed only based on a bundled and plugins to publish lists, compatible plugins are not taken in an account by intention + val projectLibrariesUsedByPlugins = computeProjectLibsUsedByPlugins(enabledPluginModules = enabledPluginModules, context = context) + val productLayout = context.productProperties.productLayout + val addPlatformCoverage = !productLayout.excludedModuleNames.contains("intellij.platform.coverage") && + hasPlatformCoverage(productLayout = productLayout, + enabledPluginModules = enabledPluginModules, + context = context) + if (context.shouldBuildDistributions()) { if (context.options.buildStepsToSkip.contains(BuildOptions.PROVIDED_MODULES_LIST_STEP)) { Span.current().addEvent("skip collecting compatible plugins because PROVIDED_MODULES_LIST_STEP was skipped") @@ -556,21 +564,49 @@ private suspend fun compileModulesForDistribution(context: BuildContext): Distri context.notifyArtifactWasBuilt(artifactPath = providedModuleFile) if (!productProperties.productLayout.buildAllCompatiblePlugins) { - return DistributionBuilderState(platform = platform, pluginsToPublish = pluginsToPublish, context = context) + val distState = DistributionBuilderState(platform = platform, pluginsToPublish = pluginsToPublish, context = context) + buildProjectArtifacts(platform = distState.platform, + enabledPluginModules = enabledPluginModules, + compilationTasks = compilationTasks, + context = context) + return distState } collectCompatiblePluginsToPublish(builtinModuleData = builtinModuleData, context = context, result = pluginsToPublish) filterPluginsToPublish(pluginsToPublish, context) + + // update enabledPluginModules to reflect changes in pluginsToPublish - used for buildProjectArtifacts + enabledPluginModules = getEnabledPluginModules(pluginsToPublish = pluginsToPublish, productProperties = context.productProperties) } } - val platform = createPlatformLayout(pluginsToPublish, context = context) + val platform = createPlatformLayout( + addPlatformCoverage = addPlatformCoverage, + projectLibrariesUsedByPlugins = projectLibrariesUsedByPlugins, + context = context, + ) val distState = DistributionBuilderState(platform = platform, pluginsToPublish = pluginsToPublish, context = context) compilationTasks.compileModules(distState.getModulesForPluginsToPublish()) - compilationTasks.buildProjectArtifacts(distState.getIncludedProjectArtifacts()) + + buildProjectArtifacts(platform = distState.platform, + enabledPluginModules = enabledPluginModules, + compilationTasks = compilationTasks, + context = context) return distState } +private fun buildProjectArtifacts(platform: PlatformLayout, + enabledPluginModules: Set, + compilationTasks: CompilationTasks, + context: BuildContext) { + val artifactNames = LinkedHashSet() + artifactNames.addAll(platform.includedArtifacts.keys) + getPluginLayoutsByJpsModuleNames(modules = enabledPluginModules, productLayout = context.productProperties.productLayout) + .flatMapTo(artifactNames) { it.includedArtifacts.keys } + + compilationTasks.buildProjectArtifacts(artifactNames) +} + suspend fun buildDistributions(context: BuildContext): Unit = spanBuilder("build distributions").useWithScope2 { checkProductProperties(context as BuildContextImpl) copyDependenciesFile(context) @@ -657,20 +693,24 @@ private fun CoroutineScope.createMavenArtifactJob(context: BuildContext, distrib } return createSkippableJob(spanBuilder("generate maven artifacts"), BuildOptions.MAVEN_ARTIFACTS_STEP, context) { - val moduleNames = ArrayList() + val moduleNames = HashSet() if (mavenArtifacts.forIdeModules) { moduleNames.addAll(distributionState.platformModules) val productLayout = context.productProperties.productLayout - moduleNames.addAll(productLayout.getIncludedPluginModules(productLayout.bundledPluginModules)) + collectIncludedPluginModules(enabledPluginModules = productLayout.bundledPluginModules, product = productLayout, result = moduleNames) } val mavenArtifactsBuilder = MavenArtifactsBuilder(context) moduleNames.addAll(mavenArtifacts.additionalModules) if (!moduleNames.isEmpty()) { - mavenArtifactsBuilder.generateMavenArtifacts(moduleNames, mavenArtifacts.squashedModules, "maven-artifacts") + mavenArtifactsBuilder.generateMavenArtifacts(moduleNamesToPublish = moduleNames, + moduleNamesToSquashAndPublish = mavenArtifacts.squashedModules, + outputDir = "maven-artifacts") } if (!mavenArtifacts.proprietaryModules.isEmpty()) { - mavenArtifactsBuilder.generateMavenArtifacts(mavenArtifacts.proprietaryModules, emptyList(), "proprietary-maven-artifacts") + mavenArtifactsBuilder.generateMavenArtifacts(moduleNamesToPublish = mavenArtifacts.proprietaryModules, + moduleNamesToSquashAndPublish = emptyList(), + outputDir = "proprietary-maven-artifacts") } } } @@ -1145,19 +1185,19 @@ private fun crossPlatformZip(macX64DistDir: Path, } } -fun getModulesToCompile(buildContext: BuildContext): Set { - val productLayout = buildContext.productProperties.productLayout - val result = LinkedHashSet() - result.addAll(productLayout.getIncludedPluginModules(java.util.Set.copyOf(productLayout.bundledPluginModules))) +fun collectModulesToCompile(context: BuildContext, result: MutableCollection) { + val productLayout = context.productProperties.productLayout + collectIncludedPluginModules(enabledPluginModules = productLayout.bundledPluginModules.toHashSet(), + product = productLayout, + result = result) collectPlatformModules(result) result.addAll(productLayout.productApiModules) result.addAll(productLayout.productImplementationModules) result.addAll(getToolModules()) - result.addAll(buildContext.productProperties.additionalModulesToCompile) + result.addAll(context.productProperties.additionalModulesToCompile) result.add("intellij.idea.community.build.tasks") result.add("intellij.platform.images.build") result.removeAll(productLayout.excludedModuleNames) - return result } // Captures information about all available inspections in a JSON format as part of an Inspectopedia project. @@ -1198,3 +1238,16 @@ internal suspend fun setLastModifiedTime(directory: Path, context: BuildContext) } } } + +/** + * @return list of all modules which output is included in the plugin's JARs + */ +internal fun collectIncludedPluginModules(enabledPluginModules: Collection, + product: ProductModulesLayout, + result: MutableCollection) { + result.addAll(enabledPluginModules) + product.pluginLayouts.asSequence() + .filter { enabledPluginModules.contains(it.mainModule) } + .flatMapTo(result) { layout -> layout.includedModules.asSequence().map { it.moduleName } } +} + diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/DistributionBuilderState.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/DistributionBuilderState.kt index 6d6e1413dd31..344625504cd0 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/DistributionBuilderState.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/DistributionBuilderState.kt @@ -35,17 +35,6 @@ class DistributionBuilderState(@JvmField val platform: PlatformLayout, fun getModulesForPluginsToPublish(): Set { return getModulesForPluginsToPublish(platform, pluginsToPublish) } - - fun getIncludedProjectArtifacts(): Set { - val result = LinkedHashSet() - result.addAll(platform.includedArtifacts.keys) - - getPluginLayoutsByJpsModuleNames(modules = getEnabledPluginModules(pluginsToPublish = pluginsToPublish, - productProperties = context.productProperties), - productLayout = context.productProperties.productLayout) - .flatMapTo(result) { it.includedArtifacts.keys } - return result - } } internal fun getModulesForPluginsToPublish(platform: PlatformLayout, pluginsToPublish: Set): Set { diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JpsCompilationRunner.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JpsCompilationRunner.kt index 8ae51e7742f4..85847a316196 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JpsCompilationRunner.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JpsCompilationRunner.kt @@ -10,6 +10,7 @@ import com.intellij.openapi.diagnostic.Logger import com.intellij.util.containers.MultiMap import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes +import io.opentelemetry.api.trace.Span import org.jetbrains.annotations.Nls import org.jetbrains.annotations.NonNls import org.jetbrains.groovy.compiler.rt.GroovyRtConstants @@ -158,8 +159,8 @@ internal class JpsCompilationRunner(private val context: CompilationContext) { val missing = artifactNames.filter { name -> artifacts.none { it.name == name } } - if (missing.isNotEmpty()) { - context.messages.error("Artifacts aren't configured in the project: " + missing.joinToString()) + check(missing.isEmpty()) { + "Artifacts aren't configured in the project: " + missing.joinToString() } artifacts.forEach { if (context.compilationData.builtArtifacts.contains(it.name) && @@ -176,12 +177,14 @@ internal class JpsCompilationRunner(private val context: CompilationContext) { val outputDir = context.getModuleOutputDir(module) if (outputDir.exists() && outputDir.isDirectory() && - outputDir.listDirectoryEntries().isNotEmpty()) false + outputDir.listDirectoryEntries().isNotEmpty()) { + false + } else { /** * See [compileMissingArtifactsModules] */ - context.messages.warning("Compilation output of module $it is missing: $outputDir") + Span.current().addEvent("Compilation output of module $it is missing: $outputDir") true } } @@ -193,11 +196,11 @@ internal class JpsCompilationRunner(private val context: CompilationContext) { resolveProjectDependencies = false) val failedToBeBuilt = artifacts.filter { if (it.outputFilePath?.let(Path::of)?.let(Files::exists) == true) { - context.messages.info("${it.name} was successfully built at ${it.outputFilePath}") + Span.current().addEvent("${it.name} was successfully built at ${it.outputFilePath}") false } else { - context.messages.warning("${it.name} is expected to be built at ${it.outputFilePath}") + Span.current().addEvent("${it.name} is expected to be built at ${it.outputFilePath}") true } } @@ -256,8 +259,9 @@ internal class JpsCompilationRunner(private val context: CompilationContext) { try { val factory = Log4jFileLoggerFactory(buildLogFile.toFile(), categoriesWithDebugLevel) JpsLoggerFactory.fileLoggerFactory = factory - context.messages.info("Build log (${if (categoriesWithDebugLevel.isEmpty()) "info" else "debug level for $categoriesWithDebugLevel"}) " + - "will be written to $buildLogFile") + context.messages.info( + "Build log (${if (categoriesWithDebugLevel.isEmpty()) "info" else "debug level for $categoriesWithDebugLevel"}) " + + "will be written to $buildLogFile") } catch (t: Throwable) { context.messages.warning("Cannot setup additional logging to $buildLogFile: ${t.message}") @@ -442,9 +446,13 @@ private class JpsMessageHandler(private val context: CompilationContext) : Messa } val buildMessages = context.messages buildMessages.info("Compilation time per target:") - val compilationTimeForTarget = compilationFinishTimeForTarget.entries.map { it.key to (it.value - compilationStartTimeForTarget.getValue(it.key)) } + val compilationTimeForTarget = compilationFinishTimeForTarget.entries.map { + it.key to (it.value - compilationStartTimeForTarget.getValue(it.key)) + } - buildMessages.info(" average: ${String.format("%.2f", ((compilationTimeForTarget.sumOf { it.second }.toDouble()) / compilationTimeForTarget.size) / 1000000)}ms") + buildMessages.info(" average: ${ + String.format("%.2f", ((compilationTimeForTarget.sumOf { it.second }.toDouble()) / compilationTimeForTarget.size) / 1000000) + }ms") val topTargets = compilationTimeForTarget.sortedBy { it.second }.asReversed().take(10) buildMessages.info(" top ${topTargets.size} targets by compilation time:") for (entry in topTargets) { @@ -516,6 +524,7 @@ private class BackedLogger(category: String?, private val fileLogger: Logger?) : } private lateinit var messageHandler: JpsMessageHandler + @Nls private const val COMPILER_NAME = "build runner" diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MavenArtifactsBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MavenArtifactsBuilder.kt index 7a86d076faa3..ae41a77aa22a 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MavenArtifactsBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/MavenArtifactsBuilder.kt @@ -3,11 +3,13 @@ package org.jetbrains.intellij.build.impl -import com.intellij.diagnostic.telemetry.useWithScope +import com.intellij.diagnostic.telemetry.useWithScope2 import com.intellij.util.text.NameUtilCore import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch import org.apache.maven.model.Dependency import org.apache.maven.model.Exclusion import org.apache.maven.model.Model @@ -30,7 +32,7 @@ import java.util.* import java.util.function.BiConsumer /** - * Generates Maven artifacts for IDE and plugin modules. Artifacts aren't generated for modules which depends on non-repository libraries. + * Generates Maven artifacts for IDE and plugin modules. Artifacts aren't generated for modules which depend on non-repository libraries. * * @see [org.jetbrains.intellij.build.ProductProperties.mavenArtifacts] * @see [org.jetbrains.intellij.build.BuildOptions.MAVEN_ARTIFACTS_STEP] @@ -63,7 +65,7 @@ open class MavenArtifactsBuilder(protected val context: BuildContext, private va val extension = JpsJavaExtensionService.getInstance().getDependencyExtension(dependency) ?: continue result.put(dependency, when (extension.scope) { JpsJavaDependencyScope.COMPILE -> - //if a dependency isn't exported transitive dependencies will include it into runtime classpath only + //if a dependency isn't exported, transitive dependencies will include it in runtime classpath only if (extension.isExported) DependencyScope.COMPILE else DependencyScope.RUNTIME JpsJavaDependencyScope.RUNTIME -> DependencyScope.RUNTIME @@ -94,17 +96,17 @@ open class MavenArtifactsBuilder(protected val context: BuildContext, private va } } - fun generateMavenArtifacts(namesOfModulesToPublish: List, - namesOfModulesToSquashAndPublish: List, - outputDir: String) { + suspend fun generateMavenArtifacts(moduleNamesToPublish: Collection, + moduleNamesToSquashAndPublish: List, + outputDir: String) { val modulesToPublish = HashMap>() - generateMavenArtifactData(namesOfModulesToPublish).forEach(BiConsumer { aModule, artifactData -> + generateMavenArtifactData(moduleNamesToPublish).forEach(BiConsumer { aModule, artifactData -> modulesToPublish.put(artifactData, listOf(aModule)) }) - val squashingMavenArtifactsData = generateMavenArtifactData(namesOfModulesToSquashAndPublish) - for (moduleName in namesOfModulesToSquashAndPublish) { + val squashingMavenArtifactsData = generateMavenArtifactData(moduleNamesToSquashAndPublish) + for (moduleName in moduleNamesToSquashAndPublish) { val module = context.findRequiredModule(moduleName) val modules = JpsJavaExtensionService.dependencies(module) .recursively().withoutSdk().includedIn(JpsJavaClasspathKind.runtime(false)).modules @@ -124,7 +126,7 @@ open class MavenArtifactsBuilder(protected val context: BuildContext, private va spanBuilder("layout maven artifacts") .setAttribute(AttributeKey.stringArrayKey("modules"), modulesToPublish.entries.map { entry -> " [${entry.value.joinToString(separator = ",") { it.name }}] -> ${entry.key.coordinates}" - }).useWithScope { + }).useWithScope2 { layoutMavenArtifacts(modulesToPublish, context.paths.artifactDir.resolve(outputDir), context) } } @@ -176,7 +178,7 @@ open class MavenArtifactsBuilder(protected val context: BuildContext, private va if (computationInProgress.contains(depModule)) { /* - It's forbidden to have compile-time circular dependencies in IntelliJ project, but there are some cycles with runtime scope + It's forbidden to have compile-time circular dependencies in the IntelliJ project, but there are some cycles with runtime scope (e.g. intellij.platform.ide.impl depends on (runtime scope) intellij.platform.configurationStore.impl which depends on intellij.platform.ide.impl). It's convenient to have such dependencies to allow running tests in classpath of their modules, so we can just ignore them while generating pom.xml files. @@ -338,46 +340,52 @@ private fun splitByCamelHumpsMergingNumbers(s: String): List { } /** - * second component of module names which describes a common group rather than a specific framework and therefore should be excluded from artifactId + * the second component of module names which describes a common group rather than a specific framework + * and therefore should be excluded from artifactId */ private val COMMON_GROUP_NAMES: Set = java.util.Set.of("platform", "vcs", "tools", "clouds") -private fun layoutMavenArtifacts(modulesToPublish: Map>, outputDir: Path, context: BuildContext) { +private suspend fun layoutMavenArtifacts(modulesToPublish: Map>, + outputDir: Path, + context: BuildContext) { val publishSourceFilter = context.productProperties.mavenArtifacts.publishSourcesFilter + coroutineScope { + for ((artifactData, modules) in modulesToPublish.entries) { + launch { + val modulesWithSources = modules.filter { + it.getSourceRoots(JavaSourceRootType.SOURCE).any() || it.getSourceRoots(JavaResourceRootType.RESOURCE).any() + } - modulesToPublish.entries.parallelStream().forEach { (artifactData, modules) -> - val modulesWithSources = modules.filter { - it.getSourceRoots(JavaSourceRootType.SOURCE).any() || it.getSourceRoots(JavaResourceRootType.RESOURCE).any() - } + val dirPath = artifactData.coordinates.directoryPath + val artifactDir = outputDir.resolve(dirPath) + Files.createDirectories(artifactDir) - val dirPath = artifactData.coordinates.directoryPath - val artifactDir = outputDir.resolve(dirPath) - Files.createDirectories(artifactDir) + generatePomXmlData(artifactData = artifactData, + file = artifactDir.resolve(artifactData.coordinates.getFileName("", "pom"))) - generatePomXmlData(artifactData = artifactData, - file = artifactDir.resolve(artifactData.coordinates.getFileName("", "pom"))) + buildJar( + targetFile = artifactDir.resolve(artifactData.coordinates.getFileName("", "jar")), + sources = modulesWithSources.map { + DirSource(dir = context.getModuleOutputDir(it), excludes = commonModuleExcludes) + }, + ) - buildJar( - targetFile = artifactDir.resolve(artifactData.coordinates.getFileName("", "jar")), - sources = modulesWithSources.map { - DirSource(dir = context.getModuleOutputDir(it), excludes = commonModuleExcludes) - }, - ) - - val publishSourcesForModules = modules.filter { publishSourceFilter(it, context) } - if (!publishSourcesForModules.isEmpty() && !modulesWithSources.isEmpty()) { - buildJar( - targetFile = artifactDir.resolve(artifactData.coordinates.getFileName("sources", "jar")), - sources = publishSourcesForModules.flatMap { module -> - module.getSourceRoots(JavaSourceRootType.SOURCE).asSequence().map { - DirSource(dir = it.path, prefix = it.properties.packagePrefix.replace('.', '/'), excludes = commonModuleExcludes) - } + - module.getSourceRoots(JavaResourceRootType.RESOURCE).asSequence().map { - DirSource(dir = it.path, prefix = it.properties.relativeOutputPath, excludes = commonModuleExcludes) - } - }, - compress = true, - ) + val publishSourcesForModules = modules.filter { publishSourceFilter(it, context) } + if (!publishSourcesForModules.isEmpty() && !modulesWithSources.isEmpty()) { + buildJar( + targetFile = artifactDir.resolve(artifactData.coordinates.getFileName("sources", "jar")), + sources = publishSourcesForModules.flatMap { module -> + module.getSourceRoots(JavaSourceRootType.SOURCE).asSequence().map { + DirSource(dir = it.path, prefix = it.properties.packagePrefix.replace('.', '/'), excludes = commonModuleExcludes) + } + + module.getSourceRoots(JavaResourceRootType.RESOURCE).asSequence().map { + DirSource(dir = it.path, prefix = it.properties.relativeOutputPath, excludes = commonModuleExcludes) + } + }, + compress = true, + ) + } + } } } } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt index 11370ffea282..c7fe7655a89b 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt @@ -18,9 +18,8 @@ import org.jetbrains.jps.model.java.JavaSourceRootType import org.jetbrains.jps.model.java.JpsJavaClasspathKind import org.jetbrains.jps.model.java.JpsJavaExtensionService import org.jetbrains.jps.model.module.JpsLibraryDependency +import org.jetbrains.jps.model.module.JpsModuleDependency import org.jetbrains.jps.model.module.JpsModuleReference -import org.jetbrains.jps.util.JpsPathUtil -import java.nio.file.Path import java.util.* private const val UTIL_JAR = "util.jar" @@ -113,9 +112,9 @@ internal fun collectPlatformModules(to: MutableCollection) { to.addAll(PLATFORM_IMPLEMENTATION_MODULES) } -private fun hasPlatformCoverage(productLayout: ProductModulesLayout, enabledPluginModules: Set, context: BuildContext): Boolean { - val modules = LinkedHashSet() - modules.addAll(productLayout.getIncludedPluginModules(enabledPluginModules)) +internal fun hasPlatformCoverage(productLayout: ProductModulesLayout, enabledPluginModules: Set, context: BuildContext): Boolean { + val modules = HashSet() + collectIncludedPluginModules(enabledPluginModules = enabledPluginModules, product = productLayout, result = modules) modules.addAll(PLATFORM_API_MODULES) modules.addAll(PLATFORM_IMPLEMENTATION_MODULES) modules.addAll(productLayout.productApiModules) @@ -126,18 +125,17 @@ private fun hasPlatformCoverage(productLayout: ProductModulesLayout, enabledPlug return true } + val javaExtensionService = JpsJavaExtensionService.getInstance() for (moduleName in modules) { - var contains = false - JpsJavaExtensionService.dependencies(context.findRequiredModule(moduleName)) - .productionOnly() - .processModules { module -> - if (!contains && module.name == coverageModuleName) { - contains = true - } + for (element in context.findRequiredModule(moduleName).dependenciesList.dependencies) { + if (element !is JpsModuleDependency || + javaExtensionService.getDependencyExtension(element)?.scope?.isIncludedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME) != true) { + continue } - if (contains) { - return true + if (element.moduleReference.moduleName == coverageModuleName) { + return true + } } } @@ -154,9 +152,22 @@ private fun addModule(relativeJarPath: String, } suspend fun createPlatformLayout(pluginsToPublish: Set, context: BuildContext): PlatformLayout { - val productLayout = context.productProperties.productLayout val enabledPluginModules = getEnabledPluginModules(pluginsToPublish = pluginsToPublish, productProperties = context.productProperties) - val projectLibrariesUsedByPlugins = computeProjectLibsUsedByPlugins(enabledPluginModules = enabledPluginModules, context = context) + val productLayout = context.productProperties.productLayout + return createPlatformLayout( + addPlatformCoverage = !productLayout.excludedModuleNames.contains("intellij.platform.coverage") && + hasPlatformCoverage(productLayout = productLayout, + enabledPluginModules = enabledPluginModules, + context = context), + projectLibrariesUsedByPlugins = computeProjectLibsUsedByPlugins(enabledPluginModules = enabledPluginModules, context = context), + context = context, + ) +} + +internal suspend fun createPlatformLayout(addPlatformCoverage: Boolean, + projectLibrariesUsedByPlugins: SortedSet, + context: BuildContext): PlatformLayout { + val productLayout = context.productProperties.productLayout val layout = PlatformLayout() // used only in modules that packed into Java layout.withoutProjectLibrary("jps-javac-extension") @@ -231,8 +242,7 @@ suspend fun createPlatformLayout(pluginsToPublish: Set, context: B explicit.addAll( toModuleItemSequence(PLATFORM_IMPLEMENTATION_MODULES, productLayout = productLayout, reason = "PLATFORM_IMPLEMENTATION_MODULES")) explicit.addAll(toModuleItemSequence(productLayout.productApiModules, productLayout = productLayout, reason = "productApiModules")) - if (hasPlatformCoverage(productLayout = productLayout, enabledPluginModules = enabledPluginModules, context = context) && - !productLayout.excludedModuleNames.contains("intellij.platform.coverage")) { + if (addPlatformCoverage) { explicit.add(ModuleItem(moduleName = "intellij.platform.coverage", relativeOutputFile = APP_JAR, reason = "coverage")) } val implicit = computeImplicitRequiredModules( @@ -269,7 +279,7 @@ suspend fun createPlatformLayout(pluginsToPublish: Set, context: B return layout } -private fun computeProjectLibsUsedByPlugins(enabledPluginModules: Set, context: BuildContext): SortedSet { +internal fun computeProjectLibsUsedByPlugins(enabledPluginModules: Set, context: BuildContext): SortedSet { val result = ObjectLinkedOpenHashSet() val jpsJavaExtensionService = JpsJavaExtensionService.getInstance() val pluginLayoutsByJpsModuleNames = getPluginLayoutsByJpsModuleNames(modules = enabledPluginModules, @@ -320,7 +330,7 @@ private fun isModuleCloseSource(moduleName: String, context: BuildContext): Bool } return sourceRoots.any { moduleSourceRoot -> - !Path.of(JpsPathUtil.urlToPath(moduleSourceRoot.url)).startsWith(context.paths.communityHomeDir) + !moduleSourceRoot.path.startsWith(context.paths.communityHomeDir) } } diff --git a/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt b/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt index b54f0cd7fe1d..e3da8a317aa9 100644 --- a/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt +++ b/python/build/src/org/jetbrains/intellij/build/pycharm/PyCharmCommunityProperties.kt @@ -1,7 +1,7 @@ // Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.pycharm -import kotlinx.collections.immutable.persistentListOf +import kotlinx.collections.immutable.persistentSetOf import org.jetbrains.intellij.build.* import org.jetbrains.intellij.build.impl.PluginLayout @@ -38,7 +38,7 @@ class PyCharmCommunityProperties(communityHome: Path) : PyCharmPropertiesBase() "intellij.jupyter.core" ) )) - productLayout.pluginModulesToPublish = persistentListOf("intellij.python.community.plugin") + productLayout.pluginModulesToPublish = persistentSetOf("intellij.python.community.plugin") } override fun copyAdditionalFilesBlocking(context: BuildContext, targetDirectory: String) {