From f83795eaffae9084ad8490ddb666aa019fc2ee35 Mon Sep 17 00:00:00 2001 From: Nikolay Chashnikov Date: Wed, 27 Aug 2025 15:37:57 +0200 Subject: [PATCH] [build scripts] generate the runtime module repository for distribution from the JPS model directly (IJPL-201730) It doesn't use the runtime module repository created for compiled modules created during JPS build anymore. This reduces the time required to build it because only modules included in the product distribution are processed now. IJ-MR-174028 GitOrigin-RevId: bda1edac26e696bef45a0100b9840d69760b670f --- .../build/impl/FrontendModuleFilterImpl.kt | 9 + .../impl/runtimeModuleRepositoryGenerator.kt | 223 ++++++++++-------- 2 files changed, 130 insertions(+), 102 deletions(-) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/FrontendModuleFilterImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/FrontendModuleFilterImpl.kt index 006fae6eed67..ce78335b08da 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/FrontendModuleFilterImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/FrontendModuleFilterImpl.kt @@ -7,6 +7,9 @@ import com.intellij.platform.runtime.product.impl.ProductModeMatcher import com.intellij.platform.runtime.repository.RuntimeModuleId import com.intellij.platform.runtime.repository.RuntimeModuleRepository import org.jetbrains.intellij.build.FrontendModuleFilter +import org.jetbrains.jps.model.JpsNamedElement +import org.jetbrains.jps.model.library.JpsLibrary +import org.jetbrains.jps.model.module.JpsModule internal class FrontendModuleFilterImpl(private val moduleRepository: RuntimeModuleRepository, productModules: ProductModules): FrontendModuleFilter { private val includedModules: Set = (sequenceOf(productModules.mainModuleGroup) + productModules.bundledPluginModuleGroups.asSequence()) @@ -48,6 +51,12 @@ val MODULES_SCRAMBLED_WITH_FRONTEND: Set by lazy { PROJECT_LIBRARIES_SCRAMBLED_WITH_FRONTEND.map { RuntimeModuleId.projectLibrary(it) } } +fun isScrambledWithFrontend(element: JpsNamedElement): Boolean = when (element) { + is JpsModule -> element.name == PLATFORM_MODULE_SCRAMBLED_WITH_FRONTEND + is JpsLibrary -> element.name in PROJECT_LIBRARIES_SCRAMBLED_WITH_FRONTEND + else -> false +} + internal object EmptyFrontendModuleFilter : FrontendModuleFilter { override fun isModuleIncluded(moduleName: String): Boolean = false override fun isProjectLibraryIncluded(libraryName: String): Boolean = false diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/runtimeModuleRepositoryGenerator.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/runtimeModuleRepositoryGenerator.kt index cb2839cfedde..5898d3645f9f 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/runtimeModuleRepositoryGenerator.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/runtimeModuleRepositoryGenerator.kt @@ -1,6 +1,7 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl +import com.intellij.devkit.runtimeModuleRepository.generator.ResourcePathsSchema import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleRepositoryGenerator import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleRepositoryGenerator.COMPACT_REPOSITORY_FILE_NAME import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleRepositoryGenerator.JAR_REPOSITORY_FILE_NAME @@ -8,7 +9,6 @@ import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleReposi import com.intellij.openapi.util.text.StringUtil import com.intellij.platform.runtime.repository.RuntimeModuleId import com.intellij.platform.runtime.repository.serialization.RawRuntimeModuleDescriptor -import com.intellij.platform.runtime.repository.serialization.RawRuntimeModuleRepositoryData import com.intellij.platform.runtime.repository.serialization.RuntimeModuleRepositorySerialization import com.intellij.util.containers.MultiMap import io.opentelemetry.api.trace.Span @@ -16,13 +16,17 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.jetbrains.annotations.ApiStatus import org.jetbrains.intellij.build.BuildContext -import org.jetbrains.intellij.build.impl.projectStructureMapping.CustomAssetEntry import org.jetbrains.intellij.build.impl.projectStructureMapping.DistributionFileEntry import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleLibraryFileEntry import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleOutputEntry import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleTestOutputEntry import org.jetbrains.intellij.build.impl.projectStructureMapping.ProjectLibraryEntry +import org.jetbrains.jps.model.JpsNamedElement +import org.jetbrains.jps.model.java.JavaModuleSourceRootTypes +import org.jetbrains.jps.model.java.JpsJavaExtensionService +import org.jetbrains.jps.model.library.JpsLibrary import org.jetbrains.jps.model.library.JpsOrderRootType +import org.jetbrains.jps.model.module.JpsModule import java.nio.file.Path import kotlin.io.path.Path import kotlin.io.path.exists @@ -38,8 +42,6 @@ import kotlin.io.path.pathString * resources should be included in the distribution, instead of taking this information from the project model. */ internal suspend fun generateRuntimeModuleRepository(entries: Sequence, context: BuildContext) { - val compiledModulesDescriptors = context.getOriginalModuleRepository().rawRepositoryData - val repositoryEntries = ArrayList() val osSpecificDistPaths = listOf(null to context.paths.distAllDir) + SUPPORTED_DISTRIBUTIONS.map { it to getOsAndArchSpecificDistDirectory(osFamily = it.os, arch = it.arch, libc = it.libcImpl, context = context) } @@ -54,7 +56,6 @@ internal suspend fun generateRuntimeModuleRepository(entries: Sequence, targetDirectory: Path, context: BuildContext) { - val compiledModulesDescriptors = context.getOriginalModuleRepository().rawRepositoryData val actualEntries = entries.map { entry -> RuntimeModuleRepositoryEntry( distribution = null, @@ -89,7 +88,6 @@ suspend fun generateRuntimeModuleRepositoryForDevBuild(entries: Sequence, - compiledModulesDescriptorsData: RawRuntimeModuleRepositoryData, context: BuildContext ) { val mainPathsForResources = computeMainPathsForResourcesCopiedToMultiplePlaces(entries, context) - val resourcePathMapping = MultiMap.createOrderedSet() + fun isMainPath(element: JpsNamedElement, path: String): Boolean { + val mainPath = mainPathsForResources[element] + return mainPath == null || mainPath == path + } + + val moduleProductionPaths = MultiMap.createOrderedSet() + val moduleTestPaths = MultiMap.createOrderedSet() + val libraryPaths = MultiMap.createOrderedSet() for (entry in entries) { - val moduleId = entry.origin.getRuntimeModuleId() ?: continue - val mainPath = mainPathsForResources[moduleId] - if (mainPath == null || mainPath == entry.relativePath) { - resourcePathMapping.putValue(moduleId, entry.relativePath) + when (entry.origin) { + is ModuleOutputEntry -> { + val module = context.findRequiredModule(entry.origin.moduleName) + if (isMainPath(module, entry.relativePath)) { + moduleProductionPaths.putValue(module, entry.relativePath) + } + } + is ModuleTestOutputEntry -> { + moduleTestPaths.putValue(context.findRequiredModule(entry.origin.moduleName), entry.relativePath) + } + is ProjectLibraryEntry -> { + val library = context.project.libraryCollection.findLibrary(entry.origin.data.libraryName) ?: error("Cannot find project-level library '${entry.origin.data.libraryName}'") + if (isMainPath(library, entry.relativePath)) { + libraryPaths.putValue(library, entry.relativePath) + } + } + is ModuleLibraryFileEntry -> { + val library = entry.origin.findLibrary(context) + if (isMainPath(library, entry.relativePath)) { + libraryPaths.putValue(library, entry.relativePath) + } + } + else -> { + continue + } } } - val compiledModulesDescriptors = compiledModulesDescriptorsData.allIds.associateBy( - keySelector = { RuntimeModuleId.raw(it) }, - valueTransform = { compiledModulesDescriptorsData.findDescriptor(it)!! }, - ) - addMappingsForDuplicatingLibraries(resourcePathMapping, compiledModulesDescriptors) - - val transitiveDependencies = LinkedHashSet() - collectTransitiveDependencies(resourcePathMapping.keySet(), compiledModulesDescriptors, transitiveDependencies) - - val distDescriptors = ArrayList() - for ((moduleId, resourcePaths) in resourcePathMapping.entrySet()) { - val descriptor = compiledModulesDescriptors[moduleId] - if (descriptor == null) { - context.messages.warning("Descriptor for '$moduleId' isn't found in module repository ${compiledModulesDescriptorsData.basePath}") - continue - } + addMappingForModulesWithoutResources(moduleProductionPaths) + addMappingsForDuplicatingLibraries(libraryPaths, moduleProductionPaths) + val distDescriptors = RuntimeModuleRepositoryGenerator.generateRuntimeModuleDescriptors( + includedProduction = moduleProductionPaths.keySet(), + includedTests = moduleTestPaths.keySet(), + includedLibraries = libraryPaths.keySet(), + resourcePathsSchema = DistributionResourcePathsSchema(moduleProductionPaths, moduleTestPaths, libraryPaths), + ).map { descriptor -> //this is a temporary workaround to skip optional dependencies which aren't included in the distribution - val dependenciesToSkip = dependenciesToSkip[descriptor.id] ?: emptySet() - - val actualDependencies = descriptor.dependencies.mapNotNull { dependency -> - when (dependency) { - in dependenciesToSkip -> null - else -> dependency - } - } - val actualResourcePaths = resourcePaths.mapTo(ArrayList()) { - if (it.startsWith("$RUNTIME_REPOSITORY_MODULES_DIR_NAME/")) it.removePrefix("$RUNTIME_REPOSITORY_MODULES_DIR_NAME/") else "../$it" - } - distDescriptors.add(RawRuntimeModuleDescriptor.create(moduleId.stringId, actualResourcePaths, actualDependencies)) - } - - /* include descriptors of aggregating modules which don't have own resources (and therefore don't have DistributionFileEntry), - but used from other modules */ - for (dependencyId in transitiveDependencies) { - if (!resourcePathMapping.containsKey(dependencyId)) { - val descriptor = compiledModulesDescriptors[dependencyId] - if (descriptor != null && descriptor.resourcePaths.isEmpty()) { - distDescriptors.add(descriptor) - } - } + val dependenciesToSkip = dependenciesToSkip[descriptor.id] ?: return@map descriptor + val actualDependencies = descriptor.dependencies.filterNot { it in dependenciesToSkip} + RawRuntimeModuleDescriptor.create(descriptor.id, descriptor.resourcePaths, actualDependencies) } val errors = ArrayList() @@ -210,6 +207,20 @@ private suspend fun generateRepositoryForDistribution( } } +private class DistributionResourcePathsSchema( + private val moduleProductionPaths: MultiMap, + private val moduleTestPaths: MultiMap, + private val libraryPaths: MultiMap +) : ResourcePathsSchema { + override fun moduleOutputPaths(module: JpsModule): List = moduleProductionPaths.get(module).convertToRelative() + override fun moduleTestOutputPaths(module: JpsModule): List = moduleTestPaths.get(module).convertToRelative() + override fun libraryPaths(library: JpsLibrary): List = libraryPaths.get(library).convertToRelative() + + private fun Collection.convertToRelative(): List = map { + if (it.startsWith("$RUNTIME_REPOSITORY_MODULES_DIR_NAME/")) it.removePrefix("$RUNTIME_REPOSITORY_MODULES_DIR_NAME/") else "../$it" + } +} + /** * Some libraries and modules are copied to multiple places in the distribution. * In order to decide which location should be specified in the runtime descriptor, this method determines the main location used the @@ -224,8 +235,9 @@ private suspend fun generateRepositoryForDistribution( private suspend fun computeMainPathsForResourcesCopiedToMultiplePlaces( entries: List, context: BuildContext, -): Map { - val singleFileProjectLibraries = context.project.libraryCollection.libraries.asSequence() +): Map { + val project = context.project + val singleFileProjectLibraries = project.libraryCollection.libraries.asSequence() .filter { it.getFiles(JpsOrderRootType.COMPILED).size == 1 } .mapTo(HashSet()) { it.name } @@ -235,28 +247,28 @@ private suspend fun computeMainPathsForResourcesCopiedToMultiplePlaces( || projectLibraryEntry.data.packMode == LibraryPackMode.STANDALONE_MERGED) } - fun ModuleLibraryFileEntry.isPackedIntoSingleJar(): Boolean { - val library = context.findRequiredModule(moduleName).libraryCollection.libraries.find { getLibraryFilename(it) == libraryName } - require(library != null) { "Cannot find module-level library '$libraryName' in '$moduleName'" } - return library.getFiles(JpsOrderRootType.COMPILED).size == 1 - } - val pathToEntries = entries.groupBy { Path(it.relativePath) } //exclude libraries which may be packed in multiple JARs from consideration, because multiple entries may not indicate that a library is copied to multiple places in such cases, //and all resource roots should be kept - val moduleIdsToPaths = entries.asSequence() - .filter { entry -> entry.origin is ProjectLibraryEntry && isPackedIntoSingleJar(entry.origin) - || entry.origin is ModuleLibraryFileEntry && entry.origin.isPackedIntoSingleJar() - || entry.origin is ModuleOutputEntry } - .groupBy({ it.origin.getRuntimeModuleId()!! }, { Path(it.relativePath) }) + val projectElementToPaths = entries.asSequence() + .mapNotNull { entry -> + val element = when (entry.origin) { + is ProjectLibraryEntry if isPackedIntoSingleJar(entry.origin) -> project.libraryCollection.findLibrary(entry.origin.data.libraryName) + is ModuleLibraryFileEntry -> entry.origin.findLibrary(context).takeIf { it.getFiles(JpsOrderRootType.COMPILED).size == 1 } + is ModuleOutputEntry -> context.findRequiredModule(entry.origin.moduleName) + else -> null + } + element?.let { it to entry.relativePath } + } + .groupBy({ it.first }, { Path(it.second) }) suspend fun isIncludedInEmbeddedFrontend(entry: DistributionFileEntry): Boolean { return entry is ModuleOutputEntry && context.getFrontendModuleFilter().isModuleIncluded(entry.moduleName) } - suspend fun chooseMainLocation(moduleId: RuntimeModuleId, paths: List): String { - val mainLocation = paths.singleOrNull { it.parent?.pathString == "lib" && moduleId !in MODULES_SCRAMBLED_WITH_FRONTEND } ?: + suspend fun chooseMainLocation(element: JpsNamedElement, paths: List): String { + val mainLocation = paths.singleOrNull { it.parent?.pathString == "lib" && !isScrambledWithFrontend(element) } ?: paths.singleOrNull { pathToEntries[it]?.size == 1 } ?: paths.singleOrNull { pathToEntries[it]?.any { entry -> isIncludedInEmbeddedFrontend(entry.origin) } == true } ?: paths.singleOrNull { it.parent?.name in setOf("client", "frontend", "frontend-split") } @@ -264,44 +276,61 @@ private suspend fun computeMainPathsForResourcesCopiedToMultiplePlaces( return mainLocation.invariantSeparatorsPathString } val sorted = paths.map { it.invariantSeparatorsPathString }.sorted() - Span.current().addEvent("cannot choose the main location for '${moduleId.stringId}' among $sorted, the first one will be used") + Span.current().addEvent("cannot choose the main location for '${element.name}' among $sorted, the first one will be used") return sorted.first() } - val mainPaths = HashMap() - for ((moduleId, paths) in moduleIdsToPaths) { + val mainPaths = HashMap() + for ((element, paths) in projectElementToPaths) { val distinctPaths = paths.distinct() if (distinctPaths.size > 1) { - mainPaths[moduleId] = chooseMainLocation(moduleId, distinctPaths) + mainPaths[element] = chooseMainLocation(element, distinctPaths) } } return mainPaths } +private fun ModuleLibraryFileEntry.findLibrary(context: BuildContext): JpsLibrary { + val library = context.findRequiredModule(moduleName).libraryCollection.libraries.find { getLibraryFilename(it) == libraryName } + require(library != null) { "Cannot find module-level library '$libraryName' in '$moduleName'" } + return library +} + +/** + * Include descriptors of aggregating modules which don't have own resources (and therefore don't have DistributionFileEntry), but used from other modules + */ +private fun addMappingForModulesWithoutResources(moduleProductionPaths: MultiMap) { + val originalModules = moduleProductionPaths.keySet().toList() + JpsJavaExtensionService.getInstance().enumerateDependencies(originalModules).runtimeOnly().productionOnly().withoutSdk().withoutLibraries().recursively().processModules { module -> + if (!moduleProductionPaths.containsKey(module) && module.sourceRoots.none { it.rootType in JavaModuleSourceRootTypes.PRODUCTION }) { + moduleProductionPaths.putValues(module, emptySet()) + } + } +} + /** * Adds mappings for libraries which aren't explicitly included in the distribution, but their JARs are included as part of other libraries. */ -private fun addMappingsForDuplicatingLibraries(resourcePathMapping: MultiMap, - compiledModulesDescriptors: Map) { - val transitiveDependencies = LinkedHashSet() - collectTransitiveDependencies(resourcePathMapping.keySet(), compiledModulesDescriptors, transitiveDependencies) - - val descriptorsByResource = HashMap>() - compiledModulesDescriptors.values.forEach { descriptor -> - descriptor.resourcePaths.groupByTo(descriptorsByResource, { it }) { descriptor } +private fun addMappingsForDuplicatingLibraries(libraryPaths: MultiMap, moduleProductionPaths: MultiMap) { + val librariesFromDependencies = HashSet(libraryPaths.keySet()) + JpsJavaExtensionService.getInstance().enumerateDependencies(moduleProductionPaths.keySet()).withoutSdk().runtimeOnly().productionOnly().recursively().processLibraries { + librariesFromDependencies.add(it) } - val includedInMapping = resourcePathMapping.keySet().toMutableSet() - for ((moduleId, resourcePathsInDist) in resourcePathMapping.entrySet().toList()) { - val includedDescriptor = compiledModulesDescriptors[moduleId] - includedDescriptor?.resourcePaths?.forEach { resourcePath -> - descriptorsByResource[resourcePath]?.forEach { anotherDescriptor -> - val anotherId = RuntimeModuleId.raw(anotherDescriptor.id) - if (anotherId.stringId != includedDescriptor.id - && anotherId in transitiveDependencies - && includedDescriptor.resourcePaths.containsAll(anotherDescriptor.resourcePaths) - && (includedDescriptor.resourcePaths == anotherDescriptor.resourcePaths || resourcePathsInDist.size == 1) - && includedInMapping.add(anotherId)) { - resourcePathMapping.putValues(anotherId, resourcePathsInDist) + + val librariesByPath = HashMap>() + librariesFromDependencies.forEach { library -> + library.getPaths(JpsOrderRootType.COMPILED).groupByTo(librariesByPath, { it }) { library } + } + val originalEntries = libraryPaths.entrySet().toList() + for ((library, resourcePathsInDist) in originalEntries) { + val libraryRoots = library.getPaths(JpsOrderRootType.COMPILED) + libraryRoots.forEach { resourcePath -> + librariesByPath[resourcePath]?.forEach { anotherLibrary -> + if (anotherLibrary != library + && !libraryPaths.containsKey(anotherLibrary) + && libraryRoots.containsAll(anotherLibrary.getPaths(JpsOrderRootType.COMPILED)) + && (libraryRoots == anotherLibrary.getPaths(JpsOrderRootType.COMPILED) || resourcePathsInDist.size == 1)) { + libraryPaths.putValues(anotherLibrary, resourcePathsInDist) } } } @@ -320,16 +349,6 @@ private fun collectTransitiveDependencies(moduleIds: Collection } } -private fun DistributionFileEntry.getRuntimeModuleId(): RuntimeModuleId? { - return when (this) { - is ModuleOutputEntry -> RuntimeModuleId.module(moduleName) - is ModuleTestOutputEntry -> RuntimeModuleId.moduleTests(moduleName) - is ModuleLibraryFileEntry -> RuntimeModuleId.moduleLibrary(moduleName, libraryName) - is ProjectLibraryEntry -> RuntimeModuleId.projectLibrary(data.libraryName) - is CustomAssetEntry -> null - } -} - internal const val RUNTIME_REPOSITORY_MODULES_DIR_NAME = "modules" internal const val MODULE_DESCRIPTORS_JAR_PATH: String = "$RUNTIME_REPOSITORY_MODULES_DIR_NAME/$JAR_REPOSITORY_FILE_NAME" const val MODULE_DESCRIPTORS_COMPACT_PATH: String = "$RUNTIME_REPOSITORY_MODULES_DIR_NAME/$COMPACT_REPOSITORY_FILE_NAME"