From 2d8a1208d6add088bda3c675235f47e9bc2dca7b Mon Sep 17 00:00:00 2001 From: Nikolay Chashnikov Date: Tue, 3 Feb 2026 17:49:24 +0100 Subject: [PATCH] IJPL-201068 runtime module repository: include paths to module-level libraries to classpath of corresponding modules Before, module-level libraries were translated to separate RuntimeModuleDescriptor in the runtime module repository, and dependencies on them were added to RuntimeModuleDescriptor instance corresponding to the JPS module. However, it didn't work well when IDE is started from source code, and output of the content module isn't merged with the module's libraries, causing problems like IJPL-201068 and IJPL-233281. Also, it greatly increased number of elements in the runtime module repository. Now paths to library JARs are added to the module classpath, and no separate RuntimeModuleDescriptor instances are created for module-level libraries. This will also make it simpler to reuse the runtime module repository for computing dependencies of modules at runtime (IJPL-157803). GitOrigin-RevId: ce62725e98a8202bd80afaa3639f1c53fd50521c --- .../RuntimeModuleRepositoryForDistribution.kt | 3 +- .../runtime/repository/RuntimeModuleId.java | 5 +- .../RuntimeModuleRepositoryForTests.kt | 5 +- .../src/RuntimeModuleRepositoryGenerator.kt | 160 ++++++++++++------ .../RuntimeModuleRepositoryBuilderTest.kt | 22 ++- 5 files changed, 138 insertions(+), 57 deletions(-) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/RuntimeModuleRepositoryForDistribution.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/RuntimeModuleRepositoryForDistribution.kt index 9e20562f45e4..3c1ebaf89315 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/RuntimeModuleRepositoryForDistribution.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/RuntimeModuleRepositoryForDistribution.kt @@ -6,6 +6,7 @@ import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleReposi import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleRepositoryGenerator.COMPACT_REPOSITORY_FILE_NAME import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleRepositoryGenerator.JAR_REPOSITORY_FILE_NAME import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleRepositoryValidator +import com.intellij.devkit.runtimeModuleRepository.generator.isProjectLevel import com.intellij.openapi.util.text.StringUtil import com.intellij.platform.runtime.repository.RuntimeModuleId import com.intellij.platform.runtime.repository.serialization.RawRuntimeModuleDescriptor @@ -183,7 +184,7 @@ private suspend fun generateRepositoryForDistribution( val distDescriptors = RuntimeModuleRepositoryGenerator.generateRuntimeModuleDescriptors( includedProduction = moduleProductionPaths.keySet(), includedTests = moduleTestPaths.keySet(), - includedLibraries = libraryPaths.keySet(), + includedProjectLibraries = libraryPaths.keySet().filter { it.isProjectLevel }, resourcePathsSchema = DistributionResourcePathsSchema(moduleProductionPaths, moduleTestPaths, libraryPaths), ).map { descriptor -> //this is a temporary workaround to skip optional dependencies which aren't included in the distribution diff --git a/platform/runtime/repository/src/com/intellij/platform/runtime/repository/RuntimeModuleId.java b/platform/runtime/repository/src/com/intellij/platform/runtime/repository/RuntimeModuleId.java index b0c3a0390034..ae05f1d8cccf 100644 --- a/platform/runtime/repository/src/com/intellij/platform/runtime/repository/RuntimeModuleId.java +++ b/platform/runtime/repository/src/com/intellij/platform/runtime/repository/RuntimeModuleId.java @@ -57,9 +57,10 @@ public final class RuntimeModuleId { } /** - * Creates ID of a runtime module corresponding to the module-level library {@code libraryName} from module {@code moduleName} in intellij - * project configuration. + * @deprecated module-level libraries are now merged with corresponding modules at runtime, it doesn't make sense to have separate IDs for + * them. */ + @Deprecated(forRemoval = true) public static @NotNull RuntimeModuleId moduleLibrary(@NotNull String moduleName, @NotNull String libraryName) { return new RuntimeModuleId(LIB_NAME_PREFIX + moduleName + "." + libraryName); } diff --git a/platform/testFramework/monorepo/tests/runtimeModuleRepository/RuntimeModuleRepositoryForTests.kt b/platform/testFramework/monorepo/tests/runtimeModuleRepository/RuntimeModuleRepositoryForTests.kt index 010daf2fc7bf..431a515dffae 100644 --- a/platform/testFramework/monorepo/tests/runtimeModuleRepository/RuntimeModuleRepositoryForTests.kt +++ b/platform/testFramework/monorepo/tests/runtimeModuleRepository/RuntimeModuleRepositoryForTests.kt @@ -3,6 +3,7 @@ package com.intellij.platform.testFramework.monorepo.runtimeModuleRepository import com.intellij.devkit.runtimeModuleRepository.generator.ResourcePathsSchema import com.intellij.devkit.runtimeModuleRepository.generator.RuntimeModuleRepositoryGenerator +import com.intellij.devkit.runtimeModuleRepository.generator.isProjectLevel import com.intellij.platform.runtime.repository.RuntimeModuleRepository import com.intellij.platform.runtime.repository.impl.RuntimeModuleRepositoryImpl import com.intellij.platform.runtime.repository.serialization.RawRuntimeModuleRepositoryData @@ -31,11 +32,11 @@ fun generateRuntimeModuleRepositoryForTests(monorepoProject: JpsProject): Runtim } } - val allLibraries = JpsJavaExtensionService.dependencies(monorepoProject).productionOnly().runtimeOnly().libraries + val allProjectLibraries = JpsJavaExtensionService.dependencies(monorepoProject).productionOnly().runtimeOnly().libraries.filter { it.isProjectLevel } val moduleDescriptors = RuntimeModuleRepositoryGenerator.generateRuntimeModuleDescriptors( includedProduction = monorepoProject.modules, includedTests = emptyList(), - includedLibraries = allLibraries, + includedProjectLibraries = allProjectLibraries, resourcePathsSchema = ResourcePathsSchemaForTests, ) diff --git a/plugins/devkit/runtimeModuleRepository/generator/src/RuntimeModuleRepositoryGenerator.kt b/plugins/devkit/runtimeModuleRepository/generator/src/RuntimeModuleRepositoryGenerator.kt index 2a7757d813eb..499a876d6caf 100644 --- a/plugins/devkit/runtimeModuleRepository/generator/src/RuntimeModuleRepositoryGenerator.kt +++ b/plugins/devkit/runtimeModuleRepository/generator/src/RuntimeModuleRepositoryGenerator.kt @@ -12,7 +12,6 @@ import org.jetbrains.jps.model.java.JpsJavaDependenciesEnumerator import org.jetbrains.jps.model.java.JpsJavaDependencyScope import org.jetbrains.jps.model.java.JpsJavaExtensionService import org.jetbrains.jps.model.library.JpsLibrary -import org.jetbrains.jps.model.library.JpsOrderRootType import org.jetbrains.jps.model.module.JpsDependencyElement import org.jetbrains.jps.model.module.JpsLibraryDependency import org.jetbrains.jps.model.module.JpsModule @@ -30,30 +29,30 @@ object RuntimeModuleRepositoryGenerator { * Generates the runtime module descriptors for all modules and libraries in [project]. */ fun generateRuntimeModuleDescriptorsForWholeProject(project: JpsProject, resourcePathsSchema: ResourcePathsSchema): List { - val libraries = LinkedHashSet() + val projectLibraries = LinkedHashSet() for (module in project.modules) { - libraries.addAll(enumerateRuntimeDependencies(module).libraries) + projectLibraries.addAll(enumerateRuntimeDependencies(module).libraries.filter { it.isProjectLevel }) } return generateRuntimeModuleDescriptors( includedProduction = project.modules, includedTests = project.modules, - includedLibraries = libraries, + includedProjectLibraries = projectLibraries, resourcePathsSchema = resourcePathsSchema ) } /** * Generates the runtime module descriptors for production parts of [includedProduction], test parts of [includedTests] and - * [includedLibraries]. + * [includedProjectLibraries]. */ fun generateRuntimeModuleDescriptors(includedProduction: Collection, includedTests: Collection, - includedLibraries: Collection, + includedProjectLibraries: Collection, resourcePathsSchema: ResourcePathsSchema): List { val descriptors = ArrayList() generateDescriptorsForModules(descriptors, includedProduction, includedTests, resourcePathsSchema) - for (library in includedLibraries) { - val moduleId = getLibraryId(library) + for (library in includedProjectLibraries) { + val moduleId = getProjectLibraryId(library) ?: error("Project-level library expected, but found: $library") descriptors.add(RawRuntimeModuleDescriptor.create(moduleId.stringId, resourcePathsSchema.libraryPaths(library), emptyList())) } return descriptors @@ -119,8 +118,8 @@ private fun generateDescriptorsForModules( } } if (includedTests.isNotEmpty()) { - val additionalDependenciesForTestsCache = HashMap>() - val productionDependenciesCache = HashMap>() + val additionalDependenciesForTestsCache = HashMap() + val productionDependenciesCache = HashMap() for (module in includedTests) { if (module.hasTestSources) { descriptors.add(createTestPartDescriptor(module = module, @@ -145,12 +144,20 @@ private val JpsModule.hasProductionSources private fun createProductionPartDescriptor(module: JpsModule, runtimeModuleNameGenerator: (JpsModule, Boolean) -> String, resourcePathsSchema: ResourcePathsSchema): RawRuntimeModuleDescriptor { val dependencies = LinkedHashSet() + val resourcePaths = if (module.hasProductionSources) resourcePathsSchema.moduleOutputPaths(module).toMutableSet() else mutableSetOf() enumerateRuntimeDependencies(module).productionOnly().processModuleAndLibraries( { dependencies.add(runtimeModuleNameGenerator(it, false)) }, - { dependencies.add(getLibraryId(it).stringId) } + { library -> + val projectLibraryId = getProjectLibraryId(library) + if (projectLibraryId != null) { + dependencies.add(projectLibraryId.stringId) + } + else { + resourcePaths.addAll(resourcePathsSchema.libraryPaths(library)) + } + } ) - val resourcePaths = if (module.hasProductionSources) resourcePathsSchema.moduleOutputPaths(module) else emptyList() - return RawRuntimeModuleDescriptor.create(runtimeModuleNameGenerator(module, false), resourcePaths, dependencies.toList()) + return RawRuntimeModuleDescriptor.create(runtimeModuleNameGenerator(module, false), resourcePaths.toList(), dependencies.toList()) } /** @@ -163,15 +170,24 @@ private fun createProductionPartDescriptor(module: JpsModule, runtimeModuleNameG private fun createTestPartDescriptor( module: JpsModule, runtimeModuleNameGenerator: (JpsModule, Boolean) -> String, - additionalDependenciesForTestsCache: MutableMap>, - productionDependenciesCache: MutableMap>, + additionalDependenciesForTestsCache: MutableMap, + productionDependenciesCache: MutableMap, resourcePathsSchema: ResourcePathsSchema, ): RawRuntimeModuleDescriptor { + val resourcePaths = if (module.hasTestSources) resourcePathsSchema.moduleTestOutputPaths(module).toMutableSet() else mutableSetOf() val dependencies = LinkedHashSet() - dependencies.addAll(collectProductionDependenciesForModule(module, productionDependenciesCache, runtimeModuleNameGenerator)) - dependencies.addAll(collectAdditionalRuntimeDependenciesForTests(module, productionDependenciesCache, additionalDependenciesForTestsCache, runtimeModuleNameGenerator)) - val resourcePaths = if (module.hasTestSources) resourcePathsSchema.moduleTestOutputPaths(module) else emptyList() - return RawRuntimeModuleDescriptor.create(runtimeModuleNameGenerator(module, true), resourcePaths, dependencies.map { it.stringId }) + val forProduction = collectProductionDependenciesForModule(module, productionDependenciesCache, runtimeModuleNameGenerator, resourcePathsSchema) + forProduction.copyTo(dependencies, resourcePaths) + + val forTests = collectAdditionalRuntimeDependenciesAndResourcesForTests( + module, + productionDependenciesCache, + additionalDependenciesForTestsCache, + runtimeModuleNameGenerator, + resourcePathsSchema + ) + forTests.copyTo(dependencies, resourcePaths) + return RawRuntimeModuleDescriptor.create(runtimeModuleNameGenerator(module, true), resourcePaths.toList(), dependencies.map { it.stringId }) } /** @@ -179,48 +195,77 @@ private fun createTestPartDescriptor( */ private fun collectProductionDependenciesForModule( module: JpsModule, - productionDependenciesCache: MutableMap>, + productionDependenciesCache: MutableMap, runtimeModuleNameGenerator: (JpsModule, Boolean) -> String, -): Set { + resourcePathsSchema: ResourcePathsSchema, +) : DependenciesAndResources { val cached = productionDependenciesCache[module] if (cached != null) { return cached } if (module.hasDescriptorForProduction) { - val result = setOf(RuntimeModuleId.raw(runtimeModuleNameGenerator(module, false))) + val result = DependenciesAndResources(setOf(RuntimeModuleId.raw(runtimeModuleNameGenerator(module, false))), emptySet()) productionDependenciesCache[module] = result return result } //if a module doesn't have its own descriptor, its dependencies should be added instead - productionDependenciesCache[module] = emptySet() //to prevent StackOverflowError in case of circular dependencies - val result = LinkedHashSet() + productionDependenciesCache[module] = emptyDependenciesAndResources //to prevent StackOverflowError in case of circular dependencies + val dependencies = LinkedHashSet() + val resourcePaths = LinkedHashSet() enumerateRuntimeDependencies(module).productionOnly().processModuleAndLibraries( { dependency -> - result.addAll(collectProductionDependenciesForModule(dependency, productionDependenciesCache, runtimeModuleNameGenerator)) + collectProductionDependenciesForModule( + dependency, + productionDependenciesCache, + runtimeModuleNameGenerator, + resourcePathsSchema + ).copyTo(dependencies, resourcePaths) }, - { result.add(getLibraryId(it)) } + { dependency -> + val projectLibraryId = getProjectLibraryId(dependency) + if (projectLibraryId != null) { + dependencies.add(projectLibraryId) + } + else { + resourcePaths.addAll(resourcePathsSchema.libraryPaths(dependency)) + } + } ) + val result = DependenciesAndResources(dependencies, resourcePaths) productionDependenciesCache[module] = result return result } +private data class DependenciesAndResources( + val dependencies: Set, + val resourcePaths: Set, +) { + fun copyTo(dependencies: MutableSet, resourcePaths: MutableCollection) { + dependencies.addAll(this.dependencies) + resourcePaths.addAll(this.resourcePaths) + } +} + +private val emptyDependenciesAndResources = DependenciesAndResources(emptySet(), emptySet()) /** * Returns IDs of descriptors which should be added to tests dependencies of [module] in addition to production dependencies. */ -private fun collectAdditionalRuntimeDependenciesForTests( +private fun collectAdditionalRuntimeDependenciesAndResourcesForTests( module: JpsModule, - productionDependenciesCache: MutableMap>, - additionalDependenciesForTestsCache: MutableMap>, - runtimeModuleNameGenerator: (JpsModule, Boolean) -> String -): Collection { + productionDependenciesCache: MutableMap, + additionalDependenciesForTestsCache: MutableMap, + runtimeModuleNameGenerator: (JpsModule, Boolean) -> String, + resourcePathsSchema: ResourcePathsSchema +): DependenciesAndResources { val cached = additionalDependenciesForTestsCache[module] if (cached != null) { return cached } - additionalDependenciesForTestsCache[module] = emptySet() ////to prevent StackOverflowError in case of circular dependencies - val result = LinkedHashSet() + additionalDependenciesForTestsCache[module] = emptyDependenciesAndResources ////to prevent StackOverflowError in case of circular dependencies + val dependencies = LinkedHashSet() + val resourcePaths = LinkedHashSet() for (dependency in module.dependenciesList.dependencies) { when (dependency) { is JpsModuleDependency -> { @@ -229,38 +274,55 @@ private fun collectAdditionalRuntimeDependenciesForTests( if (scope == JpsJavaDependencyScope.PROVIDED) continue if (dependencyModule.hasTestSources) { - result.add(RuntimeModuleId.raw(runtimeModuleNameGenerator(dependencyModule, true))) + dependencies.add(RuntimeModuleId.raw(runtimeModuleNameGenerator(dependencyModule, true))) } else { if (scope == JpsJavaDependencyScope.TEST) { - result.addAll(collectProductionDependenciesForModule(dependencyModule, productionDependenciesCache, runtimeModuleNameGenerator)) + collectProductionDependenciesForModule( + dependencyModule, + productionDependenciesCache, + runtimeModuleNameGenerator, + resourcePathsSchema + ).copyTo(dependencies, resourcePaths) } - result.addAll(collectAdditionalRuntimeDependenciesForTests(dependencyModule, productionDependenciesCache, additionalDependenciesForTestsCache, runtimeModuleNameGenerator)) + collectAdditionalRuntimeDependenciesAndResourcesForTests( + dependencyModule, + productionDependenciesCache, + additionalDependenciesForTestsCache, + runtimeModuleNameGenerator, + resourcePathsSchema + ).copyTo(dependencies, resourcePaths) } } is JpsLibraryDependency -> { if (dependency.scope == JpsJavaDependencyScope.TEST) { - dependency.library?.let { result.add(getLibraryId(it)) } + dependency.library?.let { library -> + val projectLibraryId = getProjectLibraryId(library) + if (projectLibraryId != null) { + dependencies.add(projectLibraryId) + } + else { + resourcePaths.addAll(resourcePathsSchema.libraryPaths(library)) + } + } } } } } - additionalDependenciesForTestsCache[module] = result - return result + val dependenciesAndResources = DependenciesAndResources(dependencies, resourcePaths) + additionalDependenciesForTestsCache[module] = dependenciesAndResources + return dependenciesAndResources } private val JpsDependencyElement.scope: JpsJavaDependencyScope? get() = JpsJavaExtensionService.getInstance().getDependencyExtension(this)?.scope -private fun getLibraryId(library: JpsLibrary): RuntimeModuleId { - var name = library.name - val element = (library as JpsElementBase<*>).parent.parent - if (element is JpsModule) { - val files = library.getFiles(JpsOrderRootType.COMPILED) - if (name.startsWith("#") && files.size == 1) { - name = files[0].name - } - return RuntimeModuleId.moduleLibrary((element as JpsModule).name, name) +private fun getProjectLibraryId(library: JpsLibrary): RuntimeModuleId? { + if (library.isProjectLevel) { + return RuntimeModuleId.projectLibrary(library.name) } - return RuntimeModuleId.projectLibrary(name) + return null } + +val JpsLibrary.isProjectLevel: Boolean + get() = (this as JpsElementBase<*>).parent.parent is JpsProject \ No newline at end of file diff --git a/plugins/devkit/runtimeModuleRepository/jps/testSrc/build/RuntimeModuleRepositoryBuilderTest.kt b/plugins/devkit/runtimeModuleRepository/jps/testSrc/build/RuntimeModuleRepositoryBuilderTest.kt index d8d40877da60..e7dc70e079cd 100644 --- a/plugins/devkit/runtimeModuleRepository/jps/testSrc/build/RuntimeModuleRepositoryBuilderTest.kt +++ b/plugins/devkit/runtimeModuleRepository/jps/testSrc/build/RuntimeModuleRepositoryBuilderTest.kt @@ -90,7 +90,24 @@ class RuntimeModuleRepositoryBuilderTest : RuntimeModuleRepositoryTestCase() { testDescriptor("c.tests", "c", "a.tests") } } - + + fun `test transitive dependency via module without tests but with test module-level library`() { + val a = addModule("a", withTests = true) + val b = addModule("b", a, withTests = false) + val lib = b.libraryCollection.addLibrary("lib", JpsJavaLibraryType.INSTANCE) + val dependency = b.dependenciesList.addLibraryDependency(lib) + JpsJavaExtensionService.getInstance().getOrCreateDependencyExtension(dependency).scope = JpsJavaDependencyScope.TEST + lib.addRoot(getUrl("project/lib"), JpsOrderRootType.COMPILED) + addModule("c", b, withTests = true) + buildAndCheck { + descriptor("a") + descriptor("b", "a") + descriptor("c", "b") + testDescriptor("a.tests", "a") + descriptor("c.tests", listOf("test/c", $$"$PROJECT_DIR$/lib"), listOf("c", "a.tests")) + } + } + fun `test do not add unnecessary transitive dependencies via module without tests`() { val a = addModule("a", withTests = true) val b = addModule("b", withTests = false) @@ -157,8 +174,7 @@ class RuntimeModuleRepositoryBuilderTest : RuntimeModuleRepositoryTestCase() { a.dependenciesList.addLibraryDependency(lib) lib.addRoot(getUrl("project/lib"), JpsOrderRootType.COMPILED) buildAndCheck { - descriptor("a", "lib.a.lib") - descriptor("lib.a.lib", listOf($$"$PROJECT_DIR$/lib"), emptyList()) + descriptor("a",listOf("production/a", $$"$PROJECT_DIR$/lib"), emptyList()) } }