From 6fb57f5b2d5346c7a622cde47e6d2d6c11acdf2f Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Tue, 26 Mar 2024 07:56:53 +0100 Subject: [PATCH] IJPL-866 separate product modules on a class loader level GitOrigin-RevId: 5bfe0607aec78d560348b0fb749fe73ad6bb4c2a --- .../dev-server/src/DevMainImpl.kt | 38 +++++----- .../intellij.platform.buildScripts.iml | 1 + .../intellij/build/dev}/BuildServer.kt | 2 +- .../intellij/build/dev}/IdeBuilder.kt | 25 +++---- .../intellij/build/dev}/PluginBuilder.kt | 2 +- .../intellij/build/impl/BaseLayout.kt | 2 + .../intellij/build/impl/BuildHelper.kt | 34 +++++---- .../intellij/build/impl/BuildTasksImpl.kt | 35 ++++++++-- .../intellij/build/impl/JarPackager.kt | 20 +++--- .../intellij/build/impl/PlatformLayout.kt | 19 +++-- .../intellij/build/impl/PlatformModules.kt | 30 ++++---- .../intellij/build/impl/ProjectLibraryData.kt | 2 +- .../ide/plugins/ClassLoaderConfigurator.kt | 14 ++++ .../ide/plugins/ClassPathXmlPathResolver.kt | 13 ++-- .../ide/plugins/IdeaPluginDescriptorImpl.kt | 14 +++- .../ide/plugins/PluginDescriptorLoader.kt | 70 ++++++++++++++++++- 16 files changed, 235 insertions(+), 86 deletions(-) rename platform/build-scripts/{dev-server/src => src/org/jetbrains/intellij/build/dev}/BuildServer.kt (98%) rename platform/build-scripts/{dev-server/src => src/org/jetbrains/intellij/build/dev}/IdeBuilder.kt (96%) rename platform/build-scripts/{dev-server/src => src/org/jetbrains/intellij/build/dev}/PluginBuilder.kt (98%) diff --git a/platform/build-scripts/dev-server/src/DevMainImpl.kt b/platform/build-scripts/dev-server/src/DevMainImpl.kt index 326ae43db6c6..cedfeec73030 100644 --- a/platform/build-scripts/dev-server/src/DevMainImpl.kt +++ b/platform/build-scripts/dev-server/src/DevMainImpl.kt @@ -1,12 +1,12 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("unused", "ReplaceJavaStaticMethodWithKotlinAnalog") @file:JvmName("DevMainImpl") package org.jetbrains.intellij.build.devServer import com.intellij.openapi.application.PathManager import com.intellij.platform.diagnostic.telemetry.exporters.BatchSpanProcessor -import com.intellij.util.SystemProperties import com.intellij.platform.util.coroutines.childScope +import com.intellij.util.SystemProperties import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.sdk.OpenTelemetrySdk @@ -17,6 +17,10 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.runBlocking import org.jetbrains.intellij.build.ConsoleSpanExporter import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader +import org.jetbrains.intellij.build.dev.BuildRequest +import org.jetbrains.intellij.build.dev.buildProductInProcess +import org.jetbrains.intellij.build.dev.getAdditionalModules +import org.jetbrains.intellij.build.dev.getIdeSystemProperties import org.jetbrains.intellij.build.traceManagerInitializer import java.io.File import java.nio.file.Path @@ -47,21 +51,23 @@ fun buildDevMain(): Collection { tracer to spanProcessor } - buildProductInProcess(BuildRequest( - platformPrefix = System.getProperty("idea.platform.prefix", "idea"), - additionalModules = getAdditionalModules()?.toList() ?: emptyList(), - homePath = ideaProjectRoot, - keepHttpClient = false, - platformClassPathConsumer = { classPath, runDir -> - newClassPath = classPath - homePath = runDir.toString().replace(File.separator, "/") + buildProductInProcess( + BuildRequest( + platformPrefix = System.getProperty("idea.platform.prefix", "idea"), + additionalModules = getAdditionalModules()?.toList() ?: emptyList(), + homePath = ideaProjectRoot, + keepHttpClient = false, + platformClassPathConsumer = { classPath, runDir -> + newClassPath = classPath + homePath = runDir.toString().replace(File.separator, "/") - for ((name, value) in getIdeSystemProperties(runDir)) { - System.setProperty(name, value) - } - }, - generateRuntimeModuleRepository = SystemProperties.getBooleanProperty("intellij.build.generate.runtime.module.repository", false) - )) + for ((name, value) in getIdeSystemProperties(runDir)) { + System.setProperty(name, value) + } + }, + generateRuntimeModuleRepository = SystemProperties.getBooleanProperty("intellij.build.generate.runtime.module.repository", false), + ) + ) } finally { batchSpanProcessorScope.cancel() diff --git a/platform/build-scripts/intellij.platform.buildScripts.iml b/platform/build-scripts/intellij.platform.buildScripts.iml index 5339907618e5..a7d86b899549 100644 --- a/platform/build-scripts/intellij.platform.buildScripts.iml +++ b/platform/build-scripts/intellij.platform.buildScripts.iml @@ -284,5 +284,6 @@ + \ No newline at end of file diff --git a/platform/build-scripts/dev-server/src/BuildServer.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/BuildServer.kt similarity index 98% rename from platform/build-scripts/dev-server/src/BuildServer.kt rename to platform/build-scripts/src/org/jetbrains/intellij/build/dev/BuildServer.kt index f21559ae7655..7bf1d89b5bb3 100644 --- a/platform/build-scripts/dev-server/src/BuildServer.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/BuildServer.kt @@ -1,7 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("ReplaceGetOrSet", "ReplacePutWithAssignment", "LiftReturnOrAssignment") -package org.jetbrains.intellij.build.devServer +package org.jetbrains.intellij.build.dev import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope import kotlinx.coroutines.NonCancellable diff --git a/platform/build-scripts/dev-server/src/IdeBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/IdeBuilder.kt similarity index 96% rename from platform/build-scripts/dev-server/src/IdeBuilder.kt rename to platform/build-scripts/src/org/jetbrains/intellij/build/dev/IdeBuilder.kt index f7c0279489f1..8a7c72000105 100644 --- a/platform/build-scripts/dev-server/src/IdeBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/IdeBuilder.kt @@ -1,7 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("ReplacePutWithAssignment") -package org.jetbrains.intellij.build.devServer +package org.jetbrains.intellij.build.dev import com.dynatrace.hash4j.hashing.HashFunnel import com.dynatrace.hash4j.hashing.Hashing @@ -41,12 +41,11 @@ import java.time.temporal.TemporalAdjusters import kotlin.String import kotlin.time.Duration.Companion.seconds -private val isUnpackedDist = System.getProperty("idea.dev.build.unpacked").toBoolean() - data class BuildRequest( @JvmField val platformPrefix: String, @JvmField val additionalModules: List, @JvmField val homePath: Path, + @JvmField val devRootPath: Path = homePath.normalize().toAbsolutePath().resolve("out/dev-run"), @JvmField val productionClassOutput: Path = Path.of(System.getenv("CLASSES_DIR") ?: homePath.resolve("out/classes/production").toString()).toAbsolutePath(), @JvmField val keepHttpClient: Boolean = true, @@ -57,6 +56,8 @@ data class BuildRequest( * and its generation makes build a little longer, so it should be enabled only if needed. */ @JvmField val generateRuntimeModuleRepository: Boolean = false, + + @JvmField val isUnpackedDist: Boolean = System.getProperty("idea.dev.build.unpacked").toBoolean() ) { override fun toString(): String { return "BuildRequest(platformPrefix='$platformPrefix', " + @@ -70,7 +71,7 @@ data class BuildRequest( @OptIn(ExperimentalCoroutinesApi::class) internal suspend fun buildProduct(productConfiguration: ProductConfiguration, request: BuildRequest): Path { val rootDir = withContext(Dispatchers.IO) { - val rootDir = request.homePath.normalize().toAbsolutePath().resolve("out/dev-run") + val rootDir = request.devRootPath // if symlinked to ram disk, use a real path for performance reasons and avoid any issues in ant/other code if (Files.exists(rootDir)) { // toRealPath must be called only on existing file @@ -108,7 +109,7 @@ internal suspend fun buildProduct(productConfiguration: ProductConfiguration, re val runDir = buildDir.resolve(productDirNameWithoutClassifier) val context = createBuildContext( - productConfiguration = productConfiguration, + createProductProperties = { createProductProperties(productConfiguration = productConfiguration, request = request) }, request = request, runDir = runDir, buildDir = buildDir, @@ -160,13 +161,13 @@ internal suspend fun buildProduct(productConfiguration: ProductConfiguration, re launch { val (pluginEntries, additionalEntries) = pluginDistributionEntriesDeferred.await() spanBuilder("generate plugin classpath").useWithScope(Dispatchers.IO) { - val mainData = generatePluginClassPath(pluginEntries, writeDescriptor = !isUnpackedDist) - val additionalData = additionalEntries?.let { generatePluginClassPathFromFiles(it, writeDescriptor = !isUnpackedDist) } + val mainData = generatePluginClassPath(pluginEntries, writeDescriptor = !request.isUnpackedDist) + val additionalData = additionalEntries?.let { generatePluginClassPathFromFiles(it, writeDescriptor = !request.isUnpackedDist) } val byteOut = ByteArrayOutputStream() val out = DataOutputStream(byteOut) val pluginCount = pluginEntries.size + (additionalEntries?.size ?: 0) - writePluginClassPathHeader(out = out, isJarOnly = !isUnpackedDist, pluginCount = pluginCount) + writePluginClassPathHeader(out = out, isJarOnly = !request.isUnpackedDist, pluginCount = pluginCount) out.write(mainData) additionalData?.let { out.write(it) } out.close() @@ -332,7 +333,7 @@ private suspend fun buildPlugins( } private suspend fun createBuildContext( - productConfiguration: ProductConfiguration, + createProductProperties: suspend () -> ProductProperties, request: BuildRequest, runDir: Path, jarCacheDir: Path, @@ -342,7 +343,7 @@ private suspend fun createBuildContext( // ~1 second val productProperties = async { withTimeout(30.seconds) { - createProductProperties(productConfiguration = productConfiguration, request = request) + createProductProperties() } } @@ -369,7 +370,7 @@ private suspend fun createBuildContext( BuildOptions.FUS_METADATA_BUNDLE_STEP, ) - if (isUnpackedDist && options.enableEmbeddedJetBrainsClient) { + if (request.isUnpackedDist && options.enableEmbeddedJetBrainsClient) { options.enableEmbeddedJetBrainsClient = false } @@ -497,7 +498,7 @@ private fun getBundledMainModuleNames(productProperties: ProductProperties, addi return LinkedHashSet(productProperties.productLayout.bundledPluginModules) + additionalModules } -internal fun getAdditionalModules(): Sequence? { +fun getAdditionalModules(): Sequence? { return System.getProperty("additional.modules")?.splitToSequence(',')?.map(String::trim)?.filter { it.isNotEmpty() } } diff --git a/platform/build-scripts/dev-server/src/PluginBuilder.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/PluginBuilder.kt similarity index 98% rename from platform/build-scripts/dev-server/src/PluginBuilder.kt rename to platform/build-scripts/src/org/jetbrains/intellij/build/dev/PluginBuilder.kt index 7ec6b59ea422..e834b40ae646 100644 --- a/platform/build-scripts/dev-server/src/PluginBuilder.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/dev/PluginBuilder.kt @@ -2,7 +2,7 @@ @file:Suppress("PrivatePropertyName", "LiftReturnOrAssignment") @file:OptIn(ExperimentalCoroutinesApi::class) -package org.jetbrains.intellij.build.devServer +package org.jetbrains.intellij.build.dev import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope import io.opentelemetry.api.common.AttributeKey diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BaseLayout.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BaseLayout.kt index 7e4c24992035..f9e93f7d2d92 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BaseLayout.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BaseLayout.kt @@ -54,6 +54,8 @@ sealed class BaseLayout { fun hasLibrary(name: String): Boolean = includedProjectLibraries.any { it.libraryName == name } + fun findProjectLibrary(name: String): ProjectLibraryData? = includedProjectLibraries.firstOrNull { it.libraryName == name } + @TestOnly fun isLibraryExcluded(name: String): Boolean = excludedLibraries.get(null)?.contains(name) ?: false diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildHelper.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildHelper.kt index 36290441b486..ab2ca47526b6 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildHelper.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildHelper.kt @@ -1,6 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl +import com.intellij.openapi.application.PathManager import com.intellij.platform.diagnostic.telemetry.helpers.use import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope import com.intellij.util.JavaModuleOptions @@ -83,17 +84,20 @@ suspend fun runIdea(context: CompilationContext, onError = onError) } -suspend fun runApplicationStarter(context: BuildContext, - tempDir: Path, - ideClasspath: Set, - arguments: List, - systemProperties: Map = emptyMap(), - vmOptions: List = emptyList(), - timeout: Duration = DEFAULT_TIMEOUT) { +suspend fun runApplicationStarter( + context: BuildContext, + tempDir: Path, + ideClasspath: Collection, + arguments: List, + systemProperties: Map = emptyMap(), + vmOptions: List = emptyList(), + homePath: Path = context.paths.projectHome, + timeout: Duration = DEFAULT_TIMEOUT, +) { Files.createDirectories(tempDir) val jvmArgs = mutableListOf() val systemDir = tempDir.resolve("system") - BuildUtils.addVmProperty(jvmArgs, "idea.home.path", context.paths.projectHome.toString()) + BuildUtils.addVmProperty(jvmArgs, PathManager.PROPERTY_HOME_PATH, homePath.toString()) BuildUtils.addVmProperty(jvmArgs, "idea.system.path", systemDir.toString()) BuildUtils.addVmProperty(jvmArgs, "idea.config.path", "$tempDir/config") BuildUtils.addVmProperty(jvmArgs, "idea.builtin.server.disabled", "true") @@ -120,12 +124,14 @@ suspend fun runApplicationStarter(context: BuildContext, } } disableCompatibleIgnoredPlugins(context = context, configDir = tempDir.resolve("config"), explicitlyEnabledPlugins = additionalPluginIds) - runIdea(context = context, - mainClass = context.productProperties.mainClassName, - args = arguments, - jvmArgs = jvmArgs, - classPath = effectiveIdeClasspath.toList(), - timeout = timeout) { + runIdea( + context = context, + mainClass = context.productProperties.mainClassName, + args = arguments, + jvmArgs = jvmArgs, + classPath = effectiveIdeClasspath.toList(), + timeout = timeout + ) { val logFile = systemDir.resolve("log").resolve("idea.log") if (Files.exists(logFile)) { val logFileToPublish = Files.createTempFile("idea-", ".log") diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt index 895232ac2056..b756e3274e2f 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt @@ -29,6 +29,9 @@ import org.jetbrains.idea.maven.aether.ArtifactRepositoryManager import org.jetbrains.idea.maven.aether.ProgressConsumer import org.jetbrains.intellij.build.* import org.jetbrains.intellij.build.TraceManager.spanBuilder +import org.jetbrains.intellij.build.dev.BuildRequest +import org.jetbrains.intellij.build.dev.buildProductInProcess +import org.jetbrains.intellij.build.dev.getIdeSystemProperties import org.jetbrains.intellij.build.impl.productInfo.PRODUCT_INFO_FILE_NAME import org.jetbrains.intellij.build.impl.productInfo.ProductInfoLaunchData import org.jetbrains.intellij.build.impl.productInfo.checkInArchive @@ -660,17 +663,41 @@ private suspend fun compileModulesForDistribution(context: BuildContext): Distri compilationTasks.compileModules(moduleNames = it) localizeModules(context, moduleNames = it) } + val builtinModuleData = spanBuilder("build provided module list").useWithScope { - val ideClasspath = createIdeClassPath(platform = platform, context = context) + //val ideClasspath = createIdeClassPath(platform = platform, context = context) + + var homePath: Path? = null + var newClassPath: Collection? = null + val systemProperties = HashMap() + val tempDir = context.paths.tempDir.resolve("builtinModules") + buildProductInProcess( + BuildRequest( + devRootPath = tempDir, + isUnpackedDist = true, + platformPrefix = context.productProperties.platformPrefix ?: "idea", + additionalModules = emptyList(), + homePath = context.paths.projectHome, + platformClassPathConsumer = { classPath, runDir -> + newClassPath = classPath + homePath = runDir + for ((name, value) in getIdeSystemProperties(runDir)) { + systemProperties.put(name, value) + } + } + ) + ) Files.deleteIfExists(providedModuleFile) // start the product in headless mode using com.intellij.ide.plugins.BundledPluginsLister runApplicationStarter( context = context, - tempDir = context.paths.tempDir.resolve("builtinModules"), - ideClasspath = ideClasspath, + tempDir = tempDir, + ideClasspath = newClassPath!!.map { it.toString() }, arguments = listOf("listBundledPlugins", providedModuleFile.toString()), - timeout = 30.seconds + timeout = 30.seconds, + homePath = homePath ?: context.paths.projectHome, + systemProperties = systemProperties, ) context.productProperties.customizeBuiltinModules(context = context, builtinModulesFile = providedModuleFile) try { diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt index d93338259f3d..98341eb7a9e9 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt @@ -390,7 +390,7 @@ class JarPackager private constructor( } moduleSources.add(DirSource(dir = moduleOutDir, excludes = excludes)) - if (layout != null && !item.relativeOutputFile.contains('/') && !layout.modulesWithExcludedModuleLibraries.contains(moduleName)) { + if (layout != null && !layout.modulesWithExcludedModuleLibraries.contains(moduleName)) { val jarAsset = if (packToDir) { getJarAsset( targetFile = outFile, @@ -438,8 +438,8 @@ class JarPackager private constructor( libToMetadata.put(element.library!!, projectLibraryData) } else if (isLibraryAlwaysPackedIntoPlugin(libName)) { - check(!platformLayout!!.hasLibrary(libName)) { - "Library $libName must not be included into platform layout" + platformLayout!!.findProjectLibrary(libName)?.let { + throw IllegalStateException("Library $libName must not be included into platform layout: $it") } if (layout.hasLibrary(libName)) { @@ -469,11 +469,7 @@ class JarPackager private constructor( for (i in (files.size - 1) downTo 0) { val file = files.get(i) val fileName = file.fileName.toString() - val jarName = when (layout) { - is PluginLayout -> layout.getMainJarName() - is PlatformLayout -> PlatformJarNames.APP_JAR - } - if (item.relativeOutputFile.contains('/') || isSeparateJar(fileName = fileName, file = file, jarName = jarName)) { + if (isSeparateJar(fileName = fileName, file = file, jarPath = asset.relativePath)) { files.removeAt(i) addLibrary( library = library, @@ -728,7 +724,11 @@ class JarPackager private constructor( } } -private suspend fun isSeparateJar(fileName: String, file: Path, jarName: String): Boolean { +private suspend fun isSeparateJar(fileName: String, file: Path, jarPath: String): Boolean { + if (jarPath.contains('/') && !jarPath.startsWith("modules/")) { + return true + } + if (fileName.endsWith("-rt.jar") || fileName.contains("-agent")) { return true } @@ -744,7 +744,7 @@ private suspend fun isSeparateJar(fileName: String, file: Path, jarName: String) } } if (result) { - Span.current().addEvent("$fileName contains file '$filePreventingMerging' that prevent its merging into $jarName") + Span.current().addEvent("$fileName contains file '$filePreventingMerging' that prevent its merging into $jarPath") } return result } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformLayout.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformLayout.kt index 04423f650601..020578ee3808 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformLayout.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformLayout.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl import org.jetbrains.annotations.ApiStatus.Internal @@ -36,9 +36,20 @@ class PlatformLayout: BaseLayout() { excludedProjectLibraries.add(libraryName) } - inline fun collectProjectLibrariesFromIncludedModules(context: BuildContext, consumer: (JpsLibrary, JpsModule) -> Unit) { + fun collectProjectLibrariesFromIncludedModules(context: BuildContext, consumer: (JpsLibrary, JpsModule) -> Unit) { val libsToUnpack = includedProjectLibraries.mapTo(HashSet()) { it.libraryName } - for (moduleName in includedModules.asSequence().map { it.moduleName }.distinct()) { + val uniqueGuard = HashSet() + for (item in includedModules) { + // libraries are packed into product module + if (item.reason == ModuleIncludeReasons.PRODUCT_MODULES) { + continue + } + + val moduleName = item.moduleName + if (!uniqueGuard.add(moduleName)) { + continue + } + val module = context.findRequiredModule(moduleName) for (library in JpsJavaExtensionService.dependencies(module).includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).libraries) { if (!isSkippedLibrary(library, libsToUnpack)) { @@ -48,7 +59,7 @@ class PlatformLayout: BaseLayout() { } } - fun isSkippedLibrary(library: JpsLibrary, libsToUnpack: Collection): Boolean { + private fun isSkippedLibrary(library: JpsLibrary, libsToUnpack: Collection): Boolean { return library.createReference().parentReference is JpsModuleReference || libsToUnpack.contains(library.name) || excludedProjectLibraries.contains(library.name) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt index 3e4f501caf1c..8ebb0e2b83de 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt @@ -9,7 +9,6 @@ import io.opentelemetry.api.trace.Span import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet import kotlinx.collections.immutable.PersistentList import kotlinx.collections.immutable.persistentListOf -import kotlinx.collections.immutable.persistentMapOf import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -99,9 +98,9 @@ private val PLATFORM_IMPLEMENTATION_MODULES = persistentListOf( "intellij.platform.util.commonsLangV2Shim" ) -internal val PLATFORM_CUSTOM_PACK_MODE: Map = persistentMapOf( - "jetbrains-annotations" to LibraryPackMode.STANDALONE_SEPARATE_WITHOUT_VERSION_NAME, - "intellij-coverage" to LibraryPackMode.STANDALONE_SEPARATE, +internal val PLATFORM_CUSTOM_PACK_MODE: Map = java.util.Map.of( + "jetbrains-annotations", LibraryPackMode.STANDALONE_SEPARATE_WITHOUT_VERSION_NAME, + "intellij-coverage", LibraryPackMode.STANDALONE_SEPARATE, ) internal fun collectPlatformModules(to: MutableCollection) { @@ -323,7 +322,8 @@ internal suspend fun createPlatformLayout(addPlatformCoverage: Boolean, return layout } -fun isLibraryAlwaysPackedIntoPlugin(name: String): Boolean = name == "flexmark" || name == "okhttp" +// sqlite - used by DB and "import settings" (temporarily) +fun isLibraryAlwaysPackedIntoPlugin(name: String): Boolean = name == "flexmark" || name == "okhttp" || name == "sqlite" internal fun computeProjectLibsUsedByPlugins(enabledPluginModules: Set, context: BuildContext): SortedSet { val result = ObjectLinkedOpenHashSet() @@ -467,10 +467,7 @@ private fun compute(list: List>>, } if (oldSize != result.size) { - compute(list = result.subList(oldSize, result.size).sortedBy { it.first }, - context = context, - unique = unique, - result = result) + compute(list = result.subList(oldSize, result.size).sortedBy { it.first }, context = context, unique = unique, result = result) } } @@ -507,8 +504,17 @@ private suspend fun getProductPluginContentModules(context: BuildContext, produc private fun collectProductModules(content: XmlElement, result: LinkedHashSet) { for (module in content.children("module")) { - result.add(ModuleItem(moduleName = module.attributes.get("name") ?: continue, - relativeOutputFile = "modules.jar", - reason = "productModule")) + val moduleName = module.attributes.get("name") ?: continue + result.add( + ModuleItem( + moduleName = moduleName, + relativeOutputFile = "modules/$moduleName.jar", + reason = ModuleIncludeReasons.PRODUCT_MODULES + ), + ) } +} + +internal object ModuleIncludeReasons { + const val PRODUCT_MODULES: String = "productModule" } \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ProjectLibraryData.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ProjectLibraryData.kt index 7e44680b8972..476058fc2e27 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ProjectLibraryData.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ProjectLibraryData.kt @@ -37,5 +37,5 @@ class ProjectLibraryData( override fun hashCode() = libraryName.hashCode() - override fun toString() = "ProjectLibraryData(name=$libraryName, packMode=$packMode, relativeOutputPath=$outPath)" + override fun toString() = "ProjectLibraryData(name=$libraryName, packMode=$packMode, relativeOutputPath=$outPath, reason=$reason)" } diff --git a/platform/core-impl/src/com/intellij/ide/plugins/ClassLoaderConfigurator.kt b/platform/core-impl/src/com/intellij/ide/plugins/ClassLoaderConfigurator.kt index 28b2668ca0f3..097ad2c721fa 100644 --- a/platform/core-impl/src/com/intellij/ide/plugins/ClassLoaderConfigurator.kt +++ b/platform/core-impl/src/com/intellij/ide/plugins/ClassLoaderConfigurator.kt @@ -194,6 +194,20 @@ class ClassLoaderConfigurator( } private fun configureCorePluginModuleClassLoader(module: IdeaPluginDescriptorImpl, deps: Array) { + val jarFiles = module.jarFiles + if (jarFiles != null) { + module.pluginClassLoader = PluginClassLoader( + classPath = ClassPath(jarFiles, DEFAULT_CLASSLOADER_CONFIGURATION, resourceFileFactory, false), + parents = deps, + pluginDescriptor = module, + coreLoader = coreLoader, + resolveScopeManager = null, + packagePrefix = module.packagePrefix, + libDirectories = ArrayList(), + ) + return + } + val coreUrlClassLoader = getCoreUrlClassLoaderIfPossible() if (coreUrlClassLoader == null) { setPluginClassLoaderForModuleAndOldSubDescriptors(module, coreLoader) diff --git a/platform/core-impl/src/com/intellij/ide/plugins/ClassPathXmlPathResolver.kt b/platform/core-impl/src/com/intellij/ide/plugins/ClassPathXmlPathResolver.kt index bad03f16a641..eee9ac7e07a4 100644 --- a/platform/core-impl/src/com/intellij/ide/plugins/ClassPathXmlPathResolver.kt +++ b/platform/core-impl/src/com/intellij/ide/plugins/ClassPathXmlPathResolver.kt @@ -1,12 +1,15 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.ide.plugins -import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.diagnostic.logger import com.intellij.util.lang.UrlClassLoader import com.intellij.util.xml.dom.createNonCoalescingXmlStreamReader import org.codehaus.stax2.XMLStreamReader2 -internal class ClassPathXmlPathResolver(private val classLoader: ClassLoader, @JvmField val isRunningFromSources: Boolean) : PathResolver { +internal class ClassPathXmlPathResolver( + private val classLoader: ClassLoader, + @JvmField val isRunningFromSources: Boolean, +) : PathResolver { override val isFlat: Boolean get() = true @@ -68,17 +71,17 @@ internal class ClassPathXmlPathResolver(private val classLoader: ClassLoader, @J return descriptor } - val logger = Logger.getInstance(ClassPathXmlPathResolver::class.java) + val log = logger() val moduleName = path.removeSuffix(".xml") if (isRunningFromSources && path.startsWith("intellij.") && dataLoader.emptyDescriptorIfCannotResolve) { - logger.trace("Cannot resolve $path (dataLoader=$dataLoader, classLoader=$classLoader). ") + log.trace("Cannot resolve $path (dataLoader=$dataLoader, classLoader=$classLoader). ") val descriptor = RawPluginDescriptor() descriptor.`package` = "unresolved.$moduleName" return descriptor } if (ProductLoadingStrategy.strategy.isOptionalProductModule(moduleName)) { //this check won't be needed when we are able to load optional modules directly from product-modules.xml - logger.debug("Skip module '$path' since its descriptor cannot be found and it's optional") + log.debug("Skip module '$path' since its descriptor cannot be found and it's optional") return RawPluginDescriptor().apply { `package` = "unresolved.$moduleName" } } diff --git a/platform/core-impl/src/com/intellij/ide/plugins/IdeaPluginDescriptorImpl.kt b/platform/core-impl/src/com/intellij/ide/plugins/IdeaPluginDescriptorImpl.kt index 7ebcd52e9a3b..87b6e8ce3ee5 100644 --- a/platform/core-impl/src/com/intellij/ide/plugins/IdeaPluginDescriptorImpl.kt +++ b/platform/core-impl/src/com/intellij/ide/plugins/IdeaPluginDescriptorImpl.kt @@ -194,7 +194,7 @@ class IdeaPluginDescriptorImpl( override fun getPluginPath(): Path = path - private fun createSub( + internal fun createSub( raw: RawPluginDescriptor, descriptorPath: String, context: DescriptorListLoadingContext, @@ -232,7 +232,7 @@ class IdeaPluginDescriptorImpl( // must be first because merged into raw descriptor for (module in content.modules) { val subDescriptorFile = module.configFile ?: "${module.name}.xml" - val subDescriptor = createSub( + module.descriptor = createSub( raw = pathResolver.resolveModuleFile( readContext = context, dataLoader = dataLoader, @@ -243,9 +243,17 @@ class IdeaPluginDescriptorImpl( context = context, moduleName = module.name, ) - module.descriptor = subDescriptor } + initByRawDescriptor(raw = raw, context = context, pathResolver = pathResolver, dataLoader = dataLoader) + } + + internal fun initByRawDescriptor( + raw: RawPluginDescriptor, + context: DescriptorListLoadingContext, + pathResolver: PathResolver, + dataLoader: DataLoader, + ) { if (raw.resourceBundleBaseName != null) { if (id == PluginManagerCore.CORE_ID) { LOG.warn("${raw.resourceBundleBaseName} tag is found in an xml descriptor" + diff --git a/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt b/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt index 8a168f87c3e7..6515fd0073fe 100644 --- a/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt +++ b/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt @@ -617,6 +617,7 @@ private fun CoroutineScope.loadCoreModules( context = context, pathResolver = pathResolver, useCoreClassLoader = useCoreClassLoader, + pool = pool, )!! }) } @@ -628,6 +629,7 @@ private fun CoroutineScope.loadCoreModules( path = "${PluginManagerCore.META_INF}$fileName", classLoader = classLoader, context = context, + pool = pool, pathResolver = pathResolver, useCoreClassLoader = useCoreClassLoader, ) @@ -663,6 +665,7 @@ private fun loadCoreProductPlugin( context: DescriptorListLoadingContext, pathResolver: ClassPathXmlPathResolver, useCoreClassLoader: Boolean, + pool: ZipFilePool, ): IdeaPluginDescriptorImpl? { val reader = getResourceReader(path, classLoader) ?: return null val dataLoader = object : DataLoader { @@ -682,16 +685,76 @@ private fun loadCoreProductPlugin( includeBase = null, readInto = null, ) + val libDir = Paths.get(PathManager.getLibPath()) val descriptor = IdeaPluginDescriptorImpl( raw = raw, - path = Paths.get(PathManager.getLibPath()), + path = libDir, isBundled = true, id = null, moduleName = null, useCoreClassLoader = useCoreClassLoader, ) context.debugData?.recordDescriptorPath(descriptor, raw, path) - descriptor.readExternal(raw = raw, pathResolver = pathResolver, context = context, dataLoader = dataLoader) + + val moduleDir = libDir.resolve("modules") + val moduleDirExists = Files.isDirectory(moduleDir) + + for (module in descriptor.content.modules) { + check(module.configFile == null) { + "product module must not use `/` notation for module descriptor file (configFile=${module.configFile})" + } + + val moduleName = module.name + var moduleRaw: RawPluginDescriptor? + val subDescriptorFile = "$moduleName.xml" + + if (moduleDirExists && !pathResolver.isRunningFromSources && moduleName.startsWith("intellij.")) { + val jarFile = Paths.get(PathManager.getLibPath(), "modules", "$moduleName.jar") + val resolver = pool.load(jarFile) + try { + moduleRaw = resolver.loadZipEntry(subDescriptorFile)?.let { + readModuleDescriptor( + input = it, + readContext = context, + pathResolver = pathResolver, + dataLoader = dataLoader, + includeBase = null, + readInto = null, + locationSource = jarFile.toString(), + ) + } + } + finally { + (resolver as? Closeable)?.close() + } + + if (moduleRaw != null) { + val subDescriptor = descriptor.createSub( + raw = moduleRaw, + descriptorPath = subDescriptorFile, + context = context, + moduleName = moduleName, + ) + subDescriptor.jarFiles = Java11Shim.INSTANCE.listOf(jarFile) + module.descriptor = subDescriptor + continue + } + } + + moduleRaw = pathResolver.resolveModuleFile( + readContext = context, + dataLoader = dataLoader, + path = subDescriptorFile, + readInto = null, + ) + module.descriptor = descriptor.createSub( + raw = moduleRaw, + descriptorPath = subDescriptorFile, + context = context, + moduleName = moduleName, + ) + } + descriptor.initByRawDescriptor(raw = raw, context = context, pathResolver = pathResolver, dataLoader = dataLoader) return descriptor } @@ -818,12 +881,13 @@ fun testLoadDescriptorsFromClassPath(loader: ClassLoader): List