From 31f3cf64e96ddbe9cbcf6025b496fdec86ed3cb3 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Sat, 18 Oct 2025 14:40:34 +0200 Subject: [PATCH] IJPL-209476 stage 2 - remove `modules` in lib dir GitOrigin-RevId: e0a8ec579119b33b36b1d1392181cc02207b8323 --- .../impl/BuildProcessClasspathManager.kt | 7 +- libraries/ion/module-content.yaml | 5 +- libraries/jbr/module-content.yaml | 5 +- libraries/rhino/module-content.yaml | 5 +- .../org/jetbrains/intellij/build/classpath.kt | 7 +- .../intellij/build/impl/BuildTasksImpl.kt | 1 + .../build/impl/DistributionJARsBuilder.kt | 389 +----------------- .../intellij/build/impl/JarPackager.kt | 9 +- .../intellij/build/impl/PlatformModules.kt | 105 ++--- .../impl/PluginRepositoryXmlGenerator.kt | 2 +- .../intellij/build/impl/PluginXmlPatcher.kt | 5 +- .../build/impl/deprecatedClasspath.kt | 82 ++++ .../impl/plugins/PluginAutoPublishList.kt | 4 +- .../intellij/build/impl/plugins/nonBundled.kt | 353 ++++++++++++++++ .../ide/plugins/PluginDescriptorLoader.kt | 20 +- 15 files changed, 555 insertions(+), 444 deletions(-) create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt diff --git a/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt b/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt index 437765890d93..67709c24e1e1 100644 --- a/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt +++ b/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt @@ -158,7 +158,12 @@ private fun findClassesRoot(relativePath: String, plugin: IdeaPluginDescriptor, if (PluginManagerCore.isRunningFromSources()) { val fileName = FileUtilRt.getNameWithoutExtension(PathUtilRt.getFileName(relativePath)) //try restoring module name from JAR name automatically generated by BaseLayout.convertModuleNameToFileName - val moduleName = if (relativePath.startsWith("modules/")) fileName else "intellij." + fileName.replace('-', '.') + val moduleName = if (relativePath.startsWith("modules/") || (fileName.startsWith("intellij.") && !fileName.contains('-'))) { + fileName + } + else { + "intellij." + fileName.replace('-', '.') + } val mapping = ArchivedCompilationContextUtil.archivedCompiledClassesMapping if (mapping != null) { // baseFile is ".../idea-compile-parts-v2/production//" diff --git a/libraries/ion/module-content.yaml b/libraries/ion/module-content.yaml index 55f046927ed2..f3028611864c 100644 --- a/libraries/ion/module-content.yaml +++ b/libraries/ion/module-content.yaml @@ -1,3 +1,6 @@ - name: modules: - - name: intellij.libraries.ion \ No newline at end of file + - name: intellij.libraries.ion + library: ion + files: + - name: $MAVEN_REPOSITORY$/com/amazon/ion/ion-java/1/ion-java-1.jar \ No newline at end of file diff --git a/libraries/jbr/module-content.yaml b/libraries/jbr/module-content.yaml index 07244148628f..e9554655e8bc 100644 --- a/libraries/jbr/module-content.yaml +++ b/libraries/jbr/module-content.yaml @@ -1,3 +1,6 @@ - name: modules: - - name: intellij.libraries.jbr \ No newline at end of file + - name: intellij.libraries.jbr + library: jbr-api + files: + - name: $MAVEN_REPOSITORY$/org/jetbrains/runtime/jbr-api/1/jbr-api-1.jar \ No newline at end of file diff --git a/libraries/rhino/module-content.yaml b/libraries/rhino/module-content.yaml index 9ae26312ee64..40d59f5490da 100644 --- a/libraries/rhino/module-content.yaml +++ b/libraries/rhino/module-content.yaml @@ -1,3 +1,6 @@ - name: modules: - - name: intellij.libraries.rhino \ No newline at end of file + - name: intellij.libraries.rhino + library: rhino + files: + - name: $MAVEN_REPOSITORY$/org/mozilla/rhino-runtime/1/rhino-runtime-1.jar \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/classpath.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/classpath.kt index 8888452fc8d3..3a02dd57f77b 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/classpath.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/classpath.kt @@ -15,6 +15,8 @@ import org.jetbrains.intellij.build.impl.PluginLayout import org.jetbrains.intellij.build.impl.projectStructureMapping.CustomAssetEntry import org.jetbrains.intellij.build.impl.projectStructureMapping.DistributionFileEntry import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleOutputEntry +import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleOwnedFileEntry +import org.jetbrains.intellij.build.impl.projectStructureMapping.ProjectLibraryEntry import org.jetbrains.intellij.build.io.ZipEntryProcessorResult import org.jetbrains.intellij.build.io.readZipFile import java.io.ByteArrayOutputStream @@ -26,7 +28,9 @@ import kotlin.io.path.relativeToOrSelf internal fun generateClassPathByLayoutReport(libDir: Path, entries: List, skipNioFs: Boolean): Set { val classPath = LinkedHashSet() for (entry in entries) { - val file = entry.path + if (entry is ModuleOwnedFileEntry && entry.owner?.reason == ModuleIncludeReasons.PRODUCT_MODULES) { + continue + } // exclude files like ext/platform-main.jar - if a file in lib, take only direct children in an account if ((entry.relativeOutputFile ?: "").contains('/') && !(entry is ModuleOutputEntry && entry.reason == ModuleIncludeReasons.PRODUCT_EMBEDDED_MODULES)) { @@ -47,6 +51,7 @@ internal fun generateClassPathByLayoutReport(libDir: Path, entries: List, - compressPluginArchive: Boolean, - buildPlatformLibJob: Job?, - state: DistributionBuilderState, - searchableOptionSet: SearchableOptionSetDescriptor?, - context: BuildContext, -): List>> { - return context.executeStep(spanBuilder("build non-bundled plugins").setAttribute("count", pluginsToPublish.size.toLong()), BuildOptions.NON_BUNDLED_PLUGINS_STEP) { - if (pluginsToPublish.isEmpty()) { - return@executeStep emptyList() - } - val buildKeymapPluginsTask = if (context.options.buildStepsToSkip.contains(BuildOptions.KEYMAP_PLUGINS_STEP)) { - null - } - else { - async(CoroutineName("build keymap plugins")) { - buildKeymapPlugins(targetDir = context.nonBundledPluginsToBePublished, context = context) - } - } - val moduleOutputPatcher = ModuleOutputPatcher() - val stageDir = nonBundledPluginsStageDir(context) - NioFiles.deleteRecursively(stageDir) - val dirToJar = ConcurrentLinkedQueue() - - // buildPlugins pluginBuilt listener is called concurrently - val pluginSpecs = ConcurrentLinkedQueue() - val prepareCustomPluginRepository = context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins && - !context.isStepSkipped(BuildOptions.ARCHIVE_PLUGINS) - val plugins = pluginsToPublish.sortedWith(PLUGIN_LAYOUT_COMPARATOR_BY_MAIN_MODULE) - val mappings = buildPlugins( - moduleOutputPatcher = moduleOutputPatcher, - plugins = plugins, - os = null, - targetDir = stageDir, - state = state, - context = context, - buildPlatformJob = buildPlatformLibJob, - searchableOptionSet = searchableOptionSet, - pluginBuilt = { plugin, pluginDirOrFile -> - val pluginVersion = if (plugin.mainModule == BUILT_IN_HELP_MODULE_NAME) { - context.buildNumber - } - else { - plugin.versionEvaluator.evaluate( - pluginXmlSupplier = { (context as BuildContextImpl).jarPackagerDependencyHelper.getPluginXmlContent(context.findRequiredModule(plugin.mainModule)) }, - ideBuildVersion = context.pluginBuildNumber, - context, - ).pluginVersion - } - - val targetDirectory = if (context.pluginAutoPublishList.test(plugin)) { - context.nonBundledPluginsToBePublished - } - else { - context.nonBundledPlugins - } - val destFile = targetDirectory.resolve("${plugin.directoryName}-$pluginVersion.zip") - val pluginXml = moduleOutputPatcher.getPatchedPluginXml(plugin.mainModule) - pluginSpecs.add(PluginRepositorySpec(destFile, pluginXml)) - dirToJar.add(NonBundledPlugin(sourceDir = pluginDirOrFile, targetZip = destFile, optimizedZip = !plugin.enableSymlinksAndExecutableResources)) - - handleCustomPlatformSpecificAssets( - layout = plugin, - targetPlatform = null, - context = context, - pluginDir = pluginDirOrFile, - isDevMode = true, - ) - }, - ) - - archivePlugins(items = dirToJar, compress = compressPluginArchive, withBlockMap = compressPluginArchive, context = context) - - val helpPlugin = buildHelpPlugin(context.pluginBuildNumber, context) - if (helpPlugin != null) { - val spec = buildHelpPlugin( - helpPlugin = helpPlugin, - pluginsToPublishDir = stageDir, - targetDir = context.nonBundledPluginsToBePublished, - moduleOutputPatcher = moduleOutputPatcher, - state = state, - searchableOptionSetDescriptor = searchableOptionSet, - context = context, - ) - pluginSpecs.add(spec) - } - - buildKeymapPluginsTask?.let { - for (item in it.await()) { - pluginSpecs.add(PluginRepositorySpec(pluginZip = item.first, pluginXml = item.second)) - } - } - - if (prepareCustomPluginRepository) { - val list = pluginSpecs.sortedBy { it.pluginZip } - if (list.isNotEmpty()) { - generatePluginRepositoryMetaFile(list, context.nonBundledPlugins, context.buildNumber) - } - - val pluginsToBePublished = list.filter { it.pluginZip.startsWith(context.nonBundledPluginsToBePublished) } - if (pluginsToBePublished.isNotEmpty()) { - generatePluginRepositoryMetaFile(pluginsToBePublished, context.nonBundledPluginsToBePublished, context.buildNumber) - } - } - - validatePlugins(context, pluginSpecs) - - mappings - } ?: emptyList() -} - -fun nonBundledPluginsStageDir(context: BuildContext): Path - = context.paths.tempDir.resolve("non-bundled-plugins-${context.applicationInfo.productCode}") - -private suspend fun validatePlugins(context: BuildContext, pluginSpecs: Collection) { - context.executeStep(spanBuilder("plugins validation"), BuildOptions.VALIDATE_PLUGINS_TO_BE_PUBLISHED) { span -> - for (plugin in pluginSpecs) { - val path = plugin.pluginZip - if (Files.notExists(path)) { - span.addEvent("doesn't exist, skipped", Attributes.of(AttributeKey.stringKey("path"), "$path")) - continue - } - launch(CoroutineName("$path plugin validation")) { - validatePlugin(path, context, span) - } - } - } -} - -private fun validatePlugin(file: Path, context: BuildContext, span: Span) { - val pluginManager = IdePluginManager.createManager() - val result = pluginManager.createPlugin(pluginFile = file, validateDescriptor = true) - // todo fix AddStatisticsEventLogListenerTemporary - val id = when (result) { - is PluginCreationSuccess -> result.plugin.pluginId - is PluginCreationFail -> (pluginManager.createPlugin(pluginFile = file, validateDescriptor = false) as? PluginCreationSuccess)?.plugin?.pluginId - } - for (problem in context.productProperties.validatePlugin(id, result, context)) { - val problemType = problem::class.java.simpleName - span.addEvent( - "plugin validation failed", Attributes.of( - AttributeKey.stringKey("id"), "$id", - AttributeKey.stringKey("path"), "$file", - AttributeKey.stringKey("problemType"), problemType, - ) - ) - context.messages.reportBuildProblem( - description = "${id ?: file}, $problemType: $problem", - identity = "${id ?: file}$problemType" - ) - } -} - -private suspend fun buildHelpPlugin( - helpPlugin: PluginLayout, - pluginsToPublishDir: Path, - targetDir: Path, - moduleOutputPatcher: ModuleOutputPatcher, - state: DistributionBuilderState, - searchableOptionSetDescriptor: SearchableOptionSetDescriptor?, - context: BuildContext, -): PluginRepositorySpec { - val directory = helpPlugin.directoryName - val destFile = targetDir.resolve("$directory.zip") - spanBuilder("build help plugin").setAttribute("dir", directory).use { - val targetDir = pluginsToPublishDir.resolve(directory) - buildPlugins(moduleOutputPatcher, listOf(helpPlugin), os = null, targetDir, state, context, buildPlatformJob = null, searchableOptionSetDescriptor) - zipWithCompression(targetFile = destFile, dirs = mapOf(targetDir to "")) - null - } - return PluginRepositorySpec(pluginZip = destFile, pluginXml = moduleOutputPatcher.getPatchedPluginXml(helpPlugin.mainModule)) +fun nonBundledPluginsStageDir(context: BuildContext): Path { + return context.paths.tempDir.resolve("non-bundled-plugins-${context.applicationInfo.productCode}") } internal suspend fun generateProjectStructureMapping(platformLayout: PlatformLayout, context: BuildContext): ContentReport { @@ -760,12 +575,12 @@ internal suspend fun buildPlugins( return entries } -private const val PLUGINS_DIRECTORY = "plugins" -private const val LIB_DIRECTORY = "lib" +internal const val PLUGINS_DIRECTORY = "plugins" +internal const val LIB_DIRECTORY = "lib" const val PLUGIN_CLASSPATH: String = "$PLUGINS_DIRECTORY/plugin-classpath.txt" -private val PLUGIN_LAYOUT_COMPARATOR_BY_MAIN_MODULE: Comparator = compareBy { it.mainModule } +internal val PLUGIN_LAYOUT_COMPARATOR_BY_MAIN_MODULE: Comparator = compareBy { it.mainModule } @VisibleForTesting class PluginRepositorySpec(@JvmField val pluginZip: Path, @JvmField val pluginXml: ByteArray /* content of plugin.xml */) @@ -1008,25 +823,6 @@ fun satisfiesBundlingRequirements(plugin: PluginLayout, osFamily: OsFamily?, arc } } -private suspend fun buildKeymapPlugins(targetDir: Path, context: BuildContext): List> { - val keymapDir = context.paths.communityHomeDir.resolve("platform/platform-resources/src/keymaps") - Files.createDirectories(targetDir) - return spanBuilder("build keymap plugins").use(Dispatchers.IO) { - listOf( - arrayOf("Mac OS X", "Mac OS X 10.5+"), - arrayOf("Default for GNOME"), - arrayOf("Default for KDE"), - arrayOf("Default for XWin"), - arrayOf("Emacs"), - arrayOf("Sublime Text", "Sublime Text (Mac OS X)"), - ).map { - async(CoroutineName("build keymap plugin for ${it[0]}")) { - buildKeymapPlugin(keymaps = it, buildNumber = context.buildNumber, targetDir = targetDir, keymapDir = keymapDir) - } - } - }.map { it.getCompleted() } -} - suspend fun layoutDistribution( layout: BaseLayout, platformLayout: PlatformLayout, @@ -1274,151 +1070,4 @@ private suspend fun checkModuleExcludes(moduleExcludes: Map " so it doesn't make sense to define excludes for it." } } -} - -private data class NonBundledPlugin( - @JvmField val sourceDir: Path, - @JvmField val targetZip: Path, - @JvmField val optimizedZip: Boolean, -) - -private suspend fun archivePlugins(items: Collection, compress: Boolean, withBlockMap: Boolean, context: BuildContext) { - context.executeStep( - spanBuilder = spanBuilder("archive plugins").setAttribute(AttributeKey.longKey("count"), items.size.toLong()), - stepId = BuildOptions.ARCHIVE_PLUGINS - ) { - val json by lazy { JSON.std.without(JSON.Feature.USE_FIELDS) } - for ((source, target, optimized) in items) { - launch(CoroutineName("archive plugin $source")) { - spanBuilder("archive plugin") - .setAttribute("input", source.toString()) - .setAttribute("outputFile", target.toString()) - .setAttribute("optimizedZip", optimized) - .use { - archivePlugin(optimized = optimized, target = target, compress = compress, source = source, context = context) - } - if (withBlockMap) { - spanBuilder("build plugin blockmap").setAttribute("file", target.toString()).use { - buildBlockMap(target, json) - } - } - } - } - } -} - -private fun archivePlugin(optimized: Boolean, target: Path, compress: Boolean, source: Path, context: BuildContext) { - if (optimized) { - writeNewZipWithoutIndex(target, compress) { zipCreator -> - val archiver = ZipArchiver() - if (Files.isDirectory(source)) { - archiver.setRootDir(source, source.fileName.toString()) - archiveDir(startDir = source, addFile = { archiver.addFile(it, zipCreator) }) - } - else { - archiver.setRootDir(source.parent) - archiver.addFile(source, zipCreator) - } - } - } - else { - writeNewFile(target) { outFileChannel -> - NoDuplicateZipArchiveOutputStream(outFileChannel, context.options.compressZipFiles).use { out -> - out.setUseZip64(Zip64Mode.Never) - out.dir(source, "${source.fileName}/", entryCustomizer = { entry, file, _ -> - if (Files.isExecutable(file)) { - entry.unixMode = executableFileUnixMode - } - }) - } - } - } -} - -/** - * Builds a blockmap and hash files for a plugin. - */ -private fun buildBlockMap(file: Path, json: JSON) { - val algorithm = "SHA-256" - val bytes = Files.newInputStream(file).use { input -> - json.asBytes(BlockMap(input, algorithm)) - } - - val fileParent = file.parent - val fileName = file.fileName.toString() - writeNewZipWithoutIndex(fileParent.resolve("$fileName.blockmap.zip"), compress = true) { - it.compressedData("blockmap.json", ByteBuffer.wrap(bytes)) - } - - val hashFile = fileParent.resolve("$fileName.hash.json") - Files.newInputStream(file).use { input -> - Files.newOutputStream(hashFile, *W_CREATE_NEW.toTypedArray()).use { output -> - json.write(FileHash(input, algorithm), output) - } - } -} - -private fun sortEntries(unsorted: List): List { - // sort because projectStructureMapping is a concurrent collection - // call invariantSeparatorsPathString because the result of Path ordering is platform-dependent - return unsorted.sortedWith( - compareBy( - { it.path.invariantSeparatorsPathString }, - { it.type }, - { (it as? ModuleOutputEntry)?.owner?.moduleName }, - { (it as? LibraryFileEntry)?.libraryFile?.let(::isFromLocalMavenRepo) != true }, - { (it as? LibraryFileEntry)?.libraryFile?.invariantSeparatorsPathString }, - ) - ) -} - -// also, put libraries from Maven repo ahead of others, for them to not depend on the lexicographical order of Maven repo and source path -private fun isFromLocalMavenRepo(path: Path) = path.startsWith(MAVEN_REPO) - -suspend fun createIdeClassPath(platformLayout: PlatformLayout, context: BuildContext): Collection { - val contentReport = generateProjectStructureMapping(platformLayout, context) - - val pluginLayouts = context.productProperties.productLayout.pluginLayouts - val classPath = LinkedHashSet() - - val libDir = context.paths.distAllDir.resolve("lib") - for (entry in sortEntries(contentReport.platform)) { - if (entry !is ModuleOutputEntry || !ModuleIncludeReasons.isProductModule(entry.reason)) { - val relativePath = libDir.relativize(entry.path) - if (relativePath.nameCount != 1 && !relativePath.startsWith("modules")) { - continue - } - } - - when (entry) { - is ModuleOutputEntry -> { - classPath.addAll(context.getModuleOutputRoots(context.findRequiredModule(entry.owner.moduleName))) - } - is LibraryFileEntry -> classPath.add(entry.libraryFile!!) - else -> throw UnsupportedOperationException("Entry $entry is not supported") - } - } - - val pluginDir = context.paths.distAllDir.resolve(PLUGINS_DIRECTORY) - for (entry in contentReport.bundledPlugins.flatMap { it.second }) { - val relativePath = pluginDir.relativize(entry.path) - // for plugins, our classloaders load JARs only from the "lib/" and "lib/modules/" directories - if (!(relativePath.nameCount in 3..4 && relativePath.getName(1).toString() == LIB_DIRECTORY && - (relativePath.nameCount == 3 || relativePath.getName(2).toString() == "modules"))) { - continue - } - - when (entry) { - is ModuleOutputEntry -> { - classPath.addAll(context.getModuleOutputRoots(context.findRequiredModule(entry.owner.moduleName))) - for (classpathPluginEntry in pluginLayouts.firstOrNull { it.mainModule == entry.owner.moduleName }?.scrambleClasspathPlugins ?: emptyList()) { - classPath.addAll(context.getModuleOutputRoots(context.findRequiredModule(classpathPluginEntry.pluginMainModuleName))) - } - } - is LibraryFileEntry -> classPath.add(entry.libraryFile!!) - is CustomAssetEntry -> { } - else -> throw UnsupportedOperationException("Entry $entry is not supported") - } - } - return classPath.map { it.toString() } -} +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt index 855a4c1085c7..59448d2fa0b0 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt @@ -312,6 +312,7 @@ class JarPackager private constructor( val packToDir = context.options.isUnpackedDist && !item.relativeOutputFile.contains('/') && + !item.isProductModule() && (patchedContent.isEmpty() || (patchedContent.size == 1 && patchedContent.containsKey("META-INF/plugin.xml"))) && extraExcludes.isEmpty() && moduleOutputRoots.isNotEmpty() @@ -324,7 +325,7 @@ class JarPackager private constructor( } else { assets.computeIfAbsent(outFile) { file -> - createAssetDescriptor(relativeOutputFile = item.relativeOutputFile, targetFile = file) + AssetDescriptor(isDir = false, file = file, relativePath = item.relativeOutputFile, useCacheAsTargetFile = !item.isProductModule()) } } @@ -761,7 +762,7 @@ class JarPackager private constructor( } private fun getJarAsset(targetFile: Path, relativeOutputFile: String): AssetDescriptor = assets.computeIfAbsent(targetFile) { - createAssetDescriptor(targetFile = targetFile, relativeOutputFile = relativeOutputFile) + AssetDescriptor(isDir = false, file = targetFile, relativePath = relativeOutputFile) } } @@ -1180,10 +1181,6 @@ private fun createModuleSource(module: JpsModule, outputDir: Path, excludes: Lis } } -private fun createAssetDescriptor(relativeOutputFile: String, targetFile: Path): AssetDescriptor { - return AssetDescriptor(isDir = false, file = targetFile, relativePath = relativeOutputFile) -} - private fun computeDistributionFileEntries( asset: AssetDescriptor, hasher: HashStream64, diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt index b0b645c952a6..58d998924f76 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PlatformModules.kt @@ -1,5 +1,6 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "RedundantSuppression", "ReplaceGetOrSet") + package org.jetbrains.intellij.build.impl import com.intellij.openapi.util.JDOMUtil @@ -57,7 +58,7 @@ private val PLATFORM_CORE_MODULES = java.util.List.of( "intellij.platform.usageView", "intellij.platform.execution", "intellij.platform.kernel", - + "intellij.platform.analysis.impl", "intellij.platform.diff.impl", "intellij.platform.editor.ex", @@ -86,7 +87,6 @@ private val PLATFORM_CORE_MODULES = java.util.List.of( // do we need it? "intellij.platform.sqlite", - //"fleet.rpc.server", // todo not used by platform - move to plugin "intellij.platform.ide.designer", "intellij.platform.ide.remote", @@ -148,7 +148,6 @@ internal suspend fun createPlatformLayout(projectLibrariesUsedByPlugins: SortedS "intellij.platform.util.trove", "intellij.platform.util.troveCompileOnly", ), productLayout = productLayout, layout = layout) - layout.withProjectLibrary(libraryName = "ion", jarName = UTIL_8_JAR) // maven uses JDOM in an external process addModule(UTIL_8_JAR, sequenceOf( @@ -158,7 +157,7 @@ internal suspend fun createPlatformLayout(projectLibrariesUsedByPlugins: SortedS "intellij.platform.util.base", "intellij.platform.util.base.multiplatform", "intellij.platform.diagnostic", - // it contains common telemetry related code (utils, TelemetryContext) for OpenTelemetry + // it contains common telemetry-related code (utils, TelemetryContext) for OpenTelemetry "intellij.platform.diagnostic.telemetry.rt", "intellij.platform.util", "intellij.platform.util.multiplatform", @@ -205,11 +204,6 @@ internal suspend fun createPlatformLayout(projectLibrariesUsedByPlugins: SortedS ) } - //used as a transitive dependency in the 'proxy-vole' library and excluded from it, so needs to be included explicitly - layout.withProjectLibrary("rhino") - - // used by intellij.database.jdbcConsole - put to a small util module - layout.withProjectLibrary(libraryName = "jbr-api", jarName = UTIL_JAR) // platform-loader.jar is loaded by JVM classloader as part of loading our custom PathClassLoader class - reduce file size addModule(PLATFORM_LOADER_JAR, sequenceOf( "intellij.platform.util.rt.java8", @@ -268,14 +262,17 @@ internal suspend fun createPlatformLayout(projectLibrariesUsedByPlugins: SortedS val productPluginContentModules = processAndGetProductPluginContentModules( context = context, layout = layout, - includedPlatformModulesPartialList = (layout.includedModules.asSequence().map { it.moduleName } + computeImplicitRequiredModules( - explicit = explicitModuleNames, - layout = layout, - productPluginContentModules = emptySet(), - productLayout = productLayout, - context = context, - validateImplicitPlatformModule = false, - ).asSequence().map { it.first } + explicitModuleNames).toList(), + includedPlatformModulesPartialList = layout.includedModules.asSequence().map { it.moduleName } + + computeImplicitRequiredModules( + explicit = explicitModuleNames, + layout = layout, + productPluginContentModules = emptySet(), + productLayout = productLayout, + context = context, + validateImplicitPlatformModule = false, + ).asSequence().map { it.first } + + explicitModuleNames + ) val implicit = computeImplicitRequiredModules( @@ -286,8 +283,21 @@ internal suspend fun createPlatformLayout(projectLibrariesUsedByPlugins: SortedS context = context, validateImplicitPlatformModule = context.options.validateImplicitPlatformModule, ) + + val filteredExplicit = LinkedHashSet(explicit) + for (item in productPluginContentModules) { + val iterator = filteredExplicit.iterator() + while (iterator.hasNext()) { + if (item.moduleName == iterator.next().moduleName) { + // todo - error instead of warn + Span.current().addEvent("product module MUST NOT BE explicitly specified: ${item.moduleName}") + iterator.remove() + } + } + } + layout.withModules( - (explicit.asSequence() + + (filteredExplicit.asSequence() + productPluginContentModules + implicit.asSequence().map { ModuleItem( @@ -324,11 +334,13 @@ internal suspend fun createPlatformLayout(projectLibrariesUsedByPlugins: SortedS } layout.includedProjectLibraries - .addOrGet(ProjectLibraryData( - libraryName = libName, - packMode = PLATFORM_CUSTOM_PACK_MODE.getOrDefault(libName, LibraryPackMode.MERGED), - reason = "<- ${module.name}", - )) + .addOrGet( + ProjectLibraryData( + libraryName = libName, + packMode = PLATFORM_CUSTOM_PACK_MODE.getOrDefault(libName, LibraryPackMode.MERGED), + reason = "<- ${module.name}", + ) + ) .dependentModules.computeIfAbsent("core") { mutableListOf() }.add(module.name) } @@ -359,11 +371,11 @@ internal suspend fun createPlatformLayout(projectLibrariesUsedByPlugins: SortedS */ fun collectExportedLibrariesFromLibraryModules( layout: PlatformLayout, - context: BuildContext + context: BuildContext, ): Map { val javaExtensionService = JpsJavaExtensionService.getInstance() val result = mutableMapOf() - + layout.includedModules .asSequence() .filter { it.moduleName.startsWith(LIB_MODULE_PREFIX) } @@ -382,7 +394,7 @@ fun collectExportedLibrariesFromLibraryModules( result[libName] = moduleItem.moduleName } } - + return result } @@ -544,7 +556,7 @@ private fun computeTransitive( private suspend fun processAndGetProductPluginContentModules( context: BuildContext, layout: PlatformLayout, - includedPlatformModulesPartialList: List, + includedPlatformModulesPartialList: Sequence, ): Set { val xIncludePathResolver = createXIncludePathResolver(includedPlatformModulesPartialList, context) return withContext(Dispatchers.IO) { @@ -583,7 +595,7 @@ private val COMMUNITY_IMPL_EXTENSIONS = setOf( "/META-INF/community-extensions.xml" ) -fun createXIncludePathResolver(includedPlatformModulesPartialList: List, context: BuildContext): XIncludePathResolver { +fun createXIncludePathResolver(includedPlatformModulesPartialList: Sequence, context: BuildContext): XIncludePathResolver { return object : XIncludePathResolver { override fun resolvePath(relativePath: String, base: Path?, isOptional: Boolean, isDynamic: Boolean): Path? { if ((isOptional || isDynamic || excludedPaths.contains(relativePath)) && !COMMUNITY_IMPL_EXTENSIONS.contains(relativePath)) { @@ -607,7 +619,7 @@ fun createXIncludePathResolver(includedPlatformModulesPartialList: List, } } - for (module in includedPlatformModulesPartialList) { + for (module in includedPlatformModulesPartialList.distinct()) { findFileInModuleSources(context.findRequiredModule(module), loadPath)?.let { return it } @@ -680,29 +692,24 @@ private suspend fun collectAndEmbedProductModules(root: Element, xIncludePathRes } val isEmbedded = loadingRule == "embedded" - val relativeOutFile = if (isEmbedded) { - if (isModuleCloseSource(moduleName, context = context)) { - if (frontendModuleFilter.isBackendModule(moduleName)) PRODUCT_BACKEND_JAR else PRODUCT_JAR - } - else { - // todo Change location after feature-freeze (and maybe by that time, https://youtrack.jetbrains.com/issue/IJPL-209476/extract-core-out-of-core will be ready). - // Using `modules` before release is not a safe option. - "module-$moduleName.jar" - } + val relativeOutFile = if (isEmbedded && isModuleCloseSource(moduleName, context = context)) { + if (frontendModuleFilter.isBackendModule(moduleName)) PRODUCT_BACKEND_JAR else PRODUCT_JAR } else { - "modules/$moduleName.jar" + "$moduleName.jar" } // Extract module set from parent element's source-file attribute val contentElement = moduleElement.parentElement val moduleSet = contentElement?.getAttributeValue(SOURCE_FILE_ATTRIBUTE)?.removeSuffix(".xml") - result.add(ModuleItem( - moduleName = moduleName, - relativeOutputFile = relativeOutFile, - reason = if (isEmbedded) ModuleIncludeReasons.PRODUCT_EMBEDDED_MODULES else ModuleIncludeReasons.PRODUCT_MODULES, - moduleSet = moduleSet, - )) + result.add( + ModuleItem( + moduleName = moduleName, + relativeOutputFile = relativeOutFile, + reason = if (isEmbedded) ModuleIncludeReasons.PRODUCT_EMBEDDED_MODULES else ModuleIncludeReasons.PRODUCT_MODULES, + moduleSet = moduleSet, + ) + ) PRODUCT_MODULE_IMPL_COMPOSITION.get(moduleName)?.let { it.mapTo(result) { subModuleName -> ModuleItem(moduleName = subModuleName, relativeOutputFile = relativeOutFile, reason = ModuleIncludeReasons.PRODUCT_MODULES, moduleSet = moduleSet) @@ -723,11 +730,11 @@ private suspend fun collectAndEmbedProductModules(root: Element, xIncludePathRes @Suppress("RemoveRedundantQualifierName") private val PRODUCT_MODULE_IMPL_COMPOSITION = java.util.Map.of( "intellij.rider", listOf( - "intellij.platform.debugger.modulesView" - ), + "intellij.platform.debugger.modulesView" +), "intellij.platform.rpc.backend", listOf( - "fleet.rpc.server", - ) + "fleet.rpc.server", +) ) internal object ModuleIncludeReasons { diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginRepositoryXmlGenerator.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginRepositoryXmlGenerator.kt index ccb3e030bd90..28aec232e12e 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginRepositoryXmlGenerator.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginRepositoryXmlGenerator.kt @@ -20,7 +20,7 @@ import kotlin.io.path.createDirectories internal fun generatePluginRepositoryMetaFile(pluginSpecs: List, targetDir: Path, buildNumber: String): Path { val categories = TreeMap>() for (spec in pluginSpecs) { - val p = readPlugin(spec.pluginZip, spec.pluginXml, buildNumber, targetDir) + val p = readPlugin(pluginZip = spec.pluginZip, pluginXml = spec.pluginXml, buildNumber = buildNumber, targetDirectory = targetDir) categories.computeIfAbsent(p.category) { mutableListOf() }.add(p) } Files.createDirectories(targetDir) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt index dd7651058bac..7f66e137460b 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/PluginXmlPatcher.kt @@ -78,7 +78,10 @@ internal suspend fun patchPluginXml( embedContentModules( xml = element, file = findFileInModuleSources(module = pluginModule, relativePath = "META-INF/plugin.xml")!!, - xIncludePathResolver = createXIncludePathResolver(plugin.includedModules.map { it.moduleName } + platformLayout.includedModules.map { it.moduleName }, context), + xIncludePathResolver = createXIncludePathResolver( + includedPlatformModulesPartialList = plugin.includedModules.asSequence().map { it.moduleName } + platformLayout.includedModules.asSequence().map { it.moduleName }, + context = context, + ), layout = plugin, context = context, ) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt new file mode 100644 index 000000000000..850097956638 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/deprecatedClasspath.kt @@ -0,0 +1,82 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.impl + +import org.jetbrains.intellij.build.BuildContext +import org.jetbrains.intellij.build.MAVEN_REPO +import org.jetbrains.intellij.build.impl.projectStructureMapping.CustomAssetEntry +import org.jetbrains.intellij.build.impl.projectStructureMapping.DistributionFileEntry +import org.jetbrains.intellij.build.impl.projectStructureMapping.LibraryFileEntry +import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleLibraryFileEntry +import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleOutputEntry +import org.jetbrains.intellij.build.impl.projectStructureMapping.ModuleOwnedFileEntry +import java.nio.file.Path +import kotlin.io.path.invariantSeparatorsPathString + +suspend fun createIdeClassPath(platformLayout: PlatformLayout, context: BuildContext): Collection { + val contentReport = generateProjectStructureMapping(platformLayout, context) + + val pluginLayouts = context.productProperties.productLayout.pluginLayouts + val classPath = LinkedHashSet() + + val libDir = context.paths.distAllDir.resolve("lib") + for (entry in sortEntries(contentReport.platform)) { + if (entry !is ModuleOutputEntry || !ModuleIncludeReasons.isProductModule(entry.reason)) { + val relativePath = libDir.relativize(entry.path) + if (relativePath.nameCount != 1) { + continue + } + } + + if (entry is ModuleLibraryFileEntry && entry.libraryName == "jetbrains.lets.plot.shadowed") { + continue + } + + when (entry) { + is ModuleOutputEntry -> { + classPath.addAll(context.getModuleOutputRoots(context.findRequiredModule(entry.owner.moduleName))) + } + is LibraryFileEntry -> classPath.add(entry.libraryFile!!) + else -> throw UnsupportedOperationException("Entry $entry is not supported") + } + } + + val pluginDir = context.paths.distAllDir.resolve(PLUGINS_DIRECTORY) + for (entry in contentReport.bundledPlugins.flatMap { it.second }) { + val relativePath = pluginDir.relativize(entry.path) + // for plugins, our classloaders load JARs only from the "lib/" and "lib/modules/" directories + if (!(relativePath.nameCount in 3..4 && relativePath.getName(1).toString() == LIB_DIRECTORY && + (relativePath.nameCount == 3 || relativePath.getName(2).toString() == "modules"))) { + continue + } + + when (entry) { + is ModuleOutputEntry -> { + classPath.addAll(context.getModuleOutputRoots(context.findRequiredModule(entry.owner.moduleName))) + for (classpathPluginEntry in pluginLayouts.firstOrNull { it.mainModule == entry.owner.moduleName }?.scrambleClasspathPlugins ?: emptyList()) { + classPath.addAll(context.getModuleOutputRoots(context.findRequiredModule(classpathPluginEntry.pluginMainModuleName))) + } + } + is LibraryFileEntry -> classPath.add(entry.libraryFile!!) + is CustomAssetEntry -> { } + else -> throw UnsupportedOperationException("Entry $entry is not supported") + } + } + return classPath.map { it.toString() } +} + +private fun sortEntries(unsorted: List): List { + // sort because projectStructureMapping is a concurrent collection + // call invariantSeparatorsPathString because the result of Path ordering is platform-dependent + return unsorted.sortedWith( + compareBy( + { it.path.invariantSeparatorsPathString }, + { it.type }, + { (it as? ModuleOutputEntry)?.owner?.moduleName }, + { (it as? LibraryFileEntry)?.libraryFile?.let(::isFromLocalMavenRepo) != true }, + { (it as? LibraryFileEntry)?.libraryFile?.invariantSeparatorsPathString }, + ) + ) +} + +// also, put libraries from Maven repo ahead of others, for them to not depend on the lexicographical order of Maven repo and source path +private fun isFromLocalMavenRepo(path: Path) = path.startsWith(MAVEN_REPO) \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/PluginAutoPublishList.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/PluginAutoPublishList.kt index d0936fe31028..ab603ab690a3 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/PluginAutoPublishList.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/PluginAutoPublishList.kt @@ -2,7 +2,7 @@ package org.jetbrains.intellij.build.impl.plugins import com.intellij.openapi.util.text.StringUtil -import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.ApiStatus.Internal import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.impl.PluginLayout import java.nio.file.Files @@ -18,7 +18,7 @@ import kotlin.io.path.useLines * @see [org.jetbrains.intellij.build.ProductModulesLayout.buildAllCompatiblePlugins] * @see [org.jetbrains.intellij.build.ProductModulesLayout.pluginModulesToPublish] */ -@ApiStatus.Internal +@Internal class PluginAutoPublishList(private val context: BuildContext) : Predicate { private val expectedFile: Path = context.paths.communityHomeDir.resolve("../build/plugins-autoupload.txt") diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt new file mode 100644 index 000000000000..a80138e770d6 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/plugins/nonBundled.kt @@ -0,0 +1,353 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("TestOnlyProblems") + +package org.jetbrains.intellij.build.impl.plugins + +import com.fasterxml.jackson.jr.ob.JSON +import com.intellij.openapi.util.io.NioFiles +import com.jetbrains.plugin.blockmap.core.BlockMap +import com.jetbrains.plugin.blockmap.core.FileHash +import com.jetbrains.plugin.structure.base.plugin.PluginCreationFail +import com.jetbrains.plugin.structure.base.plugin.PluginCreationSuccess +import com.jetbrains.plugin.structure.intellij.plugin.IdePluginManager +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.common.Attributes +import io.opentelemetry.api.trace.Span +import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.async +import kotlinx.coroutines.launch +import org.apache.commons.compress.archivers.zip.Zip64Mode +import org.jetbrains.intellij.build.BuildContext +import org.jetbrains.intellij.build.BuildOptions +import org.jetbrains.intellij.build.PluginBuildDescriptor +import org.jetbrains.intellij.build.SearchableOptionSetDescriptor +import org.jetbrains.intellij.build.executeStep +import org.jetbrains.intellij.build.impl.BUILT_IN_HELP_MODULE_NAME +import org.jetbrains.intellij.build.impl.BuildContextImpl +import org.jetbrains.intellij.build.impl.DistributionBuilderState +import org.jetbrains.intellij.build.impl.ModuleOutputPatcher +import org.jetbrains.intellij.build.impl.NoDuplicateZipArchiveOutputStream +import org.jetbrains.intellij.build.impl.PLUGIN_LAYOUT_COMPARATOR_BY_MAIN_MODULE +import org.jetbrains.intellij.build.impl.PluginLayout +import org.jetbrains.intellij.build.impl.PluginRepositorySpec +import org.jetbrains.intellij.build.impl.buildHelpPlugin +import org.jetbrains.intellij.build.impl.buildKeymapPlugin +import org.jetbrains.intellij.build.impl.buildPlugins +import org.jetbrains.intellij.build.impl.dir +import org.jetbrains.intellij.build.impl.executableFileUnixMode +import org.jetbrains.intellij.build.impl.generatePluginRepositoryMetaFile +import org.jetbrains.intellij.build.impl.handleCustomPlatformSpecificAssets +import org.jetbrains.intellij.build.impl.nonBundledPluginsStageDir +import org.jetbrains.intellij.build.impl.projectStructureMapping.DistributionFileEntry +import org.jetbrains.intellij.build.io.W_CREATE_NEW +import org.jetbrains.intellij.build.io.ZipArchiver +import org.jetbrains.intellij.build.io.archiveDir +import org.jetbrains.intellij.build.io.writeNewFile +import org.jetbrains.intellij.build.io.writeNewZipWithoutIndex +import org.jetbrains.intellij.build.io.zipWithCompression +import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder +import org.jetbrains.intellij.build.telemetry.use +import java.nio.ByteBuffer +import java.nio.file.Files +import java.nio.file.Path +import java.util.concurrent.ConcurrentLinkedQueue +import kotlin.io.use + +internal suspend fun buildNonBundledPlugins( + pluginsToPublish: Set, + compressPluginArchive: Boolean, + buildPlatformLibJob: Job?, + state: DistributionBuilderState, + searchableOptionSet: SearchableOptionSetDescriptor?, + context: BuildContext, +): List>> { + return context.executeStep(spanBuilder("build non-bundled plugins").setAttribute("count", state.pluginsToPublish.size.toLong()), BuildOptions.NON_BUNDLED_PLUGINS_STEP) { + doBuildNonBundledPlugins( + pluginsToPublish = pluginsToPublish, + compressPluginArchive = compressPluginArchive, + buildPlatformLibJob = buildPlatformLibJob, + state = state, + searchableOptionSet = searchableOptionSet, + isUpdateFromSources = false, + context = context, + ) + } ?: emptyList() +} + +internal suspend fun CoroutineScope.doBuildNonBundledPlugins( + pluginsToPublish: Set, + compressPluginArchive: Boolean, + buildPlatformLibJob: Job?, + state: DistributionBuilderState, + searchableOptionSet: SearchableOptionSetDescriptor?, + isUpdateFromSources: Boolean, + context: BuildContext, +): List>> { + if (pluginsToPublish.isEmpty()) { + return emptyList() + } + + val buildKeymapPluginsTask = if (context.options.buildStepsToSkip.contains(BuildOptions.KEYMAP_PLUGINS_STEP)) { + null + } + else { + async(CoroutineName("build keymap plugins")) { + buildKeymapPlugins(targetDir = context.nonBundledPluginsToBePublished, context = context) + } + } + + val moduleOutputPatcher = ModuleOutputPatcher() + val stageDir = nonBundledPluginsStageDir(context) + NioFiles.deleteRecursively(stageDir) + + // buildPlugins pluginBuilt listener is called concurrently + val pluginSpecs = ConcurrentLinkedQueue() + val isPluginArchiveEnabled = !context.isStepSkipped(BuildOptions.ARCHIVE_PLUGINS) + val prepareCustomPluginRepository = context.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins && isPluginArchiveEnabled + val plugins = pluginsToPublish.sortedWith(PLUGIN_LAYOUT_COMPARATOR_BY_MAIN_MODULE) + val isPluginValidationEnabled = !isUpdateFromSources && !context.isStepSkipped(BuildOptions.VALIDATE_PLUGINS_TO_BE_PUBLISHED) + val json: Lazy = lazy { JSON.std.without(JSON.Feature.USE_FIELDS) } + val mappings = buildPlugins( + moduleOutputPatcher = moduleOutputPatcher, + plugins = plugins, + os = null, + targetDir = stageDir, + state = state, + context = context, + buildPlatformJob = buildPlatformLibJob, + searchableOptionSet = searchableOptionSet, + pluginBuilt = { plugin, pluginDirOrFile -> + val pluginVersion = if (plugin.mainModule == BUILT_IN_HELP_MODULE_NAME) { + context.buildNumber + } + else { + plugin.versionEvaluator.evaluate( + pluginXmlSupplier = { (context as BuildContextImpl).jarPackagerDependencyHelper.getPluginXmlContent(context.findRequiredModule(plugin.mainModule)) }, + ideBuildVersion = context.pluginBuildNumber, + context, + ).pluginVersion + } + + val targetDirectory = if (context.pluginAutoPublishList.test(plugin)) { + context.nonBundledPluginsToBePublished + } + else { + context.nonBundledPlugins + } + val destFile = targetDirectory.resolve("${plugin.directoryName}-$pluginVersion.zip") + val pluginXml = moduleOutputPatcher.getPatchedPluginXml(plugin.mainModule) + pluginSpecs.add(PluginRepositorySpec(destFile, pluginXml)) + + val entries = handleCustomPlatformSpecificAssets( + layout = plugin, + targetPlatform = null, + context = context, + pluginDir = pluginDirOrFile, + isDevMode = true, + ) + + if (isPluginArchiveEnabled) { + archivePlugin( + optimizedZip = !plugin.enableSymlinksAndExecutableResources, + source = pluginDirOrFile, + target = destFile, + compress = compressPluginArchive, + withBlockMap = compressPluginArchive, + context = context, + json = json, + ) + + if (isPluginValidationEnabled) { + spanBuilder("plugin validation").use { span -> + if (Files.notExists(destFile)) { + span.addEvent("doesn't exist, skipped", Attributes.of(AttributeKey.stringKey("path"), "$destFile")) + } + else { + validatePlugin(file = destFile, context = context, span = span) + } + } + } + } + + entries + }, + ) + + val helpPlugin = buildHelpPlugin(context.pluginBuildNumber, context) + if (helpPlugin != null) { + val spec = buildHelpPlugin( + helpPlugin = helpPlugin, + pluginsToPublishDir = stageDir, + targetDir = context.nonBundledPluginsToBePublished, + moduleOutputPatcher = moduleOutputPatcher, + state = state, + searchableOptionSetDescriptor = searchableOptionSet, + context = context, + ) + pluginSpecs.add(spec) + } + + buildKeymapPluginsTask?.let { + for (item in it.await()) { + pluginSpecs.add(PluginRepositorySpec(pluginZip = item.first, pluginXml = item.second)) + } + } + + if (prepareCustomPluginRepository) { + val list = pluginSpecs.sortedBy { it.pluginZip } + if (list.isNotEmpty()) { + launch { + generatePluginRepositoryMetaFile(pluginSpecs = list, targetDir = context.nonBundledPlugins, buildNumber = context.buildNumber) + } + } + + val pluginsToBePublished = list.filter { it.pluginZip.startsWith(context.nonBundledPluginsToBePublished) } + if (pluginsToBePublished.isNotEmpty()) { + launch { + generatePluginRepositoryMetaFile(pluginSpecs = pluginsToBePublished, targetDir = context.nonBundledPluginsToBePublished, buildNumber = context.buildNumber) + } + } + } + + return mappings +} + +private suspend fun archivePlugin( + source: Path, + target: Path, + compress: Boolean, + optimizedZip: Boolean, + withBlockMap: Boolean, + json: Lazy, + context: BuildContext, +) { + spanBuilder("archive plugin") + .setAttribute("input", source.toString()) + .setAttribute("outputFile", target.toString()) + .setAttribute("optimizedZip", optimizedZip) + .use { + archivePlugin(optimized = optimizedZip, target = target, compress = compress, source = source, context = context) + } + if (withBlockMap) { + spanBuilder("build plugin blockmap").setAttribute("file", target.toString()).use { + buildBlockMap(target, json.value) + } + } +} + +private fun archivePlugin(optimized: Boolean, target: Path, compress: Boolean, source: Path, context: BuildContext) { + if (optimized) { + writeNewZipWithoutIndex(target, compress) { zipCreator -> + val archiver = ZipArchiver() + if (Files.isDirectory(source)) { + archiver.setRootDir(source, source.fileName.toString()) + archiveDir(startDir = source, addFile = { archiver.addFile(it, zipCreator) }) + } + else { + archiver.setRootDir(source.parent) + archiver.addFile(source, zipCreator) + } + } + } + else { + writeNewFile(target) { outFileChannel -> + NoDuplicateZipArchiveOutputStream(outFileChannel, context.options.compressZipFiles).use { out -> + out.setUseZip64(Zip64Mode.Never) + out.dir(source, "${source.fileName}/", entryCustomizer = { entry, file, _ -> + if (Files.isExecutable(file)) { + entry.unixMode = executableFileUnixMode + } + }) + } + } + } +} + +private suspend fun buildKeymapPlugins(targetDir: Path, context: BuildContext): List> { + val keymapDir = context.paths.communityHomeDir.resolve("platform/platform-resources/src/keymaps") + Files.createDirectories(targetDir) + return spanBuilder("build keymap plugins").use(Dispatchers.IO) { + listOf( + arrayOf("Mac OS X", "Mac OS X 10.5+"), + arrayOf("Default for GNOME"), + arrayOf("Default for KDE"), + arrayOf("Default for XWin"), + arrayOf("Emacs"), + arrayOf("Sublime Text", "Sublime Text (Mac OS X)"), + ).map { + async(CoroutineName("build keymap plugin for ${it[0]}")) { + buildKeymapPlugin(keymaps = it, buildNumber = context.buildNumber, targetDir = targetDir, keymapDir = keymapDir) + } + } + }.map { it.getCompleted() } +} + +/** + * Builds a blockmap and hash files for a plugin. + */ +private fun buildBlockMap(file: Path, json: JSON) { + val algorithm = "SHA-256" + val bytes = Files.newInputStream(file).use { input -> + json.asBytes(BlockMap(input, algorithm)) + } + + val fileParent = file.parent + val fileName = file.fileName.toString() + writeNewZipWithoutIndex(fileParent.resolve("$fileName.blockmap.zip"), compress = true) { + it.compressedData("blockmap.json", ByteBuffer.wrap(bytes)) + } + + val hashFile = fileParent.resolve("$fileName.hash.json") + Files.newInputStream(file).use { input -> + Files.newOutputStream(hashFile, *W_CREATE_NEW.toTypedArray()).use { output -> + json.write(FileHash(input, algorithm), output) + } + } +} + +private fun validatePlugin(file: Path, context: BuildContext, span: Span) { + val pluginManager = IdePluginManager.createManager() + val result = pluginManager.createPlugin(pluginFile = file, validateDescriptor = true) + // todo fix AddStatisticsEventLogListenerTemporary + val id = when (result) { + is PluginCreationSuccess -> result.plugin.pluginId + is PluginCreationFail -> (pluginManager.createPlugin(pluginFile = file, validateDescriptor = false) as? PluginCreationSuccess)?.plugin?.pluginId + } + for (problem in context.productProperties.validatePlugin(id, result, context)) { + val problemType = problem::class.java.simpleName + span.addEvent( + "plugin validation failed", Attributes.of( + AttributeKey.stringKey("id"), "$id", + AttributeKey.stringKey("path"), "$file", + AttributeKey.stringKey("problemType"), problemType, + ) + ) + context.messages.reportBuildProblem( + description = "${id ?: file}, $problemType: $problem", + identity = "${id ?: file}$problemType" + ) + } +} + +private suspend fun buildHelpPlugin( + helpPlugin: PluginLayout, + pluginsToPublishDir: Path, + targetDir: Path, + moduleOutputPatcher: ModuleOutputPatcher, + state: DistributionBuilderState, + searchableOptionSetDescriptor: SearchableOptionSetDescriptor?, + context: BuildContext, +): PluginRepositorySpec { + val directory = helpPlugin.directoryName + val destFile = targetDir.resolve("$directory.zip") + spanBuilder("build help plugin").setAttribute("dir", directory).use { + val targetDir = pluginsToPublishDir.resolve(directory) + buildPlugins(moduleOutputPatcher, listOf(helpPlugin), os = null, targetDir, state, context, buildPlatformJob = null, searchableOptionSetDescriptor) + zipWithCompression(targetFile = destFile, dirs = mapOf(targetDir to "")) + null + } + return PluginRepositorySpec(pluginZip = destFile, pluginXml = moduleOutputPatcher.getPatchedPluginXml(helpPlugin.mainModule)) +} \ No newline at end of file diff --git a/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt b/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt index 0b31ab541f4b..a9be458d0f14 100644 --- a/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt +++ b/platform/core-impl/src/com/intellij/ide/plugins/PluginDescriptorLoader.kt @@ -912,7 +912,8 @@ internal fun loadCoreProductPlugin( loadContentModuleDescriptors( descriptor = descriptor, pathResolver = pathResolver, - libDir = libDir, + moduleDir = libDir, + useModuleDirAsParent = true, loadingContext = loadingContext, dataLoader = dataLoader, isRunningFromSourcesWithoutDevBuild = isRunningFromSourcesWithoutDevBuild, @@ -924,12 +925,12 @@ internal fun loadCoreProductPlugin( private fun loadContentModuleDescriptors( descriptor: PluginMainDescriptor, pathResolver: PathResolver, - libDir: Path, + moduleDir: Path, + useModuleDirAsParent: Boolean, loadingContext: PluginDescriptorLoadingContext, dataLoader: DataLoader, isRunningFromSourcesWithoutDevBuild: Boolean, ) { - val moduleDir = libDir.resolve("modules") val moduleDirExists = Files.isDirectory(moduleDir) val loadingStrategy = ProductLoadingStrategy.strategy @@ -942,10 +943,10 @@ private fun loadContentModuleDescriptors( val subDescriptorFile = "${moduleId.name}.xml" if (moduleDirExists && - !isRunningFromSourcesWithoutDevBuild && moduleId.name.startsWith("intellij.") && + !isRunningFromSourcesWithoutDevBuild && + moduleId.name.startsWith("intellij.") && loadProductModule( - loadingStrategy = loadingStrategy, - moduleDir = moduleDir, + jarFile = if (useModuleDirAsParent) moduleDir.resolve("$moduleId.jar") else loadingStrategy.findProductContentModuleClassesRoot(moduleId, moduleDir), module = module, subDescriptorFile = subDescriptorFile, loadingContext = loadingContext, @@ -967,8 +968,7 @@ private fun loadContentModuleDescriptors( } private fun loadProductModule( - loadingStrategy: ProductLoadingStrategy, - moduleDir: Path, + jarFile: Path?, module: PluginContentDescriptor.ModuleItem, subDescriptorFile: String, loadingContext: PluginDescriptorLoadingContext, @@ -977,7 +977,6 @@ private fun loadProductModule( containerDescriptor: PluginMainDescriptor, ): Boolean { val moduleId = module.moduleId - val jarFile = loadingStrategy.findProductContentModuleClassesRoot(moduleId, moduleDir) val moduleRaw: PluginDescriptorBuilder = if (jarFile == null) { // do not log - the severity of the error is determined by the loadingStrategy, the default strategy does not return null at all PluginDescriptorBuilder.builder().apply { @@ -1262,7 +1261,8 @@ internal fun loadDescriptorFromResource( loadContentModuleDescriptors( descriptor = descriptor, pathResolver = pathResolver, - libDir = libDir, + moduleDir = libDir.resolve("modules"), + useModuleDirAsParent = false, loadingContext = loadingContext, dataLoader = dataLoader, isRunningFromSourcesWithoutDevBuild = pathResolver.isRunningFromSourcesWithoutDevBuild,