diff --git a/build/tasks/src/org/jetbrains/intellij/build/io/ZipArchiveOutputStream.kt b/build/tasks/src/org/jetbrains/intellij/build/io/ZipArchiveOutputStream.kt index 6f522699d681..a96f45cd0d0d 100644 --- a/build/tasks/src/org/jetbrains/intellij/build/io/ZipArchiveOutputStream.kt +++ b/build/tasks/src/org/jetbrains/intellij/build/io/ZipArchiveOutputStream.kt @@ -354,6 +354,8 @@ internal class ZipArchiveOutputStream(private val channel: WritableByteChannel, buffer.putShort(0) } + internal fun getChannelPosition() = channelPosition + internal fun getChannelPositionAndAdd(increment: Int): Long { val p = channelPosition channelPosition += increment.toLong() diff --git a/build/tasks/src/org/jetbrains/intellij/build/tasks/mergeJars.kt b/build/tasks/src/org/jetbrains/intellij/build/tasks/mergeJars.kt index 342d1a04527d..bbae29006e16 100644 --- a/build/tasks/src/org/jetbrains/intellij/build/tasks/mergeJars.kt +++ b/build/tasks/src/org/jetbrains/intellij/build/tasks/mergeJars.kt @@ -170,7 +170,7 @@ fun buildJar(targetFile: Path, sources: List, dryRun: Boolean = false) { } } - source.sizeConsumer?.accept((outChannel.position() - positionBefore).toInt()) + source.sizeConsumer?.accept((zipCreator.resultStream.getChannelPosition() - positionBefore).toInt()) } packageIndexBuilder.writePackageIndex(zipCreator) } @@ -181,6 +181,8 @@ private fun getIgnoredNames(): Set { val set = HashSet() // compilation cache on TC set.add(".hash") + @Suppress("SpellCheckingInspection") + set.add(".gitattributes") set.add("pom.xml") set.add("about.html") set.add("module-info.class") diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/BuildOptions.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/BuildOptions.groovy index 460a5e1c310c..9fbdfad3c0dd 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/BuildOptions.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/BuildOptions.groovy @@ -5,6 +5,7 @@ import com.intellij.openapi.util.SystemInfo import com.intellij.openapi.util.text.StringUtil import com.intellij.util.SystemProperties import groovy.transform.CompileStatic +import org.jetbrains.annotations.ApiStatus @CompileStatic final class BuildOptions { @@ -57,7 +58,7 @@ final class BuildOptions { public static final String BROKEN_PLUGINS_LIST_STEP = "broken_plugins_list" static final String PROVIDED_MODULES_LIST_STEP = "provided_modules_list" public static final String GENERATE_JAR_ORDER_STEP = "jar_order" - static final String SOURCES_ARCHIVE_STEP = "sources_archive" + public static final String SOURCES_ARCHIVE_STEP = "sources_archive" public static final String SCRAMBLING_STEP = "scramble" public static final String NON_BUNDLED_PLUGINS_STEP = "non_bundled_plugins" /** Build Maven artifacts for IDE modules. */ @@ -88,6 +89,7 @@ final class BuildOptions { static final String COMMUNITY_DIST_STEP = "community_dist" public static final String PREBUILD_SHARED_INDEXES = "prebuild_shared_indexes" public static final String SETUP_BUNDLED_MAVEN = "setup_bundled_maven" + public static final String VERIFY_CLASS_FILE_VERSIONS = "verify_class_file_versions" /** * Publish artifacts to TeamCity storage while the build is still running, immediately after the artifacts are built. * Comprises many small publication steps. @@ -244,6 +246,9 @@ final class BuildOptions { static final String VALIDATE_MODULES_STRUCTURE = "intellij.build.module.structure" boolean validateModuleStructure = System.getProperty(VALIDATE_MODULES_STRUCTURE, "false").toBoolean() + @ApiStatus.Internal + public boolean compressNonBundledPluginArchive = true + /** * Max attempts of dependencies resolution on fault. "1" means no retries. * diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BaseLayout.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BaseLayout.groovy index 01bd3d50b669..7b10a3b99a43 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BaseLayout.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BaseLayout.groovy @@ -4,6 +4,8 @@ package org.jetbrains.intellij.build.impl import com.intellij.openapi.util.text.Strings import com.intellij.util.containers.MultiMap import groovy.transform.CompileStatic +import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet +import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.NotNull /** @@ -21,7 +23,8 @@ abstract class BaseLayout { final List resourcePaths = new ArrayList<>() /** module name to entries which should be excluded from its output */ final MultiMap moduleExcludes = MultiMap.createLinked() - final Set includedProjectLibraries = new LinkedHashSet<>() + @ApiStatus.Internal + public final ObjectLinkedOpenHashSet includedProjectLibraries = new ObjectLinkedOpenHashSet<>() final Set includedModuleLibraries = new LinkedHashSet<>() /** module name to name of the module library */ final MultiMap excludedModuleLibraries = MultiMap.createLinked() diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy index dd688534107d..5aafe3414871 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy @@ -191,22 +191,24 @@ final class BuildTasksImpl extends BuildTasks { /** * Build a list with modules that the IDE will provide for plugins. */ - private static void buildProvidedModuleList(BuildContext buildContext, Path targetFile, @NotNull Collection modules) { - buildContext.executeStep(spanBuilder("build provided module list").setAttribute("size", modules.size()), - BuildOptions.PROVIDED_MODULES_LIST_STEP, new Runnable() { - @Override - void run() { - buildContext.messages.debug("Building provided module list for the following modules: $modules") - Files.deleteIfExists(targetFile) - // start the product in headless mode using com.intellij.ide.plugins.BundledPluginsLister - BuildHelper.runApplicationStarter(buildContext, buildContext.paths.tempDir.resolve("builtinModules"), modules, - List.of("listBundledPlugins", targetFile.toString())) - if (Files.notExists(targetFile)) { - buildContext.messages.error("Failed to build provided modules list: $targetFile doesn't exist") + private static void buildProvidedModuleList(BuildContext context, Path targetFile, @NotNull Collection modules) { + context.executeStep( + spanBuilder("build provided module list") + .setAttribute(AttributeKey.stringArrayKey("modules"), List.copyOf(modules)), + BuildOptions.PROVIDED_MODULES_LIST_STEP, + new Runnable() { + @Override + void run() { + Files.deleteIfExists(targetFile) + // start the product in headless mode using com.intellij.ide.plugins.BundledPluginsLister + BuildHelper.runApplicationStarter(context, context.paths.tempDir.resolve("builtinModules"), modules, + List.of("listBundledPlugins", targetFile.toString())) + if (Files.notExists(targetFile)) { + context.messages.error("Failed to build provided modules list: $targetFile doesn't exist") + } + context.notifyArtifactWasBuilt(targetFile) } - buildContext.notifyArtifactWasBuilt(targetFile) - } - }) + }) } private Path patchIdeaPropertiesFile() { @@ -310,8 +312,8 @@ idea.fatal.error.notification=disabled @NotNull private static BuildTaskRunnable createDistributionForOsTask(@NotNull OsFamily os, - @NotNull Map result, - @NotNull Function factory) { + @NotNull Map result, + @NotNull Function factory) { return BuildTaskRunnable.task(os.osId, new Consumer() { @Override void accept(BuildContext context) { @@ -340,36 +342,40 @@ idea.fatal.error.notification=disabled @Override void compileModulesFromProduct() { checkProductProperties() - compileModulesForDistribution() + compileModulesForDistribution(buildContext) } - private DistributionJARsBuilder compileModulesForDistribution() { - ProductModulesLayout productLayout = buildContext.productProperties.productLayout - Collection moduleNames = DistributionJARsBuilder.getModulesToCompile(buildContext) - MavenArtifactsProperties mavenArtifacts = buildContext.productProperties.mavenArtifacts + private DistributionJARsBuilder compileModulesForDistribution(BuildContext context) { + Set pluginsToPublish = DistributionJARsBuilder + .getPluginsByModules(context, context.productProperties.productLayout.pluginModulesToPublish) + compileModulesForDistribution(pluginsToPublish, context) + } + + private DistributionJARsBuilder compileModulesForDistribution(Set pluginsToPublish, BuildContext context) { + ProductProperties productProperties = context.productProperties + ProductModulesLayout productLayout = productProperties.productLayout + Collection moduleNames = DistributionJARsBuilder.getModulesToCompile(context) + MavenArtifactsProperties mavenArtifacts = productProperties.mavenArtifacts Set toCompile = new LinkedHashSet<>() toCompile.addAll(moduleNames) - toCompile.addAll(buildContext.proprietaryBuildTools.scrambleTool?.additionalModulesToCompile ?: Collections.emptyList()) + toCompile.addAll(context.proprietaryBuildTools.scrambleTool?.additionalModulesToCompile ?: Collections.emptyList()) toCompile.addAll(productLayout.mainModules) toCompile.addAll(mavenArtifacts.additionalModules) toCompile.addAll(mavenArtifacts.proprietaryModules) - toCompile.addAll(buildContext.productProperties.modulesToCompileTests) + toCompile.addAll(productProperties.modulesToCompileTests) compileModules(toCompile) - def pluginsToPublish = new LinkedHashSet<>( - DistributionJARsBuilder.getPluginsByModules(buildContext, buildContext.productProperties.productLayout.pluginModulesToPublish)) - - if (buildContext.shouldBuildDistributions()) { - Path providedModulesFile = buildContext.paths.artifactDir.resolve("${buildContext.applicationInfo.productCode}-builtinModules.json") - buildProvidedModuleList(buildContext, providedModulesFile, moduleNames) - if (buildContext.productProperties.productLayout.buildAllCompatiblePlugins) { - if (!buildContext.options.buildStepsToSkip.contains(BuildOptions.PROVIDED_MODULES_LIST_STEP)) { - PluginsCollector collector = new PluginsCollector(buildContext) - pluginsToPublish.addAll(collector.collectCompatiblePluginsToPublish(providedModulesFile.toString())) + if (context.shouldBuildDistributions()) { + Path providedModulesFile = context.paths.artifactDir.resolve("${context.applicationInfo.productCode}-builtinModules.json") + buildProvidedModuleList(context, providedModulesFile, moduleNames) + if (productProperties.productLayout.buildAllCompatiblePlugins) { + if (context.options.buildStepsToSkip.contains(BuildOptions.PROVIDED_MODULES_LIST_STEP)) { + context.messages.info("Skipping collecting compatible plugins because PROVIDED_MODULES_LIST_STEP was skipped") } else { - buildContext.messages.info("Skipping collecting compatible plugins because PROVIDED_MODULES_LIST_STEP was skipped") + pluginsToPublish = new LinkedHashSet<>(pluginsToPublish) + pluginsToPublish.addAll(PluginsCollector.collectCompatiblePluginsToPublish(providedModulesFile, context)) } } } @@ -414,7 +420,11 @@ idea.fatal.error.notification=disabled setupBundledMaven() logFreeDiskSpace("before compilation") - DistributionJARsBuilder distributionJARsBuilder = compileModulesForDistribution() + + Set pluginsToPublish = DistributionJARsBuilder + .getPluginsByModules(context, context.productProperties.productLayout.pluginModulesToPublish) + + DistributionJARsBuilder distributionJARsBuilder = compileModulesForDistribution(buildContext) logFreeDiskSpace("after compilation") ForkJoinTask setupJetBrainsRuntimeTask = context.shouldBuildDistributions() ? createSetupJbreTask()?.fork() : null @@ -453,7 +463,7 @@ idea.fatal.error.notification=disabled Span.current().addEvent("skip building product distributions because " + "\"intellij.build.target.os\" property is set to \"$BuildOptions.OS_NONE\"") DistributionJARsBuilder.buildSearchableOptions(context, distributionJARsBuilder.getModulesForPluginsToPublish()) - distributionJARsBuilder.createBuildNonBundledPluginsTask(true, null, context)?.fork()?.join() + distributionJARsBuilder.createBuildNonBundledPluginsTask(pluginsToPublish, true, null, context)?.fork()?.join() } return null } @@ -550,7 +560,7 @@ idea.fatal.error.notification=disabled Set pluginsToPublish = DistributionJARsBuilder.getPluginsByModules(buildContext, mainPluginModules) DistributionJARsBuilder distributionJARsBuilder = compilePlatformAndPluginModules(pluginsToPublish) DistributionJARsBuilder.buildSearchableOptions(buildContext, distributionJARsBuilder.getModulesForPluginsToPublish()) - distributionJARsBuilder.createBuildNonBundledPluginsTask(true, null, buildContext)?.fork()?.join() + distributionJARsBuilder.createBuildNonBundledPluginsTask(pluginsToPublish, true, null, buildContext)?.fork()?.join() } @Override @@ -964,7 +974,7 @@ idea.fatal.error.notification=disabled void runTestBuild() { checkProductProperties() setupBundledMaven() - DistributionJARsBuilder distributionJARsBuilder = compileModulesForDistribution() + DistributionJARsBuilder distributionJARsBuilder = compileModulesForDistribution(buildContext) distributionJARsBuilder.buildJARs(buildContext) layoutShared(buildContext) Map checkerConfig = buildContext.productProperties.versionCheckerConfig @@ -987,7 +997,7 @@ idea.fatal.error.notification=disabled buildContext.options.buildStepsToSkip.add(BuildOptions.GENERATE_JAR_ORDER_STEP) setupBundledMaven() - compileModulesForDistribution().buildJARs(buildContext, true) + compileModulesForDistribution(buildContext).buildJARs(buildContext, true) JvmArchitecture arch = CpuArch.isArm64() ? JvmArchitecture.aarch64 : JvmArchitecture.x64 if (includeBinAndRuntime) { createSetupJbreTask(arch.name()).fork().join() diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ClassVersionChecker.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ClassVersionChecker.groovy index a2aac623c492..6d401c46ffc8 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ClassVersionChecker.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ClassVersionChecker.groovy @@ -10,6 +10,7 @@ import org.apache.commons.compress.archivers.zip.ZipFile import org.apache.commons.compress.utils.SeekableInMemoryByteChannel import org.jetbrains.annotations.NotNull import org.jetbrains.intellij.build.BuildContext +import org.jetbrains.intellij.build.BuildOptions import java.nio.file.DirectoryStream import java.nio.file.Files @@ -17,9 +18,7 @@ import java.nio.file.Path import java.nio.file.StandardOpenOption import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.ForkJoinTask -import java.util.function.Supplier import java.util.zip.ZipException - /** *

* Recursively checks .class files in directories and .jar/.zip files to ensure that their versions @@ -62,11 +61,15 @@ final class ClassVersionChecker { } void checkVersions(BuildContext context, Path root) { - context.messages.block(TracerManager.spanBuilder("verify class file versions") - .setAttribute("ruleCount", myRules.size()) - .setAttribute("root", root.toString()), new Supplier() { + if (context.options.buildStepsToSkip.contains(BuildOptions.VERIFY_CLASS_FILE_VERSIONS)) { + return + } + + BuildHelper.getInstance(context).span(TracerManager.spanBuilder("verify class file versions") + .setAttribute("ruleCount", myRules.size()) + .setAttribute("root", root.toString()), new Runnable() { @Override - Void get() { + void run() { checkedJarCount = 0 checkedClassCount = 0 errors = new ConcurrentLinkedQueue<>() @@ -96,9 +99,8 @@ final class ClassVersionChecker { List unusedRules = myRules - myUsedRules if (!unusedRules.isEmpty()) { context.messages.error("Class version check rules for the following paths don't match any files, probably entries in " + - "ProductProperties::versionCheckerConfig are out of date:\n${String.join("\n", unusedRules.collect { it.path })}") + "ProductProperties::versionCheckerConfig are out of date:\n${String.join("\n", unusedRules.collect { it.path })}") } - return null } }) } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy index b7ca258ebda6..4bad070d8389 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy @@ -1,4 +1,4 @@ -// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl import com.intellij.openapi.util.Pair @@ -14,6 +14,7 @@ import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span import it.unimi.dsi.fastutil.Hash import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenCustomHashSet +import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet import kotlin.Triple import org.apache.tools.ant.types.FileSet import org.apache.tools.ant.types.resources.FileProvider @@ -76,106 +77,151 @@ final class DistributionJARsBuilder { String releaseVersion = "${context.applicationInfo.majorVersion}${context.applicationInfo.minorVersionMainPart}00" pluginXmlPatcher = new PluginXmlPatcher(releaseDate, releaseVersion) - platform = createPlatformLayout(pluginsToPublish, context) + platform = createPlatformLayout(this.pluginsToPublish, context) } static PlatformLayout createPlatformLayout(Set pluginsToPublish, BuildContext context) { ProductModulesLayout productLayout = context.productProperties.productLayout Set enabledPluginModules = getEnabledPluginModules(pluginsToPublish, context.productProperties) - Set projectLibrariesUsedByPlugins = computeProjectLibsUsedByPlugins(context, enabledPluginModules) + Set projectLibrariesUsedByPlugins = computeProjectLibsUsedByPlugins(context, enabledPluginModules) return PlatformModules.createPlatformLayout(productLayout, hasPlatformCoverage(productLayout, enabledPluginModules, context), projectLibrariesUsedByPlugins, context) } - private static boolean hasPlatformCoverage(ProductModulesLayout productLayout, - Set enabledPluginModules, - BuildContext context) { - return Stream.concat(productLayout.getIncludedPluginModules(enabledPluginModules).stream(), - productLayout.getIncludedPluginModules(enabledPluginModules).stream()) - .flatMap(new Function>() { - @Override - Stream apply(String moduleName) { - return JpsJavaExtensionService.dependencies(context.findRequiredModule(moduleName)) - .productionOnly() - .getModules() - .stream() - .map(new Function() { - @Override - String apply(JpsModule module) { - return module.name - } - }) - } - }) - .filter(Predicate.isEqual("intellij.platform.coverage")) - .findAny() - .isPresent() + private static boolean hasPlatformCoverage(ProductModulesLayout productLayout, Set enabledPluginModules, BuildContext context) { + Set modules = new LinkedHashSet<>() + modules.addAll(productLayout.getIncludedPluginModules(enabledPluginModules)) + modules.addAll(PlatformModules.PLATFORM_API_MODULES) + modules.addAll(PlatformModules.PLATFORM_IMPLEMENTATION_MODULES) + modules.addAll(productLayout.productApiModules) + modules.addAll(productLayout.productImplementationModules) + modules.addAll(productLayout.additionalPlatformJars.values()) + + String coverageModuleName = "intellij.platform.coverage" + if (modules.contains(coverageModuleName)) { + return true + } + + for (String moduleName : modules) { + boolean contains = false + JpsJavaExtensionService.dependencies(context.findRequiredModule(moduleName)) + .productionOnly() + .processModules(new com.intellij.util.Consumer() { + @Override + void consume(JpsModule module) { + if (!contains && module.name == coverageModuleName) { + contains = true + } + } + }) + + if (contains) { + return true + } + } + + return false } - private static Set computeProjectLibsUsedByPlugins(BuildContext context, Set enabledPluginModules) { - context.messages.debug("Collecting project libraries used by plugins:") + private static Set computeProjectLibsUsedByPlugins(BuildContext context, Set enabledPluginModules) { + ObjectLinkedOpenHashSet result = new ObjectLinkedOpenHashSet<>() - Collection result = new LinkedHashSet<>() - MultiMap libraries = MultiMap.createLinked() for (PluginLayout plugin : getPluginsByModules(context, enabledPluginModules)) { - libraries.clear() - collectProjectLibrariesWhichShouldBeProvidedByPlatform(plugin, libraries, context) - for (Map.Entry> entry in libraries.entrySet()) { - context.messages.debug("plugin '$plugin.mainModule', library '$entry.key.name': " + - "used in ${String.join(", ", entry.value.collect { "'$it.name'" })}") + Collection libsToUnpack = plugin.projectLibrariesToUnpack.values() + for (String moduleName in plugin.includedModuleNames) { + JpsJavaDependenciesEnumerator dependencies = JpsJavaExtensionService.dependencies(context.findRequiredModule(moduleName)) + dependencies.includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).processLibraries(new com.intellij.util.Consumer() { + @Override + void consume(JpsLibrary library) { + if (!isProjectLibraryUsedByPlugin(library, plugin, libsToUnpack)) { + return + } + + String name = library.name + ProjectLibraryData.PackMode packMode = PlatformModules.CUSTOM_PACK_MODE.getOrDefault(name, ProjectLibraryData.PackMode.MERGED) + result.addOrGet(new ProjectLibraryData(name, "", packMode)) + .dependentModules.computeIfAbsent(Objects.requireNonNull(plugin.directoryName), PlatformModules.LIST_PRODUCER).add(moduleName) + } + }) } - result.addAll(libraries.keySet()) } return result } - static MultiMap collectProjectLibrariesWhichShouldBeProvidedByPlatform(BaseLayout plugin, - MultiMap result, - BuildContext buildContext) { + private static boolean isProjectLibraryUsedByPlugin(JpsLibrary library, BaseLayout plugin, Collection libsToUnpack) { + return !(library.createReference().parentReference instanceof JpsModuleReference) && + !plugin.includedProjectLibraries.any {it.libraryName == library.name} && + !libsToUnpack.contains(library.name) + } + + static void collectProjectLibrariesWhichShouldBeProvidedByPlatform(BaseLayout plugin, + MultiMap result, + BuildContext context) { Collection libsToUnpack = plugin.projectLibrariesToUnpack.values() for (String moduleName in plugin.includedModuleNames) { - JpsModule module = buildContext.findRequiredModule(moduleName) + JpsModule module = context.findRequiredModule(moduleName) JpsJavaDependenciesEnumerator dependencies = JpsJavaExtensionService.dependencies(module) for (JpsLibrary library : dependencies.includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).libraries) { - if (!(library.createReference().parentReference instanceof JpsModuleReference) && !plugin.includedProjectLibraries.any { - it.libraryName == library.name - } && !libsToUnpack.contains(library.name)) { + if (isProjectLibraryUsedByPlugin(library, plugin, libsToUnpack)) { result.putValue(library, module) } } } - return result } private static @NotNull Set filterPluginsToPublish(@NotNull Set plugins, @NotNull BuildContext context) { - plugins = plugins.stream().filter { + if (plugins.isEmpty()) { + return plugins + } + + Set result = plugins + for (PluginLayout plugin : plugins) { // Kotlin Multiplatform Mobile plugin is excluded since: // * is compatible with Android Studio only; // * has release cycle of its // * shadows IntelliJ utility modules included via Kotlin Compiler; // * breaks searchable options index and jar order generation steps. - it.mainModule != 'kotlin-ultimate.kmm-plugin' - }.collect(Collectors.toSet()) - if (plugins.isEmpty()) { - return plugins + if (plugin.mainModule == "kotlin-ultimate.kmm-plugin") { + if (result.is(plugins)) { + result = new LinkedHashSet<>(plugins) + } + result.remove(plugin) + } + } + if (result.isEmpty()) { + return Collections.emptySet() } Set toInclude = new HashSet<>(context.options.nonBundledPluginDirectoriesToInclude) if (toInclude.isEmpty()) { - return plugins + return result } + if (toInclude.size() == 1 && toInclude.contains("none")) { - return new LinkedHashSet() + return Collections.emptySet() } - return plugins.findAll { toInclude.contains(it.directoryName) } + + Iterator iterator = result.iterator() + while (iterator.hasNext()) { + PluginLayout plugin = iterator.next() + if (!toInclude.contains(plugin.directoryName)) { + if (result.is(plugins)) { + result = new LinkedHashSet<>(plugins) + } + result.remove(plugin) + } + } + return result } private static Set getEnabledPluginModules(Set pluginsToPublish, ProductProperties productProperties) { Set result = new LinkedHashSet<>() result.addAll(productProperties.productLayout.bundledPluginModules) - pluginsToPublish.collect(result) { it.mainModule } + for (PluginLayout plugin : pluginsToPublish) { + result.add(plugin.mainModule) + } return result } @@ -183,11 +229,6 @@ final class DistributionJARsBuilder { return (platform.includedModuleNames as List) + toolModules } - static List getIncludedPlatformModules(ProductModulesLayout modulesLayout) { - return PlatformModules.PLATFORM_API_MODULES + PlatformModules.PLATFORM_IMPLEMENTATION_MODULES + modulesLayout.productApiModules + - modulesLayout.productImplementationModules + modulesLayout.additionalPlatformJars.values() - } - /** * @return module names which are required to run necessary tools from build scripts */ @@ -212,9 +253,7 @@ final class DistributionJARsBuilder { ForkJoinTask brokenPluginsTask = createBuildBrokenPluginListTask(context)?.fork() BuildHelper buildHelper = BuildHelper.getInstance(context) - buildHelper.createSkippableTask(spanBuilder("build searchable options index"), - BuildOptions.SEARCHABLE_OPTIONS_INDEX_STEP, - context) { + buildHelper.createSkippableTask(spanBuilder("build searchable options index"), BuildOptions.SEARCHABLE_OPTIONS_INDEX_STEP, context) { buildSearchableOptions(context, getModulesForPluginsToPublish()) }?.fork()?.join() @@ -262,7 +301,10 @@ final class DistributionJARsBuilder { buildPlatformTask, createBuildBundledPluginTask(pluginLayouts, buildPlatformTask, context), createBuildOsSpecificBundledPluginsTask(pluginLayouts, isUpdateFromSources, buildPlatformTask, context), - createBuildNonBundledPluginsTask(!isUpdateFromSources, buildPlatformTask, context), + createBuildNonBundledPluginsTask(pluginsToPublish, + !isUpdateFromSources && context.options.compressNonBundledPluginArchive, + buildPlatformTask, + context), ForkJoinTask.adapt(new Callable>() { @Override List call() throws Exception { @@ -386,7 +428,7 @@ final class DistributionJARsBuilder { BuildHelper helper = BuildHelper.getInstance(context) return helper.createSkippableTask( spanBuilder("build broken plugin list") - .setAttribute("buildString", buildString) + .setAttribute("buildNumber", buildString) .setAttribute("path", targetFile.toString()), BuildOptions.BROKEN_PLUGINS_LIST_STEP, context, @@ -637,7 +679,7 @@ final class DistributionJARsBuilder { } Span.current().setAttribute("satisfiableCount", pluginsToBundle.size()) return buildPlugins(new ModuleOutputPatcher(), pluginsToBundle, - context.paths.distAllDir.resolve(PLUGINS_DIRECTORY), context, buildPlatformTask, null, false) + context.paths.distAllDir.resolve(PLUGINS_DIRECTORY), context, buildPlatformTask, null) } } ) @@ -687,7 +729,7 @@ final class DistributionJARsBuilder { .setAttribute("outDir", outDir.toString()), new Supplier>() { @Override List get() { - return buildPlugins(new ModuleOutputPatcher(), osSpecificPlugins, outDir, context, buildPlatformTask, null, true) + return buildPlugins(new ModuleOutputPatcher(), osSpecificPlugins, outDir, context, buildPlatformTask, null) } }) }).collectMany { it.rawResult } @@ -744,86 +786,103 @@ final class DistributionJARsBuilder { // compressPluginArchive also means that blockmap for plugin archive will be built @Nullable - ForkJoinTask createBuildNonBundledPluginsTask(boolean compressPluginArchive, - @Nullable ForkJoinTask buildPlatformLibTask, - @NotNull BuildContext context) { + ForkJoinTask> createBuildNonBundledPluginsTask(@NotNull Set pluginsToPublish, + boolean compressPluginArchive, + @Nullable ForkJoinTask buildPlatformLibTask, + @NotNull BuildContext context) { if (pluginsToPublish.isEmpty()) { return null } - return BuildHelper.getInstance(context).createSkippableTask(spanBuilder("build non-bundled plugins") - .setAttribute("count", pluginsToPublish.size()), - BuildOptions.NON_BUNDLED_PLUGINS_STEP, - context, new Runnable() { - @Override - void run() { - Path nonBundledPluginsArtifacts = context.paths.artifactDir.resolve(context.applicationInfo.productCode + "-plugins") - Path autoUploadingDir = nonBundledPluginsArtifacts.resolve("auto-uploading") + return BuildHelper.getInstance(context).createTask( + spanBuilder("build non-bundled plugins").setAttribute("count", pluginsToPublish.size()), + new Supplier>() { + @Override + List get() { + if (context.options.buildStepsToSkip.contains(BuildOptions.NON_BUNDLED_PLUGINS_STEP)) { + Span.current().addEvent("skip") + return Collections.emptyList() + } - ForkJoinTask>> buildKeymapPluginsTask = buildKeymapPlugins(autoUploadingDir, context).fork() + Path nonBundledPluginsArtifacts = context.paths.artifactDir.resolve(context.applicationInfo.productCode + "-plugins") + Path autoUploadingDir = nonBundledPluginsArtifacts.resolve("auto-uploading") - ModuleOutputPatcher moduleOutputPatcher = new ModuleOutputPatcher() - Path stageDir = context.paths.tempDir.resolve("non-bundled-plugins-" + context.applicationInfo.productCode) + ForkJoinTask>> buildKeymapPluginsTask = buildKeymapPlugins(autoUploadingDir, context).fork() - List> dirToJar = new ArrayList<>() + ModuleOutputPatcher moduleOutputPatcher = new ModuleOutputPatcher() + Path stageDir = context.paths.tempDir.resolve("non-bundled-plugins-" + context.applicationInfo.productCode) - String defaultPluginVersion = context.buildNumber.endsWith(".SNAPSHOT") - ? context.buildNumber + ".${PluginXmlPatcher.pluginDateFormat.format(ZonedDateTime.now())}" - : context.buildNumber + List> dirToJar = new ArrayList<>() - List pluginsToIncludeInCustomRepository = new ArrayList() - Predicate autoPublishPluginChecker = loadPluginAutoPublishList(context) + String defaultPluginVersion = context.buildNumber.endsWith(".SNAPSHOT") + ? context.buildNumber + ".${PluginXmlPatcher.pluginDateFormat.format(ZonedDateTime.now())}" + : context.buildNumber - boolean prepareCustomPluginRepositoryForPublishedPlugins = context.productProperties.productLayout - .prepareCustomPluginRepositoryForPublishedPlugins - buildPlugins(moduleOutputPatcher, pluginsToPublish, stageDir, context, buildPlatformLibTask, new BiConsumer() { - @Override - void accept(PluginLayout plugin, Path pluginDir) { - Path targetDirectory = autoPublishPluginChecker.test(plugin) ? autoUploadingDir : nonBundledPluginsArtifacts - String pluginDirName = pluginDir.getFileName().toString() + List pluginsToIncludeInCustomRepository = new ArrayList() + Predicate autoPublishPluginChecker = loadPluginAutoPublishList(context) - Path moduleOutput = context.getModuleOutputDir(context.findRequiredModule(plugin.mainModule)) - Path pluginXmlPath = moduleOutput.resolve("META-INF/plugin.xml") + boolean prepareCustomPluginRepositoryForPublishedPlugins = context.productProperties.productLayout + .prepareCustomPluginRepositoryForPublishedPlugins + List mappings = buildPlugins( + moduleOutputPatcher, pluginsToPublish, stageDir, context, buildPlatformLibTask, + new BiConsumer() { + @Override + void accept(PluginLayout plugin, Path pluginDir) { + Path targetDirectory = + autoPublishPluginChecker.test(plugin) ? autoUploadingDir : + nonBundledPluginsArtifacts + String pluginDirName = pluginDir.getFileName().toString() - String pluginVersion = Files.exists(pluginXmlPath) - ? plugin.versionEvaluator.evaluate(pluginXmlPath, defaultPluginVersion, context) - : defaultPluginVersion + Path moduleOutput = + context.getModuleOutputDir(context.findRequiredModule(plugin.mainModule)) + Path pluginXmlPath = moduleOutput.resolve("META-INF/plugin.xml") - Path destFile = targetDirectory.resolve("$pluginDirName-${pluginVersion}.zip") + String pluginVersion = Files.exists(pluginXmlPath) + ? plugin.versionEvaluator.evaluate(pluginXmlPath, defaultPluginVersion, + context) + : defaultPluginVersion + + Path destFile = + targetDirectory.resolve("$pluginDirName-${pluginVersion}.zip") + if (prepareCustomPluginRepositoryForPublishedPlugins) { + byte[] pluginXml = + moduleOutputPatcher.getPatchedPluginXml(plugin.mainModule) + pluginsToIncludeInCustomRepository + .add(new PluginRepositorySpec(destFile, pluginXml)) + } + dirToJar.add(Map.entry(pluginDirName, destFile)) + } + } + ) + + BuildHelper buildHelper = BuildHelper.getInstance(context) + buildHelper.bulkZipWithPrefix(stageDir, dirToJar, compressPluginArchive) + + PluginLayout helpPlugin = BuiltInHelpPlugin.helpPlugin(context, defaultPluginVersion) + if (helpPlugin != null) { + PluginRepositorySpec spec = buildHelpPlugin(helpPlugin, stageDir, autoUploadingDir, moduleOutputPatcher, context) if (prepareCustomPluginRepositoryForPublishedPlugins) { - byte[] pluginXml = moduleOutputPatcher.getPatchedPluginXml(plugin.mainModule) - pluginsToIncludeInCustomRepository.add(new PluginRepositorySpec(destFile, pluginXml)) + pluginsToIncludeInCustomRepository.add(spec) } - dirToJar.add(Map.entry(pluginDirName, destFile)) } - }, true) - BuildHelper buildHelper = BuildHelper.getInstance(context) - buildHelper.bulkZipWithPrefix(stageDir, dirToJar, compressPluginArchive) - - PluginLayout helpPlugin = BuiltInHelpPlugin.helpPlugin(context, defaultPluginVersion) - if (helpPlugin != null) { - PluginRepositorySpec spec = buildHelpPlugin(helpPlugin, stageDir, autoUploadingDir, moduleOutputPatcher, context) if (prepareCustomPluginRepositoryForPublishedPlugins) { - pluginsToIncludeInCustomRepository.add(spec) + PluginRepositoryXmlGenerator.generate(pluginsToIncludeInCustomRepository, nonBundledPluginsArtifacts, context) + + List autoUploadingPlugins = pluginsToIncludeInCustomRepository + .findAll { it.pluginZip.startsWith(autoUploadingDir) } + PluginRepositoryXmlGenerator.generate(autoUploadingPlugins, autoUploadingDir, context) } - } - if (prepareCustomPluginRepositoryForPublishedPlugins) { - PluginRepositoryXmlGenerator.generate(pluginsToIncludeInCustomRepository, nonBundledPluginsArtifacts, context) - - List autoUploadingPlugins = pluginsToIncludeInCustomRepository - .findAll { it.pluginZip.startsWith(autoUploadingDir) } - PluginRepositoryXmlGenerator.generate(autoUploadingPlugins, autoUploadingDir, context) - } - - for (kotlin.Pair item in buildKeymapPluginsTask.join()) { - if (prepareCustomPluginRepositoryForPublishedPlugins) { - pluginsToIncludeInCustomRepository.add(new PluginRepositorySpec(item.first, item.second)) + for (kotlin.Pair item in buildKeymapPluginsTask.join()) { + if (prepareCustomPluginRepositoryForPublishedPlugins) { + pluginsToIncludeInCustomRepository.add(new PluginRepositorySpec(item.first, item.second)) + } } + return mappings } } - }) + ) } private static ForkJoinTask>> buildKeymapPlugins(Path targetDir, BuildContext context) { @@ -843,7 +902,7 @@ final class DistributionJARsBuilder { context.messages.block(spanBuilder("build help plugin").setAttribute("dir", directory), new Supplier() { @Override Void get() { - buildPlugins(moduleOutputPatcher, List.of(helpPlugin), pluginsToPublishDir, context, null, null, true) + buildPlugins(moduleOutputPatcher, List.of(helpPlugin), pluginsToPublishDir, context, null, null) BuildHelper.zipWithPrefix(context, destFile, List.of(pluginsToPublishDir.resolve(directory)), directory, true) return null } @@ -905,13 +964,13 @@ final class DistributionJARsBuilder { } if (customLayouts == null) { - if (!result.add(PluginLayout.simplePlugin(moduleName))) { + if (moduleName != "kotlin-ultimate.kmm-plugin" && !result.add(PluginLayout.simplePlugin(moduleName))) { throw new IllegalStateException("Plugin layout for module $moduleName is already added (duplicated module name?)") } } else { for (PluginLayout layout : customLayouts) { - if (!result.add(layout)) { + if (layout.mainModule != "kotlin-ultimate.kmm-plugin" && !result.add(layout)) { throw new IllegalStateException("Plugin layout for module $moduleName is already added (duplicated module name?)") } } @@ -929,8 +988,7 @@ final class DistributionJARsBuilder { Path targetDirectory, BuildContext context, @Nullable ForkJoinTask buildPlatformTask, - @Nullable BiConsumer pluginBuilt, - boolean isParallel) { + @Nullable BiConsumer pluginBuilt) { ScrambleTool scrambleTool = context.proprietaryBuildTools.scrambleTool boolean isScramblingSkipped = context.options.buildStepsToSkip.contains(BuildOptions.SCRAMBLING_STEP) @@ -951,48 +1009,43 @@ final class DistributionJARsBuilder { String directoryName = getActualPluginDirectoryName(plugin, context) Path pluginDir = targetDirectory.resolve(directoryName) - Supplier> task = new Supplier>() { - @Override - List get() throws Exception { - List result = layout(plugin, - pluginDir, - true, - moduleOutputPatcher, - plugin.moduleJars, - context) - if (!plugin.pathsToScramble.isEmpty()) { - Attributes attributes = Attributes.of(AttributeKey.stringKey("plugin"), directoryName) - if (scrambleTool == null) { - Span.current().addEvent("skip scrambling plugin because scrambleTool isn't defined, but plugin defines paths to be scrambled", - attributes) - } - else if (isScramblingSkipped) { - Span.current().addEvent("skip scrambling plugin because step is disabled", attributes) - } - else { - ForkJoinTask scrambleTask = scrambleTool.scramblePlugin(context, plugin, pluginDir, targetDirectory) - if (scrambleTask != null) { - // we can not start executing right now because the plugin can use other plugins in a scramble classpath - scrambleTasks.add(scrambleTask) + tasks.add(buildHelper.createTask( + spanBuilder("plugin").setAttribute("path", context.paths.buildOutputDir.relativize(pluginDir).toString()), + new Supplier>() { + @Override + List get() throws Exception { + List result = layout(plugin, + pluginDir, + true, + moduleOutputPatcher, + plugin.moduleJars, + context) + if (!plugin.pathsToScramble.isEmpty()) { + Attributes attributes = Attributes.of(AttributeKey.stringKey("plugin"), directoryName) + if (scrambleTool == null) { + Span.current().addEvent( + "skip scrambling plugin because scrambleTool isn't defined, but plugin defines paths to be scrambled", + attributes) + } + else if (isScramblingSkipped) { + Span.current().addEvent("skip scrambling plugin because step is disabled", attributes) + } + else { + ForkJoinTask scrambleTask = scrambleTool.scramblePlugin(context, plugin, pluginDir, targetDirectory) + if (scrambleTask != null) { + // we can not start executing right now because the plugin can use other plugins in a scramble classpath + scrambleTasks.add(scrambleTask) + } } } - } - if (pluginBuilt != null) { - pluginBuilt.accept(plugin, pluginDir) + if (pluginBuilt != null) { + pluginBuilt.accept(plugin, pluginDir) + } + return result } - return result - } - } - - if (isParallel) { - tasks.add(buildHelper.createTask(spanBuilder("plugin") - .setAttribute("path", context.paths.buildOutputDir.relativize(targetDirectory).toString()), - task)) - } - else { - task.get() - } + }) + ) } List entries = new ArrayList<>(tasks.size() * 2) diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/JarPackager.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/JarPackager.groovy index 21dde2b258a5..9b966efeb08c 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/JarPackager.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/JarPackager.groovy @@ -55,7 +55,9 @@ final class JarPackager { pack(actualModuleJars, outputDir, new BaseLayout() {}, new ModuleOutputPatcher(), false, context) } - private static final Map> EXTRA_MERGE_RULES = new LinkedHashMap<>(); + private static final Map> EXTRA_MERGE_RULES = new LinkedHashMap<>() + + private final Map libToMetadata = new HashMap<>() static { // kotlinx- and kotlin-reflect libs to one kotlinx.jar @@ -308,12 +310,13 @@ final class JarPackager { if (library == null) { context.messages.error("Project library '$libraryName' from $jarPath should be unpacked but it isn't found") } + for (File ioFile : library.getFiles(JpsOrderRootType.COMPILED)) { Path file = ioFile.toPath() sourceList.add(createZipSource.apply(file, new IntConsumer() { @Override void accept(int size) { - projectStructureMapping.add(new ProjectLibraryEntry(jarFile, library.name, file, size)) + projectStructureMapping.add(new ProjectLibraryEntry(jarFile, library.name, file, null, "explicitUnpack", size)) } })) } @@ -361,6 +364,8 @@ final class JarPackager { throw new IllegalArgumentException("Cannot find library ${libraryData.libraryName} in the project") } + libToMetadata.put(library, libraryData) + Path libOutputDir = outputDir String relativePath = libraryData.relativeOutputPath if (relativePath != null && !relativePath.isEmpty()) { @@ -399,7 +404,8 @@ final class JarPackager { } for (Map.Entry> entry : jarToModuleNames.entrySet()) { - if (entry.key.contains("/")) { + String targetFilename = entry.key + if (targetFilename.contains("/")) { continue } @@ -414,68 +420,8 @@ final class JarPackager { continue } - JpsLibraryDependency libraryDependency = (JpsLibraryDependency)element - JpsCompositeElement parent = libraryDependency.libraryReference?.parentReference?.resolve() - if (!(parent instanceof JpsModule)) { - continue - } - - if (!(JpsJavaExtensionService.instance.getDependencyExtension(element)?.scope - ?.isIncludedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME) ?: false)) { - continue - } - - JpsLibrary library = libraryDependency.library - String libraryName = LayoutBuilder.getLibraryName(library) - if (!excluded.contains(libraryName) && - !layout.includedModuleLibraries.any { it.libraryName == libraryName }) { - String libName = library.name - List files = getLibraryFiles(library, copiedFiles, true) - - if (libName == "async-profiler-windows") { - // custom name, removeVersionFromJar doesn't support strings like `2.1-ea-4` - addLibrary(library, outputDir.resolve("async-profiler-windows.jar"), files) - continue - } - - boolean isJpsModule = moduleName.endsWith(".jps") - for (int i = files.size() - 1; i >= 0; i--) { - Path file = files.get(i) - String fileName = file.fileName.toString() - if (isJpsModule) { - files.remove(i) - addLibrary(library, outputDir.resolve(fileName), List.of(file)) - } - else { - //noinspection SpellCheckingInspection - if (fileName.endsWith("-rt.jar") || fileName.startsWith("jps-") || fileName.contains("-agent") || - fileName == "yjp-controller-api-redist.jar") { - files.remove(i) - addLibrary(library, outputDir.resolve(removeVersionFromJar(fileName)), List.of(file)) - } - } - } - if (!files.isEmpty()) { - BiFunction createZipSource = buildHelper.createZipSource - Path targetFile = outputDir.resolve(entry.key) - - List sources = extraLibSources.computeIfAbsent(entry.key, new Function() { - @Override - List apply(String s) { - return new ArrayList() - } - }) - - for (Path file : files) { - sources.add(createZipSource.apply(file, new IntConsumer() { - @Override - void accept(int size) { - projectStructureMapping.add(new ModuleLibraryFileEntry(targetFile, moduleName, file, size)) - } - })) - } - } - } + packModuleLibs(moduleName, targetFilename, (JpsLibraryDependency)element, excluded, layout, outputDir, copiedFiles, + extraLibSources) } } } @@ -483,6 +429,68 @@ final class JarPackager { return toMerge } + private void packModuleLibs(String moduleName, + String targetFilename, + JpsLibraryDependency libraryDependency, + Collection excluded, + BaseLayout layout, + Path outputDir, + Map copiedFiles, + Map extraLibSources) { + JpsCompositeElement parent = libraryDependency.libraryReference?.parentReference?.resolve() + if (!(parent instanceof JpsModule)) { + return + } + + if (!(JpsJavaExtensionService.instance.getDependencyExtension(libraryDependency)?.scope + ?.isIncludedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME) ?: false)) { + return + } + + JpsLibrary library = libraryDependency.library + String libraryName = LayoutBuilder.getLibraryName(library) + if (!excluded.contains(libraryName) && + !layout.includedModuleLibraries.any { it.libraryName == libraryName }) { + String libName = library.name + List files = getLibraryFiles(library, copiedFiles, true) + + if (libName == "async-profiler-windows") { + // custom name, removeVersionFromJar doesn't support strings like `2.1-ea-4` + addLibrary(library, outputDir.resolve("async-profiler-windows.jar"), files) + return + } + + for (int i = files.size() - 1; i >= 0; i--) { + Path file = files.get(i) + String fileName = file.fileName.toString() + //noinspection SpellCheckingInspection + if (fileName.endsWith("-rt.jar") || fileName.contains("-agent") || fileName == "yjp-controller-api-redist.jar") { + files.remove(i) + addLibrary(library, outputDir.resolve(removeVersionFromJar(fileName)), List.of(file)) + } + } + if (!files.isEmpty()) { + BiFunction createZipSource = buildHelper.createZipSource + List sources = extraLibSources.computeIfAbsent(targetFilename, new Function() { + @Override + List apply(String s) { + return new ArrayList() + } + }) + + Path targetFile = outputDir.resolve(targetFilename) + for (Path file : files) { + sources.add(createZipSource.apply(file, new IntConsumer() { + @Override + void accept(int size) { + projectStructureMapping.add(new ModuleLibraryFileEntry(targetFile, moduleName, file, size)) + } + })) + } + } + } + } + private static String removeVersionFromJar(String fileName) { Matcher matcher = fileName =~ LayoutBuilder.JAR_NAME_WITH_VERSION_PATTERN if (matcher.matches()) { @@ -532,12 +540,12 @@ final class JarPackager { } } - private static IntConsumer createLibSizeConsumer(Path file, - boolean isModuleLibrary, - Collection projectStructureMapping, - Path targetFile, - JpsLibrary library, - JpsModuleReference moduleReference) { + private IntConsumer createLibSizeConsumer(Path file, + boolean isModuleLibrary, + Collection projectStructureMapping, + Path targetFile, + JpsLibrary library, + JpsModuleReference moduleReference) { return new IntConsumer() { @Override void accept(int size) { @@ -545,7 +553,8 @@ final class JarPackager { projectStructureMapping.add(new ModuleLibraryFileEntry(targetFile, moduleReference.moduleName, file, size)) } else { - projectStructureMapping.add(new ProjectLibraryEntry(targetFile, library.name, file, size)) + ProjectLibraryData libMetadata = libToMetadata.get(library) + projectStructureMapping.add(new ProjectLibraryEntry(targetFile, library.name, file, libMetadata, null, size)) } } } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformLayout.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformLayout.groovy index cf780238b524..bfec63e10bf1 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformLayout.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformLayout.groovy @@ -39,6 +39,10 @@ final class PlatformLayout extends BaseLayout { includedProjectLibraries.add(new ProjectLibraryData(libraryName, "", packMode)) } + void withProjectLibrary(ProjectLibraryData data) { + includedProjectLibraries.add(data) + } + /** * Exclude project library {@code libraryName} even if it's added to dependencies of some module or plugin included into the product */ diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformModules.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformModules.groovy index cacd9fab75e8..1a2c37ce848f 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformModules.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PlatformModules.groovy @@ -7,23 +7,33 @@ import org.jetbrains.annotations.Nullable import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.ProductModulesLayout import org.jetbrains.jps.model.library.JpsLibrary +import org.jetbrains.jps.model.module.JpsModule import org.w3c.dom.Element import org.w3c.dom.NodeList import javax.xml.parsers.DocumentBuilderFactory import java.nio.file.Files import java.nio.file.Path +import java.util.function.BiConsumer +import java.util.function.Function import static org.jetbrains.intellij.build.impl.ProjectLibraryData.PackMode @CompileStatic final class PlatformModules { + public static final Function> LIST_PRODUCER = new Function>() { + @Override + List apply(String __) { + return new ArrayList() + } + } + public static final String PRODUCT_JAR = "product.jar" /** * List of modules which are included into lib/openapi.jar in all IntelliJ based IDEs. */ - static final List PLATFORM_API_MODULES = List.of( + public static final List PLATFORM_API_MODULES = List.of( "intellij.platform.analysis", "intellij.platform.builtInServer", "intellij.platform.core", @@ -112,6 +122,13 @@ final class PlatformModules { private static final String UTIL_JAR = "util.jar" + public static final Map CUSTOM_PACK_MODE = Map.of( + // jna uses native lib + "jna", PackMode.STANDALONE_MERGED, + "jetbrains-annotations-java5", PackMode.STANDALONE_SEPARATE_WITHOUT_VERSION_NAME, + "intellij-coverage", PackMode.STANDALONE_SEPARATE, + ) + static jar(String relativeJarPath, Collection moduleNames, ProductModulesLayout productLayout, @@ -137,13 +154,13 @@ final class PlatformModules { static PlatformLayout createPlatformLayout(ProductModulesLayout productLayout, boolean hasPlatformCoverage, - Set additionalProjectLevelLibraries, - BuildContext buildContext) { + Set additionalProjectLevelLibraries, + BuildContext context) { PlatformLayout layout = new PlatformLayout() // used only in modules that packed into Java layout.excludedProjectLibraries.add("jps-javac-extension") layout.excludedProjectLibraries.add("Eclipse") - productLayout.platformLayoutCustomizer.accept(layout, buildContext) + productLayout.platformLayoutCustomizer.accept(layout, context) Set alreadyPackedModules = new HashSet<>() for (Map.Entry> entry in productLayout.additionalPlatformJars.entrySet()) { @@ -168,7 +185,7 @@ final class PlatformModules { if (isRelocated) { layout.withModule(module, BaseLayout.APP_JAR) } - else if (!buildContext.productProperties.useProductJar || module.startsWith("intellij.platform.commercial")) { + else if (!context.productProperties.useProductJar || module.startsWith("intellij.platform.commercial")) { layout.withModule(module, productLayout.mainJarName) } else { @@ -256,9 +273,9 @@ final class PlatformModules { layout.withProjectLibraryUnpackedIntoJar(libraryName, productLayout.mainJarName) } - String productPluginSourceModuleName = buildContext.productProperties.applicationInfoModule + String productPluginSourceModuleName = context.productProperties.applicationInfoModule if (productPluginSourceModuleName != null) { - List modules = getProductPluginContentModules(buildContext, productPluginSourceModuleName) + List modules = getProductPluginContentModules(context, productPluginSourceModuleName) if (modules != null) { for (String name : modules) { layout.withModule(name, BaseLayout.APP_JAR) @@ -266,32 +283,27 @@ final class PlatformModules { } } - Map customPackMode = Map.of( - // jna uses native lib - "jna", PackMode.STANDALONE_MERGED, - "jetbrains-annotations-java5", PackMode.STANDALONE_SEPARATE_WITHOUT_VERSION_NAME, - "intellij-coverage", PackMode.STANDALONE_SEPARATE, - ) - layout.projectLibrariesToUnpack.putValues(UTIL_JAR, List.of( "JDOM", "Trove4j", )) - for (JpsLibrary library in additionalProjectLevelLibraries) { - String name = library.name + for (ProjectLibraryData item in additionalProjectLevelLibraries) { + String name = item.libraryName if (!productLayout.projectLibrariesToUnpackIntoMainJar.contains(name) && !layout.projectLibrariesToUnpack.values().contains(name) && !layout.excludedProjectLibraries.contains(name)) { - layout.withProjectLibrary(name, customPackMode.getOrDefault(name, PackMode.MERGED)) + layout.includedProjectLibraries.add(item) } } - layout.collectProjectLibrariesFromIncludedModules(buildContext) { lib, module -> - if (!additionalProjectLevelLibraries.contains(lib)) { + layout.collectProjectLibrariesFromIncludedModules(context, new BiConsumer() { + @Override + void accept(JpsLibrary lib, JpsModule module) { String name = lib.name - layout.withProjectLibrary(name, customPackMode.getOrDefault(name, PackMode.MERGED)) + layout.includedProjectLibraries.addOrGet(new ProjectLibraryData(name, "", CUSTOM_PACK_MODE.getOrDefault(name, PackMode.MERGED))) + .dependentModules.computeIfAbsent("core", LIST_PRODUCER).add(module.name) } - } + }) return layout } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ProjectLibraryData.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ProjectLibraryData.groovy index e3bc70100500..b79b9b510a6a 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ProjectLibraryData.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/ProjectLibraryData.groovy @@ -2,12 +2,15 @@ package org.jetbrains.intellij.build.impl import groovy.transform.CompileStatic +import org.jetbrains.annotations.NotNull @CompileStatic final class ProjectLibraryData { final String libraryName final String relativeOutputPath final PackMode packMode + // plugin to list of modules that uses the library + final Map> dependentModules = new TreeMap<>() enum PackMode { // merged into some uber jar @@ -20,7 +23,7 @@ final class ProjectLibraryData { STANDALONE_SEPARATE_WITHOUT_VERSION_NAME, } - ProjectLibraryData(String libraryName, String relativeOutputPath, PackMode packMode) { + ProjectLibraryData(@NotNull String libraryName, String relativeOutputPath, @NotNull PackMode packMode) { this.libraryName = libraryName this.relativeOutputPath = relativeOutputPath this.packMode = packMode diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectLibraryEntry.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectLibraryEntry.groovy index d8b0f14f24cd..9837e6519f3b 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectLibraryEntry.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectLibraryEntry.groovy @@ -2,6 +2,8 @@ package org.jetbrains.intellij.build.impl.projectStructureMapping import groovy.transform.CompileStatic +import org.jetbrains.annotations.Nullable +import org.jetbrains.intellij.build.impl.ProjectLibraryData import java.nio.file.Path @@ -13,12 +15,25 @@ final class ProjectLibraryEntry extends DistributionFileEntry implements Distrib final String libraryName final Path libraryFile final int size + final @Nullable ProjectLibraryData data + final @Nullable String reason ProjectLibraryEntry(Path path, String libraryName, Path libraryFile, int size) { + this(path, libraryName, libraryFile, null, null, size) + } + + ProjectLibraryEntry(Path path, + String libraryName, + Path libraryFile, + @Nullable ProjectLibraryData data, + @Nullable String reason, + int size) { super(path, "project-library") this.libraryName = libraryName this.libraryFile = libraryFile + this.data = data + this.reason = reason this.size = size } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.groovy index 743fb9651c9b..6cbbde601241 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.groovy @@ -3,10 +3,13 @@ package org.jetbrains.intellij.build.impl.projectStructureMapping import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.util.DefaultIndenter +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import groovy.transform.CompileStatic import org.jetbrains.annotations.NotNull import org.jetbrains.annotations.Nullable import org.jetbrains.intellij.build.BuildPaths +import org.jetbrains.intellij.build.impl.ProjectLibraryData import java.nio.file.Files import java.nio.file.Path @@ -59,7 +62,7 @@ final class ProjectStructureMapping { allEntries.sort({ DistributionFileEntry a , DistributionFileEntry b -> a.path.compareTo(b.path) }) Files.newOutputStream(file).withCloseable { out -> - JsonGenerator writer = new JsonFactory().createGenerator(out).useDefaultPrettyPrinter() + JsonGenerator writer = new JsonFactory().createGenerator(out).setPrettyPrinter(new IntelliJDefaultPrettyPrinter()) writer.writeStartArray() for (DistributionFileEntry entry : allEntries) { writer.writeStartObject() @@ -88,8 +91,23 @@ final class ProjectStructureMapping { } } + private static final class IntelliJDefaultPrettyPrinter extends DefaultPrettyPrinter { + private static final DefaultIndenter INDENTER = new DefaultIndenter(" ", "\n") + + IntelliJDefaultPrettyPrinter() { + _objectFieldValueSeparatorWithSpaces = ": " + _objectIndenter = INDENTER + _arrayIndenter = INDENTER + } + + @Override + DefaultPrettyPrinter createInstance() { + return new IntelliJDefaultPrettyPrinter() + } + } + static void buildJarContentReport(Collection entries, OutputStream out, BuildPaths buildPaths) { - JsonGenerator writer = new JsonFactory().createGenerator(out).useDefaultPrettyPrinter() + JsonGenerator writer = new JsonFactory().createGenerator(out).setPrettyPrinter(new IntelliJDefaultPrettyPrinter()) Map> fileToEntry = new TreeMap<>() for (DistributionFileEntry entry : entries) { fileToEntry.computeIfAbsent(entry.path, { new ArrayList() }).add(entry) @@ -125,7 +143,7 @@ final class ProjectStructureMapping { throw new IllegalStateException("Unsupported entry: $entry") } - writer.writeNumberField("value", fileSize) + writer.writeNumberField("size", fileSize) writer.writeEndObject() } writer.writeEndArray() @@ -154,7 +172,15 @@ final class ProjectStructureMapping { for (ProjectLibraryEntry fileEntry : entry.value) { writer.writeStartObject() writer.writeStringField("name", shortenPath(fileEntry.libraryFile, buildPaths, null)) - writer.writeNumberField("value", fileEntry.size as long) + writer.writeNumberField("size", fileEntry.size as long) + + ProjectLibraryData data = fileEntry.data + if (fileEntry.reason != null) { + writer.writeStringField("reason", fileEntry.reason) + } + if (data != null && !data.dependentModules.isEmpty()) { + writeModuleDependents(writer, data) + } writer.writeEndObject() } writer.writeEndArray() @@ -163,6 +189,18 @@ final class ProjectStructureMapping { } } + private static void writeModuleDependents(JsonGenerator writer, ProjectLibraryData data) { + writer.writeObjectFieldStart("dependentModules") + for (Map.Entry> pluginAndModules : data.dependentModules.entrySet()) { + writer.writeArrayFieldStart(pluginAndModules.key) + for (String moduleName : pluginAndModules.value) { + writer.writeString(moduleName) + } + writer.writeEndArray() + } + writer.writeEndObject() + } + private static String shortenPath(Path file, BuildPaths buildPaths, @Nullable Path extraRoot) { if (file.startsWith(MAVEN_REPO)) { return "\$MAVEN_REPOSITORY\$" + File.separatorChar + MAVEN_REPO.relativize(file).toString() diff --git a/platform/platform-tests/testSrc/com/intellij/internal/statistics/utils/FusInjectionWhiteListTest.kt b/platform/platform-tests/testSrc/com/intellij/internal/statistics/utils/FusInjectionWhiteListTest.kt index 45086f3dd62a..1cad0d0c7836 100644 --- a/platform/platform-tests/testSrc/com/intellij/internal/statistics/utils/FusInjectionWhiteListTest.kt +++ b/platform/platform-tests/testSrc/com/intellij/internal/statistics/utils/FusInjectionWhiteListTest.kt @@ -1,14 +1,23 @@ -// Copyright 2000-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.internal.statistics.utils +import com.intellij.ide.plugins.PluginManagerCore import com.intellij.internal.statistic.utils.PluginInfo import com.intellij.internal.statistic.utils.PluginType -import org.junit.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +private const val tbePluginId = "org.jetbrains.toolbox-enterprise-client" class FusInjectionWhiteListTest { + companion object { + @BeforeAll + @JvmStatic + fun setUp() { + PluginManagerCore.isUnitTestMode = true + } + } @Test fun allowedPluginsTBE() { @@ -17,10 +26,10 @@ class FusInjectionWhiteListTest { .partition { it.isAllowedToInjectIntoFUS() } allowed - .forEach { assertTrue(it.isDevelopedByJetBrains()) } + .forEach { assertThat(it.isDevelopedByJetBrains()).isTrue() } notAllowed - .forEach { assertFalse(it.isDevelopedByJetBrains()) } + .forEach { assertThat(it.isDevelopedByJetBrains()).isFalse() } } @Test @@ -29,11 +38,6 @@ class FusInjectionWhiteListTest { .map { PluginInfo(it, "some.plugin.id", "1.2.3") } .filter { it.isAllowedToInjectIntoFUS() } - assertEquals(1, allowed.size) - assertEquals(PluginType.PLATFORM, allowed.first().type) - } - - companion object { - private const val tbePluginId = "org.jetbrains.toolbox-enterprise-client" + assertThat(allowed.map { it.type }).containsExactlyInAnyOrder(PluginType.PLATFORM, PluginType.FROM_SOURCES) } } \ No newline at end of file diff --git a/platform/statistics/src/com/intellij/internal/statistic/utils/PluginInfoDetector.kt b/platform/statistics/src/com/intellij/internal/statistic/utils/PluginInfoDetector.kt index a49fb36ee5fb..4cf077c42fe9 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/utils/PluginInfoDetector.kt +++ b/platform/statistics/src/com/intellij/internal/statistic/utils/PluginInfoDetector.kt @@ -198,18 +198,18 @@ data class PluginInfo(val type: PluginType, val id: String?, val version: String fun isSafeToReport() = type.isSafeToReport() fun isAllowedToInjectIntoFUS(): Boolean { - return (isDevelopedByJetBrains() && id == tbePluginId) || + return (id == tbePluginId && type.isDevelopedByJetBrains()) || (PluginManagerCore.isUnitTestMode && (type == PluginType.PLATFORM || type == PluginType.FROM_SOURCES)) } } -val jvmCore: PluginInfo = PluginInfo(PluginType.JVM_CORE, null, null) +private val jvmCore: PluginInfo = PluginInfo(PluginType.JVM_CORE, null, null) val platformPlugin: PluginInfo = PluginInfo(PluginType.PLATFORM, null, null) val unknownPlugin: PluginInfo = PluginInfo(PluginType.UNKNOWN, null, null) private val notListedPlugin = PluginInfo(PluginType.NOT_LISTED, null, null) // Mock plugin info used when we can't detect plugin by class loader because IDE is built from sources -val builtFromSources: PluginInfo = PluginInfo(PluginType.FROM_SOURCES, null, null) +private val builtFromSources = PluginInfo(PluginType.FROM_SOURCES, null, null) private val pluginIdsFromOfficialJbPluginRepo: Supplier> = TimeoutCachedValue(1, TimeUnit.HOURS) { // before loading default repository plugins lets check it's not changed, and is really official JetBrains repository