diff --git a/build/groovy/org/jetbrains/intellij/build/BaseIdeaProperties.groovy b/build/groovy/org/jetbrains/intellij/build/BaseIdeaProperties.groovy index da70e3985946..274256d70c31 100644 --- a/build/groovy/org/jetbrains/intellij/build/BaseIdeaProperties.groovy +++ b/build/groovy/org/jetbrains/intellij/build/BaseIdeaProperties.groovy @@ -101,6 +101,9 @@ abstract class BaseIdeaProperties extends ProductProperties { additionalModulesToCompile = ["jps-standalone-builder"] modulesToCompileTests = ["jps-builders"] + productLayout.buildAllCompatiblePlugins = true + productLayout.compatiblePluginsToIgnore = ['python-plugin', 'ruby-plugin-bundle', 'python-community-plugin-resources', 'AWS'] + productLayout.prepareCustomPluginRepositoryForPublishedPlugins = true } @Override diff --git a/community-resources/src/idea/IdeaApplicationInfo.xml b/community-resources/src/idea/IdeaApplicationInfo.xml index f37ae2c10a0e..f84f535e59af 100644 --- a/community-resources/src/idea/IdeaApplicationInfo.xml +++ b/community-resources/src/idea/IdeaApplicationInfo.xml @@ -15,7 +15,7 @@ - + diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/ProductModulesLayout.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/ProductModulesLayout.groovy index 35643f70e7a8..95b88c270f78 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/ProductModulesLayout.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/ProductModulesLayout.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2000-2016 JetBrains s.r.o. + * Copyright 2000-2017 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -109,6 +109,17 @@ class ProductModulesLayout { */ boolean prepareCustomPluginRepositoryForPublishedPlugins = false + /** + * If {@code true} then all plugins that compatible with an IDE will be built. + * Otherwise only plugins from {@link #pluginModulesToPublish} will be considered. + */ + boolean buildAllCompatiblePlugins = false + + /** + * List of plugin names which should not be built even if they are compatible and {@link #buildAllCompatiblePlugins} is true + */ + List compatiblePluginsToIgnore = [] + /** * Names of the main modules of plugins from {@link #pluginModulesToPublish} list where since-build/until-build range should be restricted. * These plugins will be compatible with builds which number differ from the build which produces these plugins only in the last component, @@ -126,8 +137,7 @@ class ProductModulesLayout { /** * @return list of all modules which output is included into the plugin's JARs */ - List getIncludedPluginModules() { - Set enabledPluginModules = getEnabledPluginModules() + List getIncludedPluginModules(Set enabledPluginModules) { def modulesFromNonTrivialPlugins = allNonTrivialPlugins.findAll { enabledPluginModules.contains(it.mainModule) }. collectMany { it.getActualModules(enabledPluginModules).values() } (enabledPluginModules + modulesFromNonTrivialPlugins) as List @@ -136,8 +146,4 @@ class ProductModulesLayout { List getIncludedPlatformModules() { platformApiModules + platformImplementationModules + additionalPlatformJars.values() } - - Set getEnabledPluginModules() { - (bundledPluginModules + pluginModulesToPublish) as Set - } } \ No newline at end of file diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy index 41d4604a6d13..5a6da406827e 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy @@ -204,8 +204,14 @@ idea.fatal.error.notification=disabled def sourceFile = BuildContextImpl.findApplicationInfoInSources(buildContext.project, buildContext.productProperties, buildContext.messages) def targetFile = new File(buildContext.paths.temp, sourceFile.name) def date = ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("uuuuMMddHHmm")) + + def buildTypeId = System.getProperty('teamcity.buildType.id') + def buildId = System.getProperty('teamcity.build.id') + def builtinPluginsUrl = buildId && buildTypeId ? "http://buildserver.labs.intellij.net/repository/download/$buildTypeId/$buildId:id/${buildContext.productProperties.productCode}-plugins/plugins.xml?guest=1" : "" + BuildUtils.copyAndPatchFile(sourceFile.path, targetFile.path, - ["BUILD_NUMBER": buildContext.fullBuildNumber, "BUILD_DATE": date, "BUILD": buildContext.buildNumber]) + ["BUILD_NUMBER": buildContext.fullBuildNumber, "BUILD_DATE": date, "BUILD": buildContext.buildNumber, + "BUILTIN_PLUGINS_URL": builtinPluginsUrl]) return targetFile } @@ -267,10 +273,30 @@ idea.fatal.error.notification=disabled } private compileModulesForDistribution(DistributionJARsBuilder distributionJARsBuilder) { - def moduleNames = buildContext.productProperties.productLayout.includedPluginModules + distributionJARsBuilder.platformModules + + def bundledPlugins = buildContext.productProperties.productLayout.bundledPluginModules as Set + def moduleNames = buildContext.productProperties.productLayout.getIncludedPluginModules(bundledPlugins) + + distributionJARsBuilder.platformModules + buildContext.productProperties.additionalModulesToCompile + (buildContext.proprietaryBuildTools.scrambleTool?.additionalModulesToCompile ?: []) compileModules(moduleNames, buildContext.productProperties.modulesToCompileTests) + + def productLayout = buildContext.productProperties.productLayout + + def providedModulesFilePath = "${buildContext.paths.artifacts}/${buildContext.productProperties.productCode}-builtinModules.json" + buildProvidedModulesList(providedModulesFilePath, productLayout.mainModules, productLayout.licenseFilesToBuildSearchableOptions) + def pluginsToPublish = distributionJARsBuilder.getPluginsByModules(buildContext.productProperties.productLayout.pluginModulesToPublish) + if (buildContext.productProperties.productLayout.buildAllCompatiblePlugins) { + if (!buildContext.options.buildStepsToSkip.contains(BuildOptions.PROVIDED_MODULES_LIST_STEP)) { + pluginsToPublish = new PluginsCollector(buildContext, providedModulesFilePath).collectCompatiblePluginsToPublish().findAll { + !buildContext.productProperties.productLayout.compatiblePluginsToIgnore.contains(it.mainModule) + } + } + else { + buildContext.messages.info("Skipping collecting compatible plugins because PROVIDED_MODULES_LIST_STEP was skipped") + } + } + compileModules(pluginsToPublish.collect { it.moduleJars.values() }.flatten() as List) + distributionJARsBuilder.pluginsToPublish.addAll(pluginsToPublish) } @Override @@ -391,7 +417,16 @@ idea.fatal.error.notification=disabled checkPluginModules(layout.bundledPluginModules, "productProperties.productLayout.bundledPluginModules", optionalModules) checkPluginModules(layout.pluginModulesToPublish, "productProperties.productLayout.pluginModulesToPublish", optionalModules) - if (layout.prepareCustomPluginRepositoryForPublishedPlugins && layout.pluginModulesToPublish.isEmpty()) { + if (!layout.pluginModulesToPublish.isEmpty() && layout.buildAllCompatiblePlugins && buildContext.shouldBuildDistributions()) { + buildContext.messages.warning("layout.buildAllCompatiblePlugins option is enabled. Value of layout.pluginModulesToPublish property " + + "will be ignored ($layout.pluginModulesToPublish)") + } + if (!buildContext.shouldBuildDistributions() && layout.buildAllCompatiblePlugins) { + buildContext.messages.warning("Distribution is not going to build. Hence all compatible plugins won't be built despite " + + "layout.buildAllCompatiblePlugins option is enabled. layout.pluginModulesToPublish will be used ($layout.pluginModulesToPublish)") + } + if (layout.prepareCustomPluginRepositoryForPublishedPlugins && layout.pluginModulesToPublish.isEmpty() && + !layout.buildAllCompatiblePlugins) { buildContext.messages.error("productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins option is enabled but no pluginModulesToPublish are specified") } @@ -403,7 +438,7 @@ idea.fatal.error.notification=disabled checkModules([layout.searchableOptionsModule], "productProperties.productLayout.searchableOptionsModule") checkModules(layout.pluginModulesWithRestrictedCompatibleBuildRange, "productProperties.productLayout.pluginModulesWithRestrictedCompatibleBuildRange") checkProjectLibraries(layout.projectLibrariesToUnpackIntoMainJar, "productProperties.productLayout.projectLibrariesToUnpackIntoMainJar") - nonTrivialPlugins.findAll {layout.enabledPluginModules.contains(it.mainModule)}.each { plugin -> + nonTrivialPlugins.findAll {layout.bundledPluginModules.contains(it.mainModule)}.each { plugin -> checkModules(plugin.moduleJars.values() - plugin.optionalModules, "'$plugin.mainModule' plugin") checkModules(plugin.moduleExcludes.keySet(), "'$plugin.mainModule' plugin") checkProjectLibraries(plugin.includedProjectLibraries, "'$plugin.mainModule' plugin") diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy index f48363050c2b..88f5993d3491 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy @@ -47,6 +47,7 @@ class DistributionJARsBuilder { private final Set usedModules = new LinkedHashSet<>() private final PlatformLayout platform private final File patchedApplicationInfo + private final List pluginsToPublish = [] DistributionJARsBuilder(BuildContext buildContext, File patchedApplicationInfo) { this.patchedApplicationInfo = patchedApplicationInfo @@ -80,8 +81,9 @@ class DistributionJARsBuilder { def productLayout = buildContext.productProperties.productLayout - List projectLibrariesUsedByPlugins = getPluginsByModules(productLayout.enabledPluginModules).collectMany { plugin -> - plugin.getActualModules(productLayout.enabledPluginModules).values().collectMany { + def enabledPluginModules = getEnabledPluginModules() + List projectLibrariesUsedByPlugins = getPluginsByModules(enabledPluginModules).collectMany { plugin -> + plugin.getActualModules(enabledPluginModules).values().collectMany { def module = buildContext.findRequiredModule(it) JpsJavaExtensionService.dependencies(module).includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).libraries.findAll { !(it.createReference().parentReference instanceof JpsModuleReference) && !plugin.includedProjectLibraries.contains(it.name) @@ -89,7 +91,7 @@ class DistributionJARsBuilder { } } - Set allProductDependencies = (productLayout.includedPluginModules + productLayout.includedPlatformModules).collectMany(new LinkedHashSet()) { + Set allProductDependencies = (productLayout.getIncludedPluginModules(enabledPluginModules) + productLayout.includedPlatformModules).collectMany(new LinkedHashSet()) { JpsJavaExtensionService.dependencies(buildContext.findRequiredModule(it)).productionOnly().getModules().collect {it.name} } @@ -137,6 +139,10 @@ class DistributionJARsBuilder { } } + private Set getEnabledPluginModules() { + buildContext.productProperties.productLayout.bundledPluginModules + pluginsToPublish.collect { it.mainModule } as Set + } + List getPlatformModules() { (platform.moduleJars.values() as List) + ["java-runtime", "platform-main", /*required to build searchable options index*/ @@ -216,9 +222,6 @@ class DistributionJARsBuilder { layoutBuilder.patchModuleOutput(productLayout.searchableOptionsModule, FileUtil.toSystemIndependentName(searchableOptionsDir.absolutePath)) } - String providedModulesFile = "${buildContext.paths.artifacts}/${buildContext.productProperties.productCode}-builtinModules.json" - buildTasks.buildProvidedModulesList(providedModulesFile, productLayout.mainModules, productLayout.licenseFilesToBuildSearchableOptions) - def applicationInfoFile = FileUtil.toSystemIndependentName(patchedApplicationInfo.absolutePath) def applicationInfoDir = "$buildContext.paths.temp/applicationInfo" ant.copy(file: applicationInfoFile, todir: "$applicationInfoDir/idea") @@ -243,6 +246,10 @@ class DistributionJARsBuilder { usedModules.addAll(layoutBuilder.usedModules) } + List getPluginsToPublish() { + return pluginsToPublish + } + private void buildBundledPlugins() { def productLayout = buildContext.productProperties.productLayout def layoutBuilder = createLayoutBuilder() @@ -251,11 +258,10 @@ class DistributionJARsBuilder { } void buildNonBundledPlugins() { - def ant = buildContext.ant def productLayout = buildContext.productProperties.productLayout + def ant = buildContext.ant def layoutBuilder = createLayoutBuilder() buildContext.executeStep("Build non-bundled plugins", BuildOptions.NON_BUNDLED_PLUGINS_STEP) { - def pluginsToPublish = getPluginsByModules(productLayout.pluginModulesToPublish) if (buildContext.productProperties.setPluginAndIDEVersionInPluginXml) { pluginsToPublish.each { plugin -> def moduleOutput = buildContext.getModuleOutputPath(buildContext.findRequiredModule(plugin.mainModule)) @@ -273,18 +279,21 @@ class DistributionJARsBuilder { } } - def pluginsToPublishDir = "$buildContext.paths.temp/plugins-to-publish" + def pluginsToPublishDir = "$buildContext.paths.temp/${buildContext.productProperties.productCode}-plugins-to-publish" buildPlugins(layoutBuilder, pluginsToPublish, pluginsToPublishDir) - def nonBundledPluginsArtifacts = "$buildContext.paths.artifacts/plugins" + def nonBundledPluginsArtifacts = "$buildContext.paths.artifacts/${buildContext.productProperties.productCode}-plugins" pluginsToPublish.each { plugin -> def directory = plugin.directoryName String suffix = productLayout.prepareCustomPluginRepositoryForPublishedPlugins ? "" : "-${getPluginVersion(plugin)}" - ant.zip(destfile: "$nonBundledPluginsArtifacts/$directory${suffix}.zip") { + def destFile = "$nonBundledPluginsArtifacts/$directory${suffix}.zip" + ant.zip(destfile: destFile) { zipfileset(dir: "$pluginsToPublishDir/$directory", prefix: directory) } + buildContext.notifyArtifactBuilt(destFile) } if (productLayout.prepareCustomPluginRepositoryForPublishedPlugins) { new PluginRepositoryXmlGenerator(buildContext).generate(pluginsToPublish, nonBundledPluginsArtifacts) + buildContext.notifyArtifactBuilt("$nonBundledPluginsArtifacts/plugins.xml") } } } @@ -293,7 +302,7 @@ class DistributionJARsBuilder { return plugin.versionEvaluator.apply(buildContext) } - private List getPluginsByModules(Collection modules) { + List getPluginsByModules(Collection modules) { def allNonTrivialPlugins = buildContext.productProperties.productLayout.allNonTrivialPlugins def allOptionalModules = allNonTrivialPlugins.collectMany {it.optionalModules} def nonTrivialPlugins = allNonTrivialPlugins.groupBy { it.mainModule } @@ -301,7 +310,7 @@ class DistributionJARsBuilder { } private void buildPlugins(LayoutBuilder layoutBuilder, List pluginsToInclude, String targetDirectory) { - def enabledModulesSet = buildContext.productProperties.productLayout.enabledPluginModules + def enabledModulesSet = enabledPluginModules pluginsToInclude.each { plugin -> def actualModuleJars = plugin.getActualModules(enabledModulesSet) checkOutputOfPluginModules(plugin.mainModule, actualModuleJars.values(), plugin.moduleExcludes) @@ -478,7 +487,6 @@ class DistributionJARsBuilder { JpsPathUtil.urlToPath(buildContext.findRequiredModule(moduleName).contentRootsList.urls.first()) } - private LayoutBuilder createLayoutBuilder() { new LayoutBuilder(buildContext.ant, buildContext.project, COMPRESS_JARS) } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PluginsCollector.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PluginsCollector.groovy new file mode 100644 index 000000000000..fde9f0f89b14 --- /dev/null +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PluginsCollector.groovy @@ -0,0 +1,104 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jetbrains.intellij.build.impl + +import groovy.json.JsonSlurper +import org.jetbrains.annotations.NotNull +import org.jetbrains.intellij.build.BuildContext + +class PluginsCollector { + private final String myProvidedModulesFilePath + private final BuildContext myBuildContext + + PluginsCollector(@NotNull BuildContext buildContext, @NotNull String providedModulesFilePath) { + this.myBuildContext = buildContext + this.myProvidedModulesFilePath = providedModulesFilePath + } + + List collectCompatiblePluginsToPublish() { + def parse = new JsonSlurper().parse(new File(myProvidedModulesFilePath)) as Map + Set availableModulesAndPlugins = new HashSet(parse['modules'] as Collection) + availableModulesAndPlugins.addAll(parse['plugins'] as Collection) + + def descriptorsMap = collectPluginDescriptors() + def pluginDescriptors = new HashSet(descriptorsMap.values()) + return pluginDescriptors.findAll { isPluginCompatible(it, availableModulesAndPlugins, descriptorsMap) }.collect { it.pluginLayout } + } + + private boolean isPluginCompatible(@NotNull PluginDescriptor plugin, + @NotNull Set availableModulesAndPlugins, + @NotNull Map nonCheckedPlugins) { + nonCheckedPlugins.remove(plugin.id) + for (requiredDependency in plugin.requiredDependencies) { + if (availableModulesAndPlugins.contains(requiredDependency)) { + continue + } + def requiredPlugin = nonCheckedPlugins.get(requiredDependency) + if (requiredPlugin != null && isPluginCompatible(requiredPlugin, availableModulesAndPlugins, nonCheckedPlugins)) { + continue + } + return false + } + availableModulesAndPlugins.add(plugin.id) + return true + } + + private Map collectPluginDescriptors() { + def pluginDescriptors = new HashMap() + def productLayout = myBuildContext.productProperties.productLayout + def nonTrivialPlugins = productLayout.allNonTrivialPlugins.groupBy { it.mainModule } + myBuildContext.project.modules.each { + if (productLayout.bundledPluginModules.contains(it.name)) { + return + } + PluginLayout pluginLayout = nonTrivialPlugins[it.name]?.first() + if (pluginLayout == null) { + pluginLayout = PluginLayout.plugin(it.name) + } + def pluginXml = myBuildContext.findFileInModuleSources(it.name, "META-INF/plugin.xml") + if (pluginXml == null) { + return + } + + def xml = new XmlParser().parse(pluginXml) + String id = xml.id.text() ?: xml.name.text() + if (!id) { + return + } + def requiredDependencies = new HashSet() + for (dependency in xml.depends) { + if (dependency.@optional != 'true') { + requiredDependencies += dependency.text() + } + } + + pluginDescriptors[id] = new PluginDescriptor(id, requiredDependencies, pluginLayout) + } + return pluginDescriptors + } + + private class PluginDescriptor { + private final String id + private final Set requiredDependencies + private final PluginLayout pluginLayout + + PluginDescriptor(String id, Set requiredDependencies, PluginLayout pluginLayout) { + this.id = id + this.requiredDependencies = requiredDependencies + this.pluginLayout = pluginLayout + } + } +} diff --git a/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java b/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java index 6c2eec5be676..43defb8bd35f 100644 --- a/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java +++ b/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java @@ -844,7 +844,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx { myPluginsDownloadUrl = downloadUrl != null ? downloadUrl : myPluginManagerUrl + (closed ? "" : "/") + "pluginManager/"; if (!getBuild().isSnapshot()) { - myBuiltinPluginsUrl = pluginsElement.getAttributeValue(ATTRIBUTE_BUILTIN_URL); + myBuiltinPluginsUrl = StringUtil.nullize(pluginsElement.getAttributeValue(ATTRIBUTE_BUILTIN_URL)); } } else {