Build scripts: build all compatible plugins

2026-09-27 10:03:11 +07:00 · 2017-09-07 21:45:11 +03:00
parent f9649e7748
commit 2d17b50077
7 changed files with 183 additions and 27 deletions
@@ -101,6 +101,9 @@ abstract class BaseIdeaProperties extends ProductProperties {

    additionalModulesToCompile = ["jps-standalone-builder"]
    modulesToCompileTests = ["jps-builders"]
+    productLayout.buildAllCompatiblePlugins = true
+    productLayout.compatiblePluginsToIgnore = ['python-plugin', 'ruby-plugin-bundle', 'python-community-plugin-resources', 'AWS']
+    productLayout.prepareCustomPluginRepositoryForPublishedPlugins = true
  }

  @Override
@@ -15,7 +15,7 @@

  <editor background-url="/idea_logo_welcome.png"/>

-  <plugins url="http://plugins.jetbrains.com/"/>
+  <plugins url="http://plugins.jetbrains.com/" builtin-url="__BUILTIN_PLUGINS_URL__"/>
  <update-urls check="http://www.jetbrains.com/updates/updates.xml"
               patches="http://download.jetbrains.com/idea/"/>

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2016 JetBrains s.r.o.
+ * Copyright 2000-2017 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -109,6 +109,17 @@ class ProductModulesLayout {
   */
  boolean prepareCustomPluginRepositoryForPublishedPlugins = false

+  /**
+   * If {@code true} then all plugins that compatible with an IDE will be built.
+   * Otherwise only plugins from {@link #pluginModulesToPublish} will be considered.
+   */
+  boolean buildAllCompatiblePlugins = false
+
+  /**
+   * List of plugin names which should not be built even if they are compatible and {@link #buildAllCompatiblePlugins} is true
+   */
+  List<String> compatiblePluginsToIgnore = []
+
  /**
   * Names of the main modules of plugins from {@link #pluginModulesToPublish} list where since-build/until-build range should be restricted.
   * These plugins will be compatible with builds which number differ from the build which produces these plugins only in the last component,
@@ -126,8 +137,7 @@ class ProductModulesLayout {
  /**
   * @return list of all modules which output is included into the plugin's JARs
   */
-  List<String> getIncludedPluginModules() {
-    Set<String> enabledPluginModules = getEnabledPluginModules()
+  List<String> getIncludedPluginModules(Set<String> enabledPluginModules) {
    def modulesFromNonTrivialPlugins = allNonTrivialPlugins.findAll { enabledPluginModules.contains(it.mainModule) }.
      collectMany { it.getActualModules(enabledPluginModules).values() }
    (enabledPluginModules + modulesFromNonTrivialPlugins) as List<String>
@@ -136,8 +146,4 @@ class ProductModulesLayout {
  List<String> getIncludedPlatformModules() {
    platformApiModules + platformImplementationModules + additionalPlatformJars.values()
  }
-
-  Set<String> getEnabledPluginModules() {
-    (bundledPluginModules + pluginModulesToPublish) as Set<String>
-  }
 }
@@ -204,8 +204,14 @@ idea.fatal.error.notification=disabled
    def sourceFile = BuildContextImpl.findApplicationInfoInSources(buildContext.project, buildContext.productProperties, buildContext.messages)
    def targetFile = new File(buildContext.paths.temp, sourceFile.name)
    def date = ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("uuuuMMddHHmm"))
+
+    def buildTypeId = System.getProperty('teamcity.buildType.id')
+    def buildId = System.getProperty('teamcity.build.id')
+    def builtinPluginsUrl = buildId && buildTypeId ? "http://buildserver.labs.intellij.net/repository/download/$buildTypeId/$buildId:id/${buildContext.productProperties.productCode}-plugins/plugins.xml?guest=1" : ""
+
    BuildUtils.copyAndPatchFile(sourceFile.path, targetFile.path,
-                                ["BUILD_NUMBER": buildContext.fullBuildNumber, "BUILD_DATE": date, "BUILD": buildContext.buildNumber])
+                                ["BUILD_NUMBER": buildContext.fullBuildNumber, "BUILD_DATE": date, "BUILD": buildContext.buildNumber,
+                                "BUILTIN_PLUGINS_URL": builtinPluginsUrl])
    return targetFile
  }

@@ -267,10 +273,30 @@ idea.fatal.error.notification=disabled
  }

  private compileModulesForDistribution(DistributionJARsBuilder distributionJARsBuilder) {
-    def moduleNames = buildContext.productProperties.productLayout.includedPluginModules + distributionJARsBuilder.platformModules +
+    def bundledPlugins = buildContext.productProperties.productLayout.bundledPluginModules as Set<String>
+    def moduleNames = buildContext.productProperties.productLayout.getIncludedPluginModules(bundledPlugins) +
+                      distributionJARsBuilder.platformModules +
                      buildContext.productProperties.additionalModulesToCompile +
                      (buildContext.proprietaryBuildTools.scrambleTool?.additionalModulesToCompile ?: [])
    compileModules(moduleNames, buildContext.productProperties.modulesToCompileTests)
+
+    def productLayout = buildContext.productProperties.productLayout
+
+    def providedModulesFilePath = "${buildContext.paths.artifacts}/${buildContext.productProperties.productCode}-builtinModules.json"
+    buildProvidedModulesList(providedModulesFilePath, productLayout.mainModules, productLayout.licenseFilesToBuildSearchableOptions)
+    def pluginsToPublish = distributionJARsBuilder.getPluginsByModules(buildContext.productProperties.productLayout.pluginModulesToPublish)
+    if (buildContext.productProperties.productLayout.buildAllCompatiblePlugins) {
+      if (!buildContext.options.buildStepsToSkip.contains(BuildOptions.PROVIDED_MODULES_LIST_STEP)) {
+        pluginsToPublish = new PluginsCollector(buildContext, providedModulesFilePath).collectCompatiblePluginsToPublish().findAll {
+          !buildContext.productProperties.productLayout.compatiblePluginsToIgnore.contains(it.mainModule)
+        }
+      }
+      else {
+        buildContext.messages.info("Skipping collecting compatible plugins because PROVIDED_MODULES_LIST_STEP was skipped")
+      }
+    }
+    compileModules(pluginsToPublish.collect { it.moduleJars.values()  }.flatten() as List<String>)
+    distributionJARsBuilder.pluginsToPublish.addAll(pluginsToPublish)
  }

  @Override
@@ -391,7 +417,16 @@ idea.fatal.error.notification=disabled
    checkPluginModules(layout.bundledPluginModules, "productProperties.productLayout.bundledPluginModules", optionalModules)
    checkPluginModules(layout.pluginModulesToPublish, "productProperties.productLayout.pluginModulesToPublish", optionalModules)

-    if (layout.prepareCustomPluginRepositoryForPublishedPlugins && layout.pluginModulesToPublish.isEmpty()) {
+    if (!layout.pluginModulesToPublish.isEmpty() && layout.buildAllCompatiblePlugins && buildContext.shouldBuildDistributions()) {
+      buildContext.messages.warning("layout.buildAllCompatiblePlugins option is enabled. Value of layout.pluginModulesToPublish property " +
+                                    "will be ignored ($layout.pluginModulesToPublish)")
+    }
+    if (!buildContext.shouldBuildDistributions() && layout.buildAllCompatiblePlugins) {
+      buildContext.messages.warning("Distribution is not going to build. Hence all compatible plugins won't be built despite " +
+                                    "layout.buildAllCompatiblePlugins option is enabled. layout.pluginModulesToPublish will be used ($layout.pluginModulesToPublish)")
+    }
+    if (layout.prepareCustomPluginRepositoryForPublishedPlugins && layout.pluginModulesToPublish.isEmpty() &&
+        !layout.buildAllCompatiblePlugins) {
      buildContext.messages.error("productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins option is enabled but no pluginModulesToPublish are specified")
    }

@@ -403,7 +438,7 @@ idea.fatal.error.notification=disabled
    checkModules([layout.searchableOptionsModule], "productProperties.productLayout.searchableOptionsModule")
    checkModules(layout.pluginModulesWithRestrictedCompatibleBuildRange, "productProperties.productLayout.pluginModulesWithRestrictedCompatibleBuildRange")
    checkProjectLibraries(layout.projectLibrariesToUnpackIntoMainJar, "productProperties.productLayout.projectLibrariesToUnpackIntoMainJar")
-    nonTrivialPlugins.findAll {layout.enabledPluginModules.contains(it.mainModule)}.each { plugin ->
+    nonTrivialPlugins.findAll {layout.bundledPluginModules.contains(it.mainModule)}.each { plugin ->
      checkModules(plugin.moduleJars.values() - plugin.optionalModules, "'$plugin.mainModule' plugin")
      checkModules(plugin.moduleExcludes.keySet(), "'$plugin.mainModule' plugin")
      checkProjectLibraries(plugin.includedProjectLibraries, "'$plugin.mainModule' plugin")
@@ -47,6 +47,7 @@ class DistributionJARsBuilder {
  private final Set<String> usedModules = new LinkedHashSet<>()
  private final PlatformLayout platform
  private final File patchedApplicationInfo
+  private final List<PluginLayout> pluginsToPublish = []

  DistributionJARsBuilder(BuildContext buildContext, File patchedApplicationInfo) {
    this.patchedApplicationInfo = patchedApplicationInfo
@@ -80,8 +81,9 @@ class DistributionJARsBuilder {

    def productLayout = buildContext.productProperties.productLayout

-    List<JpsLibrary> projectLibrariesUsedByPlugins = getPluginsByModules(productLayout.enabledPluginModules).collectMany { plugin ->
-      plugin.getActualModules(productLayout.enabledPluginModules).values().collectMany {
+    def enabledPluginModules = getEnabledPluginModules()
+    List<JpsLibrary> projectLibrariesUsedByPlugins = getPluginsByModules(enabledPluginModules).collectMany { plugin ->
+      plugin.getActualModules(enabledPluginModules).values().collectMany {
        def module = buildContext.findRequiredModule(it)
        JpsJavaExtensionService.dependencies(module).includedIn(JpsJavaClasspathKind.PRODUCTION_RUNTIME).libraries.findAll {
          !(it.createReference().parentReference instanceof JpsModuleReference) && !plugin.includedProjectLibraries.contains(it.name)
@@ -89,7 +91,7 @@ class DistributionJARsBuilder {
      }
    }

-    Set<String> allProductDependencies = (productLayout.includedPluginModules + productLayout.includedPlatformModules).collectMany(new LinkedHashSet<String>()) {
+    Set<String> allProductDependencies = (productLayout.getIncludedPluginModules(enabledPluginModules) + productLayout.includedPlatformModules).collectMany(new LinkedHashSet<String>()) {
      JpsJavaExtensionService.dependencies(buildContext.findRequiredModule(it)).productionOnly().getModules().collect {it.name}
    }

@@ -137,6 +139,10 @@ class DistributionJARsBuilder {
    }
  }

+  private Set<String> getEnabledPluginModules() {
+    buildContext.productProperties.productLayout.bundledPluginModules + pluginsToPublish.collect { it.mainModule } as Set<String>
+  }
+
  List<String> getPlatformModules() {
    (platform.moduleJars.values() as List<String>) +
    ["java-runtime", "platform-main", /*required to build searchable options index*/
@@ -216,9 +222,6 @@ class DistributionJARsBuilder {
      layoutBuilder.patchModuleOutput(productLayout.searchableOptionsModule, FileUtil.toSystemIndependentName(searchableOptionsDir.absolutePath))
    }

-    String providedModulesFile = "${buildContext.paths.artifacts}/${buildContext.productProperties.productCode}-builtinModules.json"
-    buildTasks.buildProvidedModulesList(providedModulesFile, productLayout.mainModules, productLayout.licenseFilesToBuildSearchableOptions)
-
    def applicationInfoFile = FileUtil.toSystemIndependentName(patchedApplicationInfo.absolutePath)
    def applicationInfoDir = "$buildContext.paths.temp/applicationInfo"
    ant.copy(file: applicationInfoFile, todir: "$applicationInfoDir/idea")
@@ -243,6 +246,10 @@ class DistributionJARsBuilder {
    usedModules.addAll(layoutBuilder.usedModules)
  }

+  List<PluginLayout> getPluginsToPublish() {
+    return pluginsToPublish
+  }
+
  private void buildBundledPlugins() {
    def productLayout = buildContext.productProperties.productLayout
    def layoutBuilder = createLayoutBuilder()
@@ -251,11 +258,10 @@ class DistributionJARsBuilder {
  }

  void buildNonBundledPlugins() {
-    def ant = buildContext.ant
    def productLayout = buildContext.productProperties.productLayout
+    def ant = buildContext.ant
    def layoutBuilder = createLayoutBuilder()
    buildContext.executeStep("Build non-bundled plugins", BuildOptions.NON_BUNDLED_PLUGINS_STEP) {
-      def pluginsToPublish = getPluginsByModules(productLayout.pluginModulesToPublish)
      if (buildContext.productProperties.setPluginAndIDEVersionInPluginXml) {
        pluginsToPublish.each { plugin ->
          def moduleOutput = buildContext.getModuleOutputPath(buildContext.findRequiredModule(plugin.mainModule))
@@ -273,18 +279,21 @@ class DistributionJARsBuilder {
        }
      }

-      def pluginsToPublishDir = "$buildContext.paths.temp/plugins-to-publish"
+      def pluginsToPublishDir = "$buildContext.paths.temp/${buildContext.productProperties.productCode}-plugins-to-publish"
      buildPlugins(layoutBuilder, pluginsToPublish, pluginsToPublishDir)
-      def nonBundledPluginsArtifacts = "$buildContext.paths.artifacts/plugins"
+      def nonBundledPluginsArtifacts = "$buildContext.paths.artifacts/${buildContext.productProperties.productCode}-plugins"
      pluginsToPublish.each { plugin ->
        def directory = plugin.directoryName
        String suffix = productLayout.prepareCustomPluginRepositoryForPublishedPlugins ? "" : "-${getPluginVersion(plugin)}"
-        ant.zip(destfile: "$nonBundledPluginsArtifacts/$directory${suffix}.zip") {
+        def destFile = "$nonBundledPluginsArtifacts/$directory${suffix}.zip"
+        ant.zip(destfile: destFile) {
          zipfileset(dir: "$pluginsToPublishDir/$directory", prefix: directory)
        }
+        buildContext.notifyArtifactBuilt(destFile)
      }
      if (productLayout.prepareCustomPluginRepositoryForPublishedPlugins) {
        new PluginRepositoryXmlGenerator(buildContext).generate(pluginsToPublish, nonBundledPluginsArtifacts)
+        buildContext.notifyArtifactBuilt("$nonBundledPluginsArtifacts/plugins.xml")
      }
    }
  }
@@ -293,7 +302,7 @@ class DistributionJARsBuilder {
    return plugin.versionEvaluator.apply(buildContext)
  }

-  private List<PluginLayout> getPluginsByModules(Collection<String> modules) {
+  List<PluginLayout> getPluginsByModules(Collection<String> modules) {
    def allNonTrivialPlugins = buildContext.productProperties.productLayout.allNonTrivialPlugins
    def allOptionalModules = allNonTrivialPlugins.collectMany {it.optionalModules}
    def nonTrivialPlugins = allNonTrivialPlugins.groupBy { it.mainModule }
@@ -301,7 +310,7 @@ class DistributionJARsBuilder {
  }

  private void buildPlugins(LayoutBuilder layoutBuilder, List<PluginLayout> pluginsToInclude, String targetDirectory) {
-    def enabledModulesSet = buildContext.productProperties.productLayout.enabledPluginModules
+    def enabledModulesSet = enabledPluginModules
    pluginsToInclude.each { plugin ->
      def actualModuleJars = plugin.getActualModules(enabledModulesSet)
      checkOutputOfPluginModules(plugin.mainModule, actualModuleJars.values(), plugin.moduleExcludes)
@@ -478,7 +487,6 @@ class DistributionJARsBuilder {
    JpsPathUtil.urlToPath(buildContext.findRequiredModule(moduleName).contentRootsList.urls.first())
  }

-
  private LayoutBuilder createLayoutBuilder() {
    new LayoutBuilder(buildContext.ant, buildContext.project, COMPRESS_JARS)
  }
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2000-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jetbrains.intellij.build.impl
+
+import groovy.json.JsonSlurper
+import org.jetbrains.annotations.NotNull
+import org.jetbrains.intellij.build.BuildContext
+
+class PluginsCollector {
+  private final String myProvidedModulesFilePath
+  private final BuildContext myBuildContext
+
+  PluginsCollector(@NotNull BuildContext buildContext, @NotNull String providedModulesFilePath) {
+    this.myBuildContext = buildContext
+    this.myProvidedModulesFilePath = providedModulesFilePath
+  }
+
+  List<PluginLayout> collectCompatiblePluginsToPublish() {
+    def parse = new JsonSlurper().parse(new File(myProvidedModulesFilePath)) as Map
+    Set<String> availableModulesAndPlugins = new HashSet<String>(parse['modules'] as Collection)
+    availableModulesAndPlugins.addAll(parse['plugins'] as Collection)
+
+    def descriptorsMap = collectPluginDescriptors()
+    def pluginDescriptors = new HashSet<PluginDescriptor>(descriptorsMap.values())
+    return pluginDescriptors.findAll { isPluginCompatible(it, availableModulesAndPlugins, descriptorsMap) }.collect { it.pluginLayout }
+  }
+
+  private boolean isPluginCompatible(@NotNull PluginDescriptor plugin,
+                                     @NotNull Set<String> availableModulesAndPlugins,
+                                     @NotNull Map<String, PluginDescriptor> nonCheckedPlugins) {
+    nonCheckedPlugins.remove(plugin.id)
+    for (requiredDependency in plugin.requiredDependencies) {
+      if (availableModulesAndPlugins.contains(requiredDependency)) {
+        continue
+      }
+      def requiredPlugin = nonCheckedPlugins.get(requiredDependency)
+      if (requiredPlugin != null && isPluginCompatible(requiredPlugin, availableModulesAndPlugins, nonCheckedPlugins)) {
+        continue
+      }
+      return false
+    }
+    availableModulesAndPlugins.add(plugin.id)
+    return true
+  }
+
+  private Map<String, PluginDescriptor> collectPluginDescriptors() {
+    def pluginDescriptors = new HashMap<String, PluginDescriptor>()
+    def productLayout = myBuildContext.productProperties.productLayout
+    def nonTrivialPlugins = productLayout.allNonTrivialPlugins.groupBy { it.mainModule }
+    myBuildContext.project.modules.each {
+      if (productLayout.bundledPluginModules.contains(it.name)) {
+        return
+      }
+      PluginLayout pluginLayout = nonTrivialPlugins[it.name]?.first()
+      if (pluginLayout == null) {
+        pluginLayout = PluginLayout.plugin(it.name)
+      }
+      def pluginXml = myBuildContext.findFileInModuleSources(it.name, "META-INF/plugin.xml")
+      if (pluginXml == null) {
+        return
+      }
+
+      def xml = new XmlParser().parse(pluginXml)
+      String id = xml.id.text() ?: xml.name.text()
+      if (!id) {
+        return
+      }
+      def requiredDependencies = new HashSet()
+      for (dependency in xml.depends) {
+        if (dependency.@optional != 'true') {
+          requiredDependencies += dependency.text()
+        }
+      }
+
+      pluginDescriptors[id] = new PluginDescriptor(id, requiredDependencies, pluginLayout)
+    }
+    return pluginDescriptors
+  }
+
+  private class PluginDescriptor {
+    private final String id
+    private final Set<String> requiredDependencies
+    private final PluginLayout pluginLayout
+
+    PluginDescriptor(String id, Set<String> requiredDependencies, PluginLayout pluginLayout) {
+      this.id = id
+      this.requiredDependencies = requiredDependencies
+      this.pluginLayout = pluginLayout
+    }
+  }
+}
@@ -844,7 +844,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
      myPluginsDownloadUrl = downloadUrl != null ? downloadUrl : myPluginManagerUrl + (closed ? "" : "/") + "pluginManager/";

      if (!getBuild().isSnapshot()) {
-        myBuiltinPluginsUrl = pluginsElement.getAttributeValue(ATTRIBUTE_BUILTIN_URL);
+        myBuiltinPluginsUrl = StringUtil.nullize(pluginsElement.getAttributeValue(ATTRIBUTE_BUILTIN_URL));
      }
    }
    else {