Add ability to build non-bundled plugins while updating from sources and install the via plugins repository

- additional bundled plugin modules are disabled as ruby and python plugin can be now installed in a natural way
- snapshot distributions are now allowed to use builtin plugins repository url
- add date to snapshot plugin versions to make IDEA understand that they can be updated

GitOrigin-RevId: 8ffb3c9c8f760f0fd4caf02e117c4d89fdfbed87

build/scripts/idea_community.gant

@@ -1,5 +1,6 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 
+import com.intellij.util.SystemProperties
 import org.jetbrains.intellij.build.BuildOptions
 import org.jetbrains.intellij.build.IdeaCommunityBuilder
 import org.jetbrains.jps.idea.IdeaProjectLoader
@@ -30,9 +31,12 @@ target('update-from-sources': 'Update locally installed distribution from compil
   options.useCompiledClassesFromProjectOutput = true
   options.buildStepsToSkip << BuildOptions.SVGICONS_PREBUILD_STEP
   options.buildStepsToSkip << BuildOptions.SEARCHABLE_OPTIONS_INDEX_STEP
-  options.buildStepsToSkip << BuildOptions.PROVIDED_MODULES_LIST_STEP
   options.buildStepsToSkip << BuildOptions.SOURCES_ARCHIVE_STEP
   options.buildStepsToSkip << BuildOptions.SVGICONS_PREBUILD_STEP
+  if (!SystemProperties.getBooleanProperty('intellij.build.local.plugins.repository', false)) {
+    options.buildStepsToSkip << BuildOptions.PROVIDED_MODULES_LIST_STEP
+    options.buildStepsToSkip << BuildOptions.NON_BUNDLED_PLUGINS_STEP
+  }
   //when IDEA CE is updated from IDEA UE sources project should be loaded from IDEA UE directory
   String projectHome = isDefined("devIdeaHome") ? devIdeaHome : home
   new IdeaCommunityBuilder(home, options, projectHome).buildUnpackedDistribution("$options.outputRootPath/$distOutputRelativePath")

build/update.cmd

@@ -31,7 +31,7 @@ CD "%DEV_IDEA_HOME%"
 
 SET ANT_HOME=%DEV_IDEA_HOME%\lib\ant
 SET EXEC_ANT="%JAVA_HOME%\bin\java.exe" -Xmx512m -Dant.home="%ANT_HOME%" -classpath "%ANT_HOME%\lib\ant-launcher.jar" org.apache.tools.ant.launch.Launcher
-%EXEC_ANT% -f build/update.xml -Dwork.idea.home="%WORK_IDEA_HOME%"
+%EXEC_ANT% -f build/update.xml -Dwork.idea.home="%WORK_IDEA_HOME%" -Dintellij.build.local.plugins.repository="%BUILD_LOCAL_PLUGINS_REPOSITORY%"
 IF NOT ERRORLEVEL 0 GOTO failed
 IF NOT EXIST "%DEV_IDEA_HOME%\out\deploy" GOTO failed
 

build/update.sh

@@ -36,7 +36,7 @@ echo "Updating $WORK_IDEA_HOME from compiled classes in $DEV_IDEA_HOME"
 
 ANT_HOME="$DEV_IDEA_HOME/lib/ant"
 "$JAVA_BIN" -Xmx512m -Dant.home="$ANT_HOME" -classpath "$ANT_HOME/lib/ant-launcher.jar" org.apache.tools.ant.launch.Launcher \
- -f "$DEV_IDEA_HOME/build/update.xml" -Dwork.idea.home="$WORK_IDEA_HOME"
+ -f "$DEV_IDEA_HOME/build/update.xml" -Dwork.idea.home="$WORK_IDEA_HOME" -Dintellij.build.local.plugins.repository=$BUILD_LOCAL_PLUGINS_REPOSITORY
 
 if [ "$?" != "0" ]; then
   echo "Update failed; work IDEA build not modified"

build/update.xml

@@ -14,6 +14,7 @@
       <java classname="org.apache.tools.ant.Main" dir="${project.home}" fork="true" failonerror="true">
         <jvmarg line="-Xmx512m"/>
         <jvmarg line="&quot;-Dintellij.build.output.root=@{deploy}&quot;"/>
+        <jvmarg line="&quot;-Dintellij.build.local.plugins.repository=${intellij.build.local.plugins.repository}&quot;"/>
         <jvmarg line="&quot;-DdistOutputRelativePath=@{distOutputRelativePath}&quot;"/>
         <jvmarg line="&quot;-Dgant.script=@{script}&quot;"/>
         <jvmarg line="&quot;-Dgant.target=@{target}&quot;"/>

platform/build-scripts/groovy/org/jetbrains/intellij/build/ArtifactsServer.groovy

@@ -16,6 +16,7 @@
 package org.jetbrains.intellij.build
 
 import groovy.transform.CompileStatic
+import org.jetbrains.annotations.NotNull
 
 @CompileStatic
 interface ArtifactsServer {
@@ -23,5 +24,5 @@ interface ArtifactsServer {
    * Returns full URL which can be used to download an artifact.
    * @param relativePath path to an artifact from {@link org.jetbrains.intellij.build.BuildPaths#artifacts} directory
    */
-  String urlToArtifact(String relativePath)
+  String urlToArtifact(@NotNull BuildContext buildContext, @NotNull String relativePath)
 }

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy

@@ -192,8 +192,8 @@ idea.fatal.error.notification=disabled
     def artifactsServer = buildContext.proprietaryBuildTools.artifactsServer
     def builtinPluginsRepoUrl = ""
     if (artifactsServer != null && buildContext.productProperties.productLayout.prepareCustomPluginRepositoryForPublishedPlugins) {
-      builtinPluginsRepoUrl = artifactsServer.urlToArtifact("${buildContext.applicationInfo.productCode}-plugins/plugins.xml")
-      if (!builtinPluginsRepoUrl.startsWith("https:")) {
+      builtinPluginsRepoUrl = artifactsServer.urlToArtifact(buildContext, "${buildContext.applicationInfo.productCode}-plugins/plugins.xml")
+      if (builtinPluginsRepoUrl.startsWith("http:")) {
         buildContext.messages.error("Insecure artifact server: " + builtinPluginsRepoUrl)
       }
     }

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy

@@ -728,7 +728,8 @@ class DistributionJARsBuilder {
         def includeInCustomRepository = productLayout.prepareCustomPluginRepositoryForPublishedPlugins && publishingSpec.includeInCustomPluginRepository
 
         def directory = getActualPluginDirectoryName(plugin, buildContext)
-        String suffix = includeInCustomRepository ? "" : "-${getPluginVersion(plugin)}"
+        def pluginVersion = getPluginVersion(plugin)
+        String suffix = includeInCustomRepository ? "" : "-$pluginVersion"
         def targetDirectory = publishingSpec.includeIntoDirectoryForAutomaticUploading &&
                               whiteList.contains(plugin.mainModule)
           ? "$nonBundledPluginsArtifacts/auto-uploading"
@@ -737,6 +738,7 @@ class DistributionJARsBuilder {
 
         if (includeInCustomRepository) {
           pluginsToIncludeInCustomRepository.add(new PluginRepositorySpec(pluginZip: destFile.toString(),
+                                                                          pluginVersion: pluginVersion,
                                                                           pluginXml: pluginXmlFiles[plugin]))
         }
 

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PluginLayout.groovy

@@ -33,7 +33,9 @@ class PluginLayout extends BaseLayout {
   String directoryName
   final Set<String> optionalModules = new LinkedHashSet<>()
   private boolean doNotCreateSeparateJarForLocalizableResources
-  Function<BuildContext, String> versionEvaluator = { BuildContext context -> context.buildNumber } as Function<BuildContext, String>
+  Function<BuildContext, String> versionEvaluator = { BuildContext context ->
+    context.buildNumber.endsWith(".SNAPSHOT") ? context.buildNumber + ".${new Date().format('yyyyMMdd')}" : context.buildNumber
+  } as Function<BuildContext, String>
   boolean directoryNameSetExplicitly
   PluginPublishingSpec defaultPublishingSpec
   PluginBundlingRestrictions bundlingRestrictions

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PluginRepositorySpec.groovy

@@ -6,5 +6,6 @@ import groovy.transform.CompileStatic
 @CompileStatic
 class PluginRepositorySpec {
   String pluginZip
+  String pluginVersion
   String pluginXml
 }

platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/PluginRepositoryXmlGenerator.groovy

@@ -23,7 +23,7 @@ class PluginRepositoryXmlGenerator {
   void generate(List<PluginRepositorySpec> pluginSpecs, String targetDirectory) {
     def categories = new TreeMap<String, List<Plugin>>()
     pluginSpecs.each { spec ->
-      def p = readPlugin(new File(spec.pluginZip), new File(spec.pluginXml), buildContext.buildNumber, new File(targetDirectory))
+      def p = readPlugin(new File(spec.pluginZip), new File(spec.pluginXml), spec.pluginVersion, buildContext.buildNumber, new File(targetDirectory))
       categories.putIfAbsent(p.category, [])
       categories[p.category] << p
     }
@@ -56,7 +56,7 @@ class PluginRepositoryXmlGenerator {
 
   @SuppressWarnings("GrUnresolvedAccess")
   @CompileDynamic
-  private Plugin readPlugin(File pluginZip, File pluginXml, String buildNumber, File targetDirectory) {
+  private Plugin readPlugin(File pluginZip, File pluginXml, String pluginVersion, String buildNumber, File targetDirectory) {
     def xml = new XmlParser().parse(pluginXml)
     def versionNode = xml."idea-version"[0]
 
@@ -74,7 +74,7 @@ class PluginRepositoryXmlGenerator {
       vendor: xml.vendor.text(),
       sinceBuild: versionNode?.attribute("since-build") ?: buildNumber,
       untilBuild: versionNode?.attribute("until-build") ?: buildNumber,
-      version: buildNumber,
+      version: pluginVersion,
       description: xml.description.text(),
       relativeFilePath: FileUtil.getRelativePath(targetDirectory, pluginZip),
       size: pluginZip.length(),

platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java

@@ -858,9 +858,7 @@ public final class ApplicationInfoImpl extends ApplicationInfoEx {
         myPluginsDownloadUrl = downloadUrl;
       }
 
-      if (!getBuild().isSnapshot()) {
-        myBuiltinPluginsUrl = StringUtil.nullize(pluginsElement.getAttributeValue(ATTRIBUTE_BUILTIN_URL));
-      }
+      myBuiltinPluginsUrl = StringUtil.nullize(pluginsElement.getAttributeValue(ATTRIBUTE_BUILTIN_URL));
     }
 
     final String pluginsHost = System.getProperty(IDEA_PLUGINS_HOST_PROPERTY);