build scripts redesign: added option to use incremental compilation and to use compiled classes from an archive

build/groovy/org/jetbrains/intellij/build/BuildOptions.groovy

@@ -46,4 +46,14 @@ class BuildOptions {
    */
   public static final String BUILD_DMG_WITHOUT_BUNDLED_JRE = "intellij.build.dmg.without.bundled.jre"
   boolean buildDmgWithoutBundledJre = SystemProperties.getBooleanProperty(BUILD_DMG_WITHOUT_BUNDLED_JRE, SystemProperties.getBooleanProperty("artifact.mac.no.jdk", false))
+
+  /**
+   * Path to a zip file containing 'production' and 'test' directories with compiled classes of the project modules inside.
+   */
+  String pathToCompiledClassesArchive = System.getProperty("intellij.build.compiled.classes.archive")
+
+  /**
+   * If {@code true} the project modules will be compiled incrementally
+   */
+  boolean incrementalCompilation = SystemProperties.getBooleanProperty("intellij.build.incremental.compilation", false)
 }

build/groovy/org/jetbrains/intellij/build/impl/BuildContextImpl.groovy

@@ -17,15 +17,8 @@ package org.jetbrains.intellij.build.impl
 
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.text.StringUtil
-import com.intellij.util.SystemProperties
 import org.codehaus.gant.GantBuilder
-import org.jetbrains.intellij.build.ApplicationInfoProperties
-import org.jetbrains.intellij.build.BuildContext
-import org.jetbrains.intellij.build.BuildOptions
-import org.jetbrains.intellij.build.BuildPaths
-import org.jetbrains.intellij.build.MacHostProperties
-import org.jetbrains.intellij.build.ProductProperties
-import org.jetbrains.intellij.build.SignTool
+import org.jetbrains.intellij.build.*
 import org.jetbrains.jps.gant.JpsGantProjectBuilder
 import org.jetbrains.jps.model.JpsGlobal
 import org.jetbrains.jps.model.JpsProject
@@ -34,13 +27,13 @@ import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService
 import org.jetbrains.jps.model.serialization.JpsProjectLoader
 import org.jetbrains.jps.util.JpsPathUtil
-
 /**
  * @author nik
  */
 class BuildContextImpl extends BuildContext {
   private final JpsGlobal global
   private final underTeamCity
+  final List<String> outputDirectoriesToKeep = []
 
   BuildContextImpl(GantBuilder ant, JpsGantProjectBuilder projectBuilder, JpsProject project, JpsGlobal global,
                    String communityHome, String projectHome, String buildOutputRoot, ProductProperties productProperties,
@@ -80,18 +73,33 @@ class BuildContextImpl extends BuildContext {
     }
     JpsModelSerializationDataService.getOrCreatePathVariablesConfiguration(global).addPathVariable("KOTLIN_BUNDLED", bundledKotlinPath)
 
-    projectBuilder.buildIncrementally = SystemProperties.getBooleanProperty("jps.build.incrementally", false)
-    def dataDirName = projectBuilder.buildIncrementally ? ".jps-incremental-build" : ".jps-build-data"
-    projectBuilder.dataStorageRoot = new File("$projectHome/$dataDirName")
+    checkOptions()
+    projectBuilder.buildIncrementally = options.incrementalCompilation
+    def dataDirName = options.incrementalCompilation ? ".jps-build-data-incremental" : ".jps-build-data"
+    projectBuilder.dataStorageRoot = new File(paths.buildOutputRoot, dataDirName)
     def tempDir = System.getProperty("teamcity.build.tempDir") ?: System.getProperty("java.io.tmpdir")
-    projectBuilder.setupAdditionalLogging(new File("$tempDir/system/build-log/build.log"), System.getProperty("jps.build.debug.logging.categories", ""))
+    projectBuilder.setupAdditionalLogging(new File("$tempDir/system/build-log/build.log"), System.getProperty("intellij.build.debug.logging.categories", ""))
 
     def pathVariables = JpsModelSerializationDataService.computeAllPathVariables(global)
     JpsProjectLoader.loadProject(project, pathVariables, projectHome)
     projectBuilder.exportModuleOutputProperties()
     messages.info("Loaded project $projectHome: ${project.modules.size()} modules, ${project.libraryCollection.libraries.size()} libraries")
+
+    def classesDirName = "classes"
+    def classesOutput = "$paths.buildOutputRoot/$classesDirName"
+    if (options.pathToCompiledClassesArchive != null) {
+      messages.block("Unpack compiled classes archive") {
+        FileUtil.delete(new File(classesOutput))
+        ant.unzip(src: options.pathToCompiledClassesArchive, dest: classesOutput)
+      }
+      outputDirectoriesToKeep.add(classesDirName)
+    }
+    if (options.incrementalCompilation) {
+      outputDirectoriesToKeep.add(dataDirName)
+      outputDirectoriesToKeep.add(classesDirName)
+    }
     if (!options.useCompiledClassesFromProjectOutput) {
-      projectBuilder.targetFolder = "$paths.buildOutputRoot/classes"
+      projectBuilder.targetFolder = classesOutput
     }
     else {
       def outputDir = JpsPathUtil.urlToFile(JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(project).outputUrl)
@@ -103,6 +111,21 @@ class BuildContextImpl extends BuildContext {
     suppressWarnings()
   }
 
+  private void checkOptions() {
+    if (options.useCompiledClassesFromProjectOutput && options.incrementalCompilation) {
+      messages.warning("'${BuildOptions.USE_COMPILED_CLASSES_PROPERTY}' is specified, so 'incremental compilation' option will be ignored")
+      options.incrementalCompilation = false
+    }
+    if (options.pathToCompiledClassesArchive != null && options.incrementalCompilation) {
+      messages.warning("Paths to the compiled project output is specified, so 'incremental compilation' option will be ignored")
+      options.incrementalCompilation = false
+    }
+    if (options.pathToCompiledClassesArchive != null && options.useCompiledClassesFromProjectOutput) {
+      messages.warning("'${BuildOptions.USE_COMPILED_CLASSES_PROPERTY}' is specified, so the archive with compiled project output won't be used")
+      options.pathToCompiledClassesArchive = null
+    }
+  }
+
   private void suppressWarnings() {
     def compilerOptions = JpsJavaExtensionService.instance.getOrCreateCompilerConfiguration(project).currentCompilerOptions
     compilerOptions.GENERATE_NO_WARNINGS = true

build/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy

@@ -205,7 +205,14 @@ idea.fatal.error.notification=disabled
     buildContext.messages.block("Clean output") {
       def outputPath = buildContext.paths.buildOutputRoot
       buildContext.messages.progress("Cleaning output directory $outputPath")
-      FileUtil.delete(new File(outputPath))
+      new File(outputPath).listFiles()?.each {
+        if (buildContext instanceof BuildContextImpl && buildContext.outputDirectoriesToKeep.contains(it.name)) {
+          buildContext.messages.info("Skipped cleaning for $it.absolutePath")
+        }
+        else {
+          FileUtil.delete(it)
+        }
+      }
     }
   }
 
@@ -215,6 +222,10 @@ idea.fatal.error.notification=disabled
       buildContext.messages.info("Compilation skipped, the compiled classes from the project output will be used")
       return
     }
+    if (buildContext.options.pathToCompiledClassesArchive != null) {
+      buildContext.messages.info("Compilation skipped, the compiled classes from '${buildContext.options.pathToCompiledClassesArchive}' will be used")
+      return
+    }
 
     buildContext.projectBuilder.cleanOutput()
     buildContext.projectBuilder.buildProduction()