archive non-bundled plugins in parallel

GitOrigin-RevId: 39064924c2d8021839a91f8efa9b8025fa02b9d2
2026-09-27 10:03:11 +07:00 · 2020-12-23 11:26:07 +00:00
parent 84f4b76e09
commit 4e1ed1070d
13 changed files with 97 additions and 53 deletions
@@ -89,7 +89,7 @@ class IntelliJCoreArtifactsBuilder {
               file: "$coreArtifactDir/README.txt")
      processCoreLayout(coreArtifactDir, new ProjectStructureMapping(), true)
      ant.move(file: "$coreArtifactDir/annotations-java5.jar", tofile: "$coreArtifactDir/annotations.jar")
-      buildContext.notifyArtifactBuilt(coreArtifactDir.toString())
+      buildContext.notifyArtifactWasBuilt(coreArtifactDir)

      new ClassVersionChecker(["": "1.8", "intellij-core-analysis-deprecated.jar": "11"]).checkVersions(buildContext, coreArtifactDir)

@@ -8,6 +8,10 @@ fun Logger.error(message: String) {
  log(Logger.Level.ERROR, null as ResourceBundle?, message)
 }

+fun Logger.error(error: Throwable) {
+  log(Logger.Level.ERROR, null as ResourceBundle?, error.message, error)
+}
+
 fun Logger.info(message: String) {
  log(Logger.Level.INFO, null as ResourceBundle?, message)
 }
@@ -7,6 +7,10 @@ import java.nio.file.Path
 import java.nio.file.StandardOpenOption
 import java.time.Duration
 import java.util.*
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
 import java.util.zip.Deflater
 import java.util.zip.ZipEntry

@@ -58,6 +62,32 @@ fun zip(targetFile: Path, dirs: Map<Path, String>, compress: Boolean = true, add
  logger?.info("${targetFile.fileName} created in ${formatDuration(System.currentTimeMillis() - start)}")
 }

+fun bulkZipWithPrefix(commonSourceDir: Path, items: List<Map.Entry<String, Path>>, compress: Boolean, logger: System.Logger) {
+  val pool = Executors.newWorkStealingPool()
+  logger.debug { "Create ${items.size} archives in parallel (commonSourceDir=$commonSourceDir)" }
+  val error = AtomicReference<Throwable>()
+  for (item in items) {
+    pool.execute {
+      if (error.get() != null) {
+        return@execute
+      }
+
+      try {
+        zip(item.value, mapOf(commonSourceDir.resolve(item.key) to item.key), compress, logger = logger)
+      }
+      catch (e: Throwable) {
+        error.compareAndSet(null, e)
+      }
+    }
+  }
+
+  pool.shutdown()
+  pool.awaitTermination(1, TimeUnit.HOURS)
+  error.get()?.let {
+    throw it
+  }
+}
+
 private fun formatDuration(value: Long): String {
  return Duration.ofMillis(value).toString().substring(2)
    .replace(Regex("(\\d[HMS])(?!$)"), "$1 ")
@@ -123,4 +153,8 @@ private fun compressDir(startDir: Path, archiver: ZipArchiver) {
      }
    }
  }
+}
+
+internal fun createIoTaskExecutorPool(): ExecutorService {
+  return Executors.newWorkStealingPool(if (Runtime.getRuntime().availableProcessors() > 2) 4 else 2)
 }
@@ -5,10 +5,7 @@ import com.google.common.hash.HashFunction
 import com.google.common.hash.Hashing
 import it.unimi.dsi.fastutil.ints.IntOpenHashSet
 import org.apache.commons.compress.archivers.zip.ZipFile
-import org.jetbrains.intellij.build.io.ZipFileWriter
-import org.jetbrains.intellij.build.io.deleteDir
-import org.jetbrains.intellij.build.io.info
-import org.jetbrains.intellij.build.io.runJava
+import org.jetbrains.intellij.build.io.*
 import java.lang.System.Logger
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
@@ -16,7 +13,6 @@ import java.nio.channels.FileChannel
 import java.nio.file.*
 import java.util.*
 import java.util.concurrent.Callable
-import java.util.concurrent.Executors
 import java.util.concurrent.Future
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.zip.ZipEntry
@@ -124,7 +120,7 @@ internal fun doReorderJars(sourceToNames: Map<Path, List<String>>,
                           sourceDir: Path,
                           targetDir: Path,
                           logger: Logger): List<PackageIndexEntry> {
-  val executor = Executors.newWorkStealingPool(if (Runtime.getRuntime().availableProcessors() > 2) 4 else 2)
+  val executor = createIoTaskExecutorPool()

  val results = mutableListOf<Future<PackageIndexEntry?>>()
  val errorOccurred = AtomicBoolean()
@@ -1,18 +1,4 @@
-/*
- * Copyright 2000-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package org.jetbrains.intellij.build

 import groovy.transform.CompileStatic
@@ -21,6 +7,8 @@ import org.jetbrains.jps.model.JpsModel
 import org.jetbrains.jps.model.JpsProject
 import org.jetbrains.jps.model.module.JpsModule

+import java.nio.file.Path
+
@CompileStatic
 interface CompilationContext {
  AntBuilder getAnt()
@@ -49,5 +37,8 @@ interface CompilationContext {

  List<String> getModuleRuntimeClasspath(JpsModule module, boolean forTests)

+  // "Was" added due to Groovy bug (compilation error - cannot find method with same name but different parameter type)
+  void notifyArtifactWasBuilt(Path artifactPath)
+
  void notifyArtifactBuilt(String artifactPath)
 }
@@ -198,6 +198,11 @@ final class BuildContextImpl extends BuildContext {
    compilationContext.notifyArtifactBuilt(artifactPath)
  }

+  @Override
+  void notifyArtifactWasBuilt(Path artifactPath) {
+    compilationContext.notifyArtifactWasBuilt(artifactPath)
+  }
+
  @Override
  @Nullable Path findFileInModuleSources(String moduleName, String relativePath) {
    for (Pair<Path, String> info : getSourceRootsWithPrefixes(findRequiredModule(moduleName)) ) {
@@ -21,6 +21,7 @@ final class BuildHelper {

  private final UrlClassLoader helperClassLoader
  private final MethodHandle zipHandle
+  private final MethodHandle bulkZipWithPrefixHandle
  private final MethodHandle runJavaHandle
  final MethodHandle brokenPluginsTask
  final MethodHandle reorderJars
@@ -43,6 +44,10 @@ final class BuildHelper {
                                  "zip",
                                  MethodType.methodType(voidClass, path, Map.class as Class<?>, bool, bool, logger))

+    bulkZipWithPrefixHandle = lookup.findStatic(helperClassLoader.loadClass("org.jetbrains.intellij.build.io.ZipKt"),
+                                                "bulkZipWithPrefix",
+                                                MethodType.methodType(voidClass, path, List.class as Class<?>, bool, logger))
+
    runJavaHandle = lookup.findStatic(helperClassLoader.loadClass("org.jetbrains.intellij.build.io.ProcessKt"),
                                      "runJava", MethodType.methodType(voidClass, String.class as Class<?>, iterable, iterable, iterable,
                                                                       logger))
@@ -78,13 +83,11 @@ final class BuildHelper {
    getInstance(buildContext).zipHandle.invokeWithArguments(targetFile, map, true, false, buildContext.messages)
  }

-  /**
-   * JAR differs from ZIP in our case - for ZIP directory entries are never created,
-   * for JAR directory entries are created, including parent directories, if there is at least one resource file in it.
-   */
-  static void jarWithPrefix(@NotNull BuildContext buildContext, @NotNull Path targetFile, Path dir, String prefix, boolean compress) {
-    getInstance(buildContext).zipHandle
-      .invokeWithArguments(targetFile, Collections.singletonMap(dir, prefix), compress, true, buildContext.messages)
+  static void bulkZipWithPrefix(@NotNull BuildContext buildContext,
+                                @NotNull Path commonSourceDir,
+                                @NotNull List<Map.Entry<String, Path>> items,
+                                boolean compress) {
+    getInstance(buildContext).bulkZipWithPrefixHandle.invokeWithArguments(commonSourceDir, items, compress, buildContext.messages)
  }

  /**
@@ -110,7 +110,7 @@ final class BuildTasksImpl extends BuildTasks {
      if (!Files.exists(targetFile)) {
        buildContext.messages.error("Failed to build provided modules list: $targetFile doesn't exist")
      }
-      buildContext.notifyArtifactBuilt(targetFile.toString())
+      buildContext.notifyArtifactWasBuilt(targetFile)
    })
  }

@@ -574,7 +574,7 @@ idea.fatal.error.notification=disabled
  private void copyDependenciesFile() {
    File outputFile = new File(buildContext.paths.artifacts, "dependencies.txt")
    FileUtil.copy(buildContext.dependenciesProperties.file, outputFile)
-    buildContext.notifyArtifactBuilt(outputFile.toString())
+    buildContext.notifyArtifactWasBuilt(outputFile.toPath())
  }

  @CompileStatic(TypeCheckingMode.SKIP)
@@ -29,6 +29,8 @@ import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService
 import org.jetbrains.jps.model.serialization.JpsProjectLoader
 import org.jetbrains.jps.util.JpsPathUtil

+import java.nio.file.Files
+import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.concurrent.atomic.AtomicLong
 import java.util.function.BiFunction
@@ -435,49 +437,54 @@ class CompilationContextImpl implements CompilationContext {
  }

  private static final AtomicLong totalSizeOfProducedArtifacts = new AtomicLong()
+
  @Override
  void notifyArtifactBuilt(String artifactPath) {
+    notifyArtifactWasBuilt(Paths.get(artifactPath).toAbsolutePath().normalize())
+  }
+
+  @Override
+  void notifyArtifactWasBuilt(Path file) {
    if (options.buildStepsToSkip.contains(BuildOptions.TEAMCITY_ARTIFACTS_PUBLICATION)) {
      return
    }
-    def file = new File(artifactPath)
-    def artifactsDir = new File(paths.artifacts)

-    if (file.isFile()) {
+    Path artifactsDir = Paths.get(paths.artifacts)
+    if (Files.isRegularFile(file)) {
      //temporary workaround until TW-54541 is fixed: if build is going to produce big artifacts and we have lack of free disk space it's better not to send 'artifactBuilt' message to avoid "No space left on device" errors
      def fileSize = file.size()
      if (fileSize > 1000000) {
-        def producedSize = totalSizeOfProducedArtifacts.addAndGet(fileSize)
-        def willBePublishedWhenBuildFinishes = FileUtil.isAncestor(artifactsDir, file, true)
+        long producedSize = totalSizeOfProducedArtifacts.addAndGet(fileSize)
+        boolean willBePublishedWhenBuildFinishes = FileUtil.isAncestor(artifactsDir.toString(), file.toString(), true)

        long oneGb = 1024L * 1024 * 1024
        long requiredAdditionalSpace = oneGb * 6
        long requiredSpaceForArtifacts = oneGb * 9
-        long availableSpace = file.freeSpace
+        long availableSpace = Files.getFileStore(file).getUsableSpace()
        //heuristics: a build publishes at most 9Gb of artifacts and requires some additional space for compiled classes, dependencies, temp files, etc.
        // So we'll publish an artifact earlier only if there will be enough space for its copy.
        def skipPublishing = willBePublishedWhenBuildFinishes && availableSpace < (requiredSpaceForArtifacts - producedSize) + requiredAdditionalSpace + fileSize
-        messages.debug("Checking free space before publishing $artifactPath (${StringUtil.formatFileSize(fileSize)}): ")
+        messages.debug("Checking free space before publishing $file (${StringUtil.formatFileSize(fileSize)}): ")
        messages.debug(" total produced: ${StringUtil.formatFileSize(producedSize)}")
        messages.debug(" available space: ${StringUtil.formatFileSize(availableSpace)}")
        messages.debug(" ${skipPublishing ? "will be" : "won't be"} skipped")
        if (skipPublishing) {
-          messages.info("Artifact $artifactPath won't be published early to avoid caching on agent (workaround for TW-54541)")
+          messages.info("Artifact $file won't be published early to avoid caching on agent (workaround for TW-54541)")
          return
        }
      }
    }

-    def pathToReport = file.absolutePath
-
-    def targetDirectoryPath = ""
-    if (FileUtil.isAncestor(artifactsDir, file.parentFile, true)) {
-      targetDirectoryPath = FileUtil.toSystemIndependentName(FileUtil.getRelativePath(artifactsDir, file.parentFile) ?: "")
+    String targetDirectoryPath = ""
+    if (file.parent.startsWith(artifactsDir)) {
+      targetDirectoryPath = FileUtil.toSystemIndependentName(artifactsDir.relativize(file.parent).toString())
    }

-    if (file.isDirectory()) {
-      targetDirectoryPath = (targetDirectoryPath ? targetDirectoryPath + "/"  : "") + file.name
+    if (Files.isDirectory(file)) {
+      targetDirectoryPath = (targetDirectoryPath ? targetDirectoryPath + "/"  : "") + file.fileName
    }
+
+    String pathToReport = file.toString()
    if (targetDirectoryPath) {
      pathToReport += "=>" + targetDirectoryPath
    }
@@ -502,6 +509,6 @@ class BuildPathsImpl extends BuildPaths {
    this.projectHome = projectHome
    this.jdkHome = jdkHome
    this.kotlinHome = kotlinHome
-    artifacts = "$buildOutputRoot/artifacts"
+    artifacts = "${this.buildOutputRoot}/artifacts"
  }
 }
@@ -625,6 +625,7 @@ final class DistributionJARsBuilder {

      Path autoUploadingDir = nonBundledPluginsArtifacts.resolve("auto-uploading")
      Path patchedPluginXmlDir = buildContext.paths.tempDir.resolve("patched-plugin-xml")
+      List<Map.Entry<String, Path>> toArchive = new ArrayList<>()
      for (plugin in pluginsToPublish) {
        String directory = getActualPluginDirectoryName(plugin, buildContext)
        Path targetDirectory = whiteList.contains(plugin.mainModule)
@@ -639,9 +640,12 @@ final class DistributionJARsBuilder {
          }
          pluginsToIncludeInCustomRepository.add(new PluginRepositorySpec(pluginZip: destFile.toString(), pluginXml: pluginXml.toString()))
        }
+        toArchive.add(new AbstractMap.SimpleImmutableEntry(directory, destFile))
+      }

-        BuildHelper.jarWithPrefix(buildContext, destFile, pluginsToPublishDir.resolve(directory), directory, compressPluginArchive)
-        buildContext.notifyArtifactBuilt(destFile.toString())
+      BuildHelper.bulkZipWithPrefix(buildContext, pluginsToPublishDir, toArchive, compressPluginArchive)
+      for (Map.Entry<String, Path> item : toArchive) {
+        buildContext.notifyArtifactWasBuilt(item.value)
      }

      for (PluginRepositorySpec item in KeymapPluginsBuilder.buildKeymapPlugins(buildContext, autoUploadingDir)) {
@@ -660,7 +664,7 @@ final class DistributionJARsBuilder {

      if (productLayout.prepareCustomPluginRepositoryForPublishedPlugins) {
        new PluginRepositoryXmlGenerator(buildContext).generate(pluginsToIncludeInCustomRepository, nonBundledPluginsArtifacts.toString())
-        buildContext.notifyArtifactBuilt(nonBundledPluginsArtifacts.resolve("plugins.xml").toString())
+        buildContext.notifyArtifactWasBuilt(nonBundledPluginsArtifacts.resolve("plugins.xml"))
      }
    }
  }
@@ -68,7 +68,7 @@ final class KeymapPluginsBuilder {
      }
    }
    Path resultFile = targetDir.resolve("${shortName}Keymap.zip")
-    buildContext.notifyArtifactBuilt(resultFile.toString())
+    buildContext.notifyArtifactWasBuilt(resultFile)
    return new PluginRepositorySpec(pluginZip: resultFile.toString(),
                                    pluginXml: metaInf.resolve("plugin.xml").toString())
  }
@@ -333,7 +333,7 @@ final class MacDmgBuilder {
          include(name: '**/' + logFileName)
        }
      }
-      buildContext.notifyArtifactBuilt(new File(artifactsPath, logFileName).absolutePath)
+      buildContext.notifyArtifactWasBuilt(new File(artifactsPath, logFileName).toPath())
      buildContext.messages.error("SSH command failed, details are available in $logFileName: $e.message", e)
    }
  }
@@ -312,7 +312,7 @@ final class WindowsDistributionBuilder extends OsSpecificDistributionBuilder {
      List<Path> dirs = [Paths.get(buildContext.paths.distAll), winDistPath, productJsonDir] + jreDirectoryPaths
      BuildHelper.zip(buildContext, targetFile, dirs)
      ProductInfoValidator.checkInArchive(buildContext, targetFile.toString(), "")
-      buildContext.notifyArtifactBuilt(targetFile.toString())
+      buildContext.notifyArtifactWasBuilt(targetFile)
      return targetFile
    }
  }