From 4e1ed1070d3b288938dc90f5d803362dbbf77ddd Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Sat, 19 Dec 2020 11:23:42 +0100 Subject: [PATCH] archive non-bundled plugins in parallel GitOrigin-RevId: 39064924c2d8021839a91f8efa9b8025fa02b9d2 --- .../build/IntelliJCoreArtifactsBuilder.groovy | 2 +- .../org/jetbrains/intellij/build/io/log.kt | 4 ++ .../org/jetbrains/intellij/build/io/zip.kt | 34 ++++++++++++++++ .../intellij/build/tasks/reorderJars.kt | 8 +--- .../intellij/build/CompilationContext.groovy | 21 +++------- .../build/impl/BuildContextImpl.groovy | 5 +++ .../intellij/build/impl/BuildHelper.groovy | 17 ++++---- .../intellij/build/impl/BuildTasksImpl.groovy | 4 +- .../build/impl/CompilationContextImpl.groovy | 39 +++++++++++-------- .../build/impl/DistributionJARsBuilder.groovy | 10 +++-- .../build/impl/KeymapPluginsBuilder.groovy | 2 +- .../intellij/build/impl/MacDmgBuilder.groovy | 2 +- .../impl/WindowsDistributionBuilder.groovy | 2 +- 13 files changed, 97 insertions(+), 53 deletions(-) diff --git a/build/groovy/org/jetbrains/intellij/build/IntelliJCoreArtifactsBuilder.groovy b/build/groovy/org/jetbrains/intellij/build/IntelliJCoreArtifactsBuilder.groovy index 2ac6445f40d7..51ed35e7d433 100644 --- a/build/groovy/org/jetbrains/intellij/build/IntelliJCoreArtifactsBuilder.groovy +++ b/build/groovy/org/jetbrains/intellij/build/IntelliJCoreArtifactsBuilder.groovy @@ -89,7 +89,7 @@ class IntelliJCoreArtifactsBuilder { file: "$coreArtifactDir/README.txt") processCoreLayout(coreArtifactDir, new ProjectStructureMapping(), true) ant.move(file: "$coreArtifactDir/annotations-java5.jar", tofile: "$coreArtifactDir/annotations.jar") - buildContext.notifyArtifactBuilt(coreArtifactDir.toString()) + buildContext.notifyArtifactWasBuilt(coreArtifactDir) new ClassVersionChecker(["": "1.8", "intellij-core-analysis-deprecated.jar": "11"]).checkVersions(buildContext, coreArtifactDir) diff --git a/build/tasks/src/org/jetbrains/intellij/build/io/log.kt b/build/tasks/src/org/jetbrains/intellij/build/io/log.kt index eda25f9fa6bc..146252c06e71 100644 --- a/build/tasks/src/org/jetbrains/intellij/build/io/log.kt +++ b/build/tasks/src/org/jetbrains/intellij/build/io/log.kt @@ -8,6 +8,10 @@ fun Logger.error(message: String) { log(Logger.Level.ERROR, null as ResourceBundle?, message) } +fun Logger.error(error: Throwable) { + log(Logger.Level.ERROR, null as ResourceBundle?, error.message, error) +} + fun Logger.info(message: String) { log(Logger.Level.INFO, null as ResourceBundle?, message) } diff --git a/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt b/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt index 292770a99592..13def68c6365 100644 --- a/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt +++ b/build/tasks/src/org/jetbrains/intellij/build/io/zip.kt @@ -7,6 +7,10 @@ import java.nio.file.Path import java.nio.file.StandardOpenOption import java.time.Duration import java.util.* +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference import java.util.zip.Deflater import java.util.zip.ZipEntry @@ -58,6 +62,32 @@ fun zip(targetFile: Path, dirs: Map, compress: Boolean = true, add logger?.info("${targetFile.fileName} created in ${formatDuration(System.currentTimeMillis() - start)}") } +fun bulkZipWithPrefix(commonSourceDir: Path, items: List>, compress: Boolean, logger: System.Logger) { + val pool = Executors.newWorkStealingPool() + logger.debug { "Create ${items.size} archives in parallel (commonSourceDir=$commonSourceDir)" } + val error = AtomicReference() + for (item in items) { + pool.execute { + if (error.get() != null) { + return@execute + } + + try { + zip(item.value, mapOf(commonSourceDir.resolve(item.key) to item.key), compress, logger = logger) + } + catch (e: Throwable) { + error.compareAndSet(null, e) + } + } + } + + pool.shutdown() + pool.awaitTermination(1, TimeUnit.HOURS) + error.get()?.let { + throw it + } +} + private fun formatDuration(value: Long): String { return Duration.ofMillis(value).toString().substring(2) .replace(Regex("(\\d[HMS])(?!$)"), "$1 ") @@ -123,4 +153,8 @@ private fun compressDir(startDir: Path, archiver: ZipArchiver) { } } } +} + +internal fun createIoTaskExecutorPool(): ExecutorService { + return Executors.newWorkStealingPool(if (Runtime.getRuntime().availableProcessors() > 2) 4 else 2) } \ No newline at end of file diff --git a/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt b/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt index 5fc398f5bc92..a98a0442bcd9 100644 --- a/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt +++ b/build/tasks/src/org/jetbrains/intellij/build/tasks/reorderJars.kt @@ -5,10 +5,7 @@ import com.google.common.hash.HashFunction import com.google.common.hash.Hashing import it.unimi.dsi.fastutil.ints.IntOpenHashSet import org.apache.commons.compress.archivers.zip.ZipFile -import org.jetbrains.intellij.build.io.ZipFileWriter -import org.jetbrains.intellij.build.io.deleteDir -import org.jetbrains.intellij.build.io.info -import org.jetbrains.intellij.build.io.runJava +import org.jetbrains.intellij.build.io.* import java.lang.System.Logger import java.nio.ByteBuffer import java.nio.ByteOrder @@ -16,7 +13,6 @@ import java.nio.channels.FileChannel import java.nio.file.* import java.util.* import java.util.concurrent.Callable -import java.util.concurrent.Executors import java.util.concurrent.Future import java.util.concurrent.atomic.AtomicBoolean import java.util.zip.ZipEntry @@ -124,7 +120,7 @@ internal fun doReorderJars(sourceToNames: Map>, sourceDir: Path, targetDir: Path, logger: Logger): List { - val executor = Executors.newWorkStealingPool(if (Runtime.getRuntime().availableProcessors() > 2) 4 else 2) + val executor = createIoTaskExecutorPool() val results = mutableListOf>() val errorOccurred = AtomicBoolean() diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/CompilationContext.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/CompilationContext.groovy index 174e547bb1e8..bea562ed91ba 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/CompilationContext.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/CompilationContext.groovy @@ -1,18 +1,4 @@ -/* - * Copyright 2000-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package org.jetbrains.intellij.build import groovy.transform.CompileStatic @@ -21,6 +7,8 @@ import org.jetbrains.jps.model.JpsModel import org.jetbrains.jps.model.JpsProject import org.jetbrains.jps.model.module.JpsModule +import java.nio.file.Path + @CompileStatic interface CompilationContext { AntBuilder getAnt() @@ -49,5 +37,8 @@ interface CompilationContext { List getModuleRuntimeClasspath(JpsModule module, boolean forTests) + // "Was" added due to Groovy bug (compilation error - cannot find method with same name but different parameter type) + void notifyArtifactWasBuilt(Path artifactPath) + void notifyArtifactBuilt(String artifactPath) } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildContextImpl.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildContextImpl.groovy index 3c0fedfc6277..8864d7880140 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildContextImpl.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildContextImpl.groovy @@ -198,6 +198,11 @@ final class BuildContextImpl extends BuildContext { compilationContext.notifyArtifactBuilt(artifactPath) } + @Override + void notifyArtifactWasBuilt(Path artifactPath) { + compilationContext.notifyArtifactWasBuilt(artifactPath) + } + @Override @Nullable Path findFileInModuleSources(String moduleName, String relativePath) { for (Pair info : getSourceRootsWithPrefixes(findRequiredModule(moduleName)) ) { diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildHelper.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildHelper.groovy index cdb3c36d01de..548c138ba03d 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildHelper.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildHelper.groovy @@ -21,6 +21,7 @@ final class BuildHelper { private final UrlClassLoader helperClassLoader private final MethodHandle zipHandle + private final MethodHandle bulkZipWithPrefixHandle private final MethodHandle runJavaHandle final MethodHandle brokenPluginsTask final MethodHandle reorderJars @@ -43,6 +44,10 @@ final class BuildHelper { "zip", MethodType.methodType(voidClass, path, Map.class as Class, bool, bool, logger)) + bulkZipWithPrefixHandle = lookup.findStatic(helperClassLoader.loadClass("org.jetbrains.intellij.build.io.ZipKt"), + "bulkZipWithPrefix", + MethodType.methodType(voidClass, path, List.class as Class, bool, logger)) + runJavaHandle = lookup.findStatic(helperClassLoader.loadClass("org.jetbrains.intellij.build.io.ProcessKt"), "runJava", MethodType.methodType(voidClass, String.class as Class, iterable, iterable, iterable, logger)) @@ -78,13 +83,11 @@ final class BuildHelper { getInstance(buildContext).zipHandle.invokeWithArguments(targetFile, map, true, false, buildContext.messages) } - /** - * JAR differs from ZIP in our case - for ZIP directory entries are never created, - * for JAR directory entries are created, including parent directories, if there is at least one resource file in it. - */ - static void jarWithPrefix(@NotNull BuildContext buildContext, @NotNull Path targetFile, Path dir, String prefix, boolean compress) { - getInstance(buildContext).zipHandle - .invokeWithArguments(targetFile, Collections.singletonMap(dir, prefix), compress, true, buildContext.messages) + static void bulkZipWithPrefix(@NotNull BuildContext buildContext, + @NotNull Path commonSourceDir, + @NotNull List> items, + boolean compress) { + getInstance(buildContext).bulkZipWithPrefixHandle.invokeWithArguments(commonSourceDir, items, compress, buildContext.messages) } /** diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy index 8e2eb18e8007..e8ad59dcf57c 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/BuildTasksImpl.groovy @@ -110,7 +110,7 @@ final class BuildTasksImpl extends BuildTasks { if (!Files.exists(targetFile)) { buildContext.messages.error("Failed to build provided modules list: $targetFile doesn't exist") } - buildContext.notifyArtifactBuilt(targetFile.toString()) + buildContext.notifyArtifactWasBuilt(targetFile) }) } @@ -574,7 +574,7 @@ idea.fatal.error.notification=disabled private void copyDependenciesFile() { File outputFile = new File(buildContext.paths.artifacts, "dependencies.txt") FileUtil.copy(buildContext.dependenciesProperties.file, outputFile) - buildContext.notifyArtifactBuilt(outputFile.toString()) + buildContext.notifyArtifactWasBuilt(outputFile.toPath()) } @CompileStatic(TypeCheckingMode.SKIP) diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/CompilationContextImpl.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/CompilationContextImpl.groovy index fab5bb3f7721..399c32b05e41 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/CompilationContextImpl.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/CompilationContextImpl.groovy @@ -29,6 +29,8 @@ import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService import org.jetbrains.jps.model.serialization.JpsProjectLoader import org.jetbrains.jps.util.JpsPathUtil +import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths import java.util.concurrent.atomic.AtomicLong import java.util.function.BiFunction @@ -435,49 +437,54 @@ class CompilationContextImpl implements CompilationContext { } private static final AtomicLong totalSizeOfProducedArtifacts = new AtomicLong() + @Override void notifyArtifactBuilt(String artifactPath) { + notifyArtifactWasBuilt(Paths.get(artifactPath).toAbsolutePath().normalize()) + } + + @Override + void notifyArtifactWasBuilt(Path file) { if (options.buildStepsToSkip.contains(BuildOptions.TEAMCITY_ARTIFACTS_PUBLICATION)) { return } - def file = new File(artifactPath) - def artifactsDir = new File(paths.artifacts) - if (file.isFile()) { + Path artifactsDir = Paths.get(paths.artifacts) + if (Files.isRegularFile(file)) { //temporary workaround until TW-54541 is fixed: if build is going to produce big artifacts and we have lack of free disk space it's better not to send 'artifactBuilt' message to avoid "No space left on device" errors def fileSize = file.size() if (fileSize > 1000000) { - def producedSize = totalSizeOfProducedArtifacts.addAndGet(fileSize) - def willBePublishedWhenBuildFinishes = FileUtil.isAncestor(artifactsDir, file, true) + long producedSize = totalSizeOfProducedArtifacts.addAndGet(fileSize) + boolean willBePublishedWhenBuildFinishes = FileUtil.isAncestor(artifactsDir.toString(), file.toString(), true) long oneGb = 1024L * 1024 * 1024 long requiredAdditionalSpace = oneGb * 6 long requiredSpaceForArtifacts = oneGb * 9 - long availableSpace = file.freeSpace + long availableSpace = Files.getFileStore(file).getUsableSpace() //heuristics: a build publishes at most 9Gb of artifacts and requires some additional space for compiled classes, dependencies, temp files, etc. // So we'll publish an artifact earlier only if there will be enough space for its copy. def skipPublishing = willBePublishedWhenBuildFinishes && availableSpace < (requiredSpaceForArtifacts - producedSize) + requiredAdditionalSpace + fileSize - messages.debug("Checking free space before publishing $artifactPath (${StringUtil.formatFileSize(fileSize)}): ") + messages.debug("Checking free space before publishing $file (${StringUtil.formatFileSize(fileSize)}): ") messages.debug(" total produced: ${StringUtil.formatFileSize(producedSize)}") messages.debug(" available space: ${StringUtil.formatFileSize(availableSpace)}") messages.debug(" ${skipPublishing ? "will be" : "won't be"} skipped") if (skipPublishing) { - messages.info("Artifact $artifactPath won't be published early to avoid caching on agent (workaround for TW-54541)") + messages.info("Artifact $file won't be published early to avoid caching on agent (workaround for TW-54541)") return } } } - def pathToReport = file.absolutePath - - def targetDirectoryPath = "" - if (FileUtil.isAncestor(artifactsDir, file.parentFile, true)) { - targetDirectoryPath = FileUtil.toSystemIndependentName(FileUtil.getRelativePath(artifactsDir, file.parentFile) ?: "") + String targetDirectoryPath = "" + if (file.parent.startsWith(artifactsDir)) { + targetDirectoryPath = FileUtil.toSystemIndependentName(artifactsDir.relativize(file.parent).toString()) } - if (file.isDirectory()) { - targetDirectoryPath = (targetDirectoryPath ? targetDirectoryPath + "/" : "") + file.name + if (Files.isDirectory(file)) { + targetDirectoryPath = (targetDirectoryPath ? targetDirectoryPath + "/" : "") + file.fileName } + + String pathToReport = file.toString() if (targetDirectoryPath) { pathToReport += "=>" + targetDirectoryPath } @@ -502,6 +509,6 @@ class BuildPathsImpl extends BuildPaths { this.projectHome = projectHome this.jdkHome = jdkHome this.kotlinHome = kotlinHome - artifacts = "$buildOutputRoot/artifacts" + artifacts = "${this.buildOutputRoot}/artifacts" } } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy index b8775435d7a3..03e23aaa6d42 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/DistributionJARsBuilder.groovy @@ -625,6 +625,7 @@ final class DistributionJARsBuilder { Path autoUploadingDir = nonBundledPluginsArtifacts.resolve("auto-uploading") Path patchedPluginXmlDir = buildContext.paths.tempDir.resolve("patched-plugin-xml") + List> toArchive = new ArrayList<>() for (plugin in pluginsToPublish) { String directory = getActualPluginDirectoryName(plugin, buildContext) Path targetDirectory = whiteList.contains(plugin.mainModule) @@ -639,9 +640,12 @@ final class DistributionJARsBuilder { } pluginsToIncludeInCustomRepository.add(new PluginRepositorySpec(pluginZip: destFile.toString(), pluginXml: pluginXml.toString())) } + toArchive.add(new AbstractMap.SimpleImmutableEntry(directory, destFile)) + } - BuildHelper.jarWithPrefix(buildContext, destFile, pluginsToPublishDir.resolve(directory), directory, compressPluginArchive) - buildContext.notifyArtifactBuilt(destFile.toString()) + BuildHelper.bulkZipWithPrefix(buildContext, pluginsToPublishDir, toArchive, compressPluginArchive) + for (Map.Entry item : toArchive) { + buildContext.notifyArtifactWasBuilt(item.value) } for (PluginRepositorySpec item in KeymapPluginsBuilder.buildKeymapPlugins(buildContext, autoUploadingDir)) { @@ -660,7 +664,7 @@ final class DistributionJARsBuilder { if (productLayout.prepareCustomPluginRepositoryForPublishedPlugins) { new PluginRepositoryXmlGenerator(buildContext).generate(pluginsToIncludeInCustomRepository, nonBundledPluginsArtifacts.toString()) - buildContext.notifyArtifactBuilt(nonBundledPluginsArtifacts.resolve("plugins.xml").toString()) + buildContext.notifyArtifactWasBuilt(nonBundledPluginsArtifacts.resolve("plugins.xml")) } } } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/KeymapPluginsBuilder.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/KeymapPluginsBuilder.groovy index 5673a80c0e40..a956c4931d2e 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/KeymapPluginsBuilder.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/KeymapPluginsBuilder.groovy @@ -68,7 +68,7 @@ final class KeymapPluginsBuilder { } } Path resultFile = targetDir.resolve("${shortName}Keymap.zip") - buildContext.notifyArtifactBuilt(resultFile.toString()) + buildContext.notifyArtifactWasBuilt(resultFile) return new PluginRepositorySpec(pluginZip: resultFile.toString(), pluginXml: metaInf.resolve("plugin.xml").toString()) } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.groovy index 0460e9a13ac0..9bc99fd574f9 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/MacDmgBuilder.groovy @@ -333,7 +333,7 @@ final class MacDmgBuilder { include(name: '**/' + logFileName) } } - buildContext.notifyArtifactBuilt(new File(artifactsPath, logFileName).absolutePath) + buildContext.notifyArtifactWasBuilt(new File(artifactsPath, logFileName).toPath()) buildContext.messages.error("SSH command failed, details are available in $logFileName: $e.message", e) } } diff --git a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.groovy b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.groovy index 5ec2643d66ce..8e28690fbf3a 100644 --- a/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.groovy +++ b/platform/build-scripts/groovy/org/jetbrains/intellij/build/impl/WindowsDistributionBuilder.groovy @@ -312,7 +312,7 @@ final class WindowsDistributionBuilder extends OsSpecificDistributionBuilder { List dirs = [Paths.get(buildContext.paths.distAll), winDistPath, productJsonDir] + jreDirectoryPaths BuildHelper.zip(buildContext, targetFile, dirs) ProductInfoValidator.checkInArchive(buildContext, targetFile.toString(), "") - buildContext.notifyArtifactBuilt(targetFile.toString()) + buildContext.notifyArtifactWasBuilt(targetFile) return targetFile } }