diff --git a/build/jvm-rules/src/jvm-args.bzl b/build/jvm-rules/src/jvm-args.bzl index c6a42ed61b3d..d7f1e41b497b 100644 --- a/build/jvm-rules/src/jvm-args.bzl +++ b/build/jvm-rules/src/jvm-args.bzl @@ -17,6 +17,7 @@ def get_jvm_flags(flags): # kotlin compiler "-Dkotlin.environment.keepalive=true", "-Didea.io.use.nio2=true", + "-Dio.netty.allocator.useCachedMagazinesForNonEventLoopThreads=true", # https://github.com/netty/netty/issues/11532 "-Dio.netty.tryReflectionSetAccessible=true", # see TargetConfigurationDigestProperty.KOTLIN_VERSION - we invalidate cache if kotlinc version changed diff --git a/build/jvm-rules/src/misc/BUILD.bazel b/build/jvm-rules/src/misc/BUILD.bazel index 1ac93072ad7e..304476c18e6c 100644 --- a/build/jvm-rules/src/misc/BUILD.bazel +++ b/build/jvm-rules/src/misc/BUILD.bazel @@ -21,6 +21,9 @@ java_binary( jvm_flags = [ "-Djava.awt.headless=true", "-Dapple.awt.UIElement=true", + "-Dio.netty.allocator.useCachedMagazinesForNonEventLoopThreads=true", + # https://github.com/netty/netty/issues/11532 + "-Dio.netty.tryReflectionSetAccessible=true", ], visibility = ["//visibility:public"], ) diff --git a/build/jvm-rules/zip/src/ZipArchiveOutputStream.kt b/build/jvm-rules/zip/src/ZipArchiveOutputStream.kt index 2878990eede5..10405fb36e03 100644 --- a/build/jvm-rules/zip/src/ZipArchiveOutputStream.kt +++ b/build/jvm-rules/zip/src/ZipArchiveOutputStream.kt @@ -2,7 +2,6 @@ package org.jetbrains.intellij.build.io import io.netty.buffer.ByteBuf -import io.netty.buffer.ByteBufAllocator import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.Companion.FLUSH_THRESHOLD import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.Companion.INITIAL_BUFFER_CAPACITY import java.io.IOException @@ -60,7 +59,7 @@ class ZipArchiveOutputStream( } private var finished = false - private val buffer = ByteBufAllocator.DEFAULT.directBuffer(INITIAL_BUFFER_CAPACITY) + private val buffer = byteBufferAllocator.directBuffer(INITIAL_BUFFER_CAPACITY) private var channelPosition = 0L @Suppress("DuplicatedCode") @@ -317,7 +316,7 @@ class ZipArchiveOutputStream( if (headerAndDataSize > INITIAL_BUFFER_CAPACITY) { // instead of resizing the current buffer, it's preferable to obtain a buffer of the required size from a pool - buffer = ByteBufAllocator.DEFAULT.directBuffer(headerAndDataSize) + buffer = byteBufferAllocator.directBuffer(headerAndDataSize) releaseBuffer = true } diff --git a/build/jvm-rules/zip/src/ZipFileWriter.kt b/build/jvm-rules/zip/src/ZipFileWriter.kt index fc85ff751f4c..16e64a5933c3 100644 --- a/build/jvm-rules/zip/src/ZipFileWriter.kt +++ b/build/jvm-rules/zip/src/ZipFileWriter.kt @@ -3,7 +3,6 @@ package org.jetbrains.intellij.build.io -import io.netty.buffer.ByteBufAllocator import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.CompressedSizeAndCrc import java.nio.ByteBuffer import java.nio.channels.FileChannel @@ -131,7 +130,7 @@ private fun compressAndWriteFile( private fun doDeflate(chunkSize: Int, deflater: Deflater, writer: (ByteBuffer) -> Unit): Int { var compressedSize = 0 - ByteBufAllocator.DEFAULT.directBuffer(chunkSize).use { nettyOutput -> + byteBufferAllocator.directBuffer(chunkSize).use { nettyOutput -> val output = nettyOutput.internalNioBuffer(nettyOutput.writerIndex(), chunkSize)!! val oldPosition = output.position() diff --git a/build/jvm-rules/zip/src/ZipIndexWriter.kt b/build/jvm-rules/zip/src/ZipIndexWriter.kt index 904f1bcc73a4..8b60538829e8 100644 --- a/build/jvm-rules/zip/src/ZipIndexWriter.kt +++ b/build/jvm-rules/zip/src/ZipIndexWriter.kt @@ -3,13 +3,12 @@ package org.jetbrains.intellij.build.io import com.dynatrace.hash4j.hashing.Hashing import io.netty.buffer.ByteBuf -import io.netty.buffer.ByteBufAllocator import java.util.zip.ZipEntry internal const val INDEX_FORMAT_VERSION: Byte = 4 class ZipIndexWriter(@JvmField val packageIndexBuilder: PackageIndexBuilder?) { - private var buffer: ByteBuf? = ByteBufAllocator.DEFAULT.directBuffer(64 * 1024) + private var buffer: ByteBuf? = byteBufferAllocator.directBuffer(64 * 1024) private var entryCount = 0 diff --git a/build/jvm-rules/zip/src/byteBuffer.kt b/build/jvm-rules/zip/src/byteBuffer.kt index a58395a15ed3..08e889db2be2 100644 --- a/build/jvm-rules/zip/src/byteBuffer.kt +++ b/build/jvm-rules/zip/src/byteBuffer.kt @@ -1,6 +1,7 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.io +import io.netty.buffer.AdaptiveByteBufAllocator import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBufAllocator import java.lang.invoke.MethodHandles @@ -8,6 +9,19 @@ import java.lang.invoke.MethodType import java.nio.ByteBuffer import java.nio.ByteOrder +val byteBufferAllocator: ByteBufAllocator = run { + System.setProperty("io.netty.tryReflectionSetAccessible", "true") + + if (System.getProperty("io.netty.allocator.useCachedMagazinesForNonEventLoopThreads") == "true" && + System.getProperty("io.netty.allocator.type", "adaptive") == "adaptive") { + val allocator = ByteBufAllocator.DEFAULT + if (allocator is AdaptiveByteBufAllocator) { + return@run allocator + } + } + AdaptiveByteBufAllocator(true, true) +} + // not thread-safe, intended only for single thread for one time use internal class ByteBufferAllocator() : AutoCloseable { private var byteBuf: ByteBuf? = null @@ -22,13 +36,9 @@ internal class ByteBufferAllocator() : AutoCloseable { } if (result == null) { - result = ByteBufAllocator.DEFAULT.directBuffer(roundUpInt(size, 65_536)) + result = byteBufferAllocator.directBuffer(roundUpInt(size, 65_536)) byteBuf = result } - else { - result.clear() - } - return result.internalNioBuffer(result.writerIndex(), size).order(ByteOrder.LITTLE_ENDIAN).also { nioByteBuffer = it } } diff --git a/build/jvm-rules/zip/src/zipReader.kt b/build/jvm-rules/zip/src/zipReader.kt index 359939323d32..1f8137937559 100644 --- a/build/jvm-rules/zip/src/zipReader.kt +++ b/build/jvm-rules/zip/src/zipReader.kt @@ -20,8 +20,13 @@ typealias EntryProcessor = (String, () -> ByteBuffer) -> Unit fun readZipFile(file: Path, entryProcessor: EntryProcessor) { // FileChannel is strongly required because only FileChannel provides `read(ByteBuffer dst, long position)` method - // ability to read data without setting channel position, as setting channel position will require synchronization - mapFileAndUse(file) { buffer, fileSize -> - readZipEntries(buffer = buffer, fileSize = fileSize, entryProcessor = entryProcessor) + try { + mapFileAndUse(file) { buffer, fileSize -> + readZipEntries(buffer = buffer, fileSize = fileSize, entryProcessor = entryProcessor) + } + } + catch (e: IOException) { + throw IOException("Cannot read $file", e) } } @@ -60,9 +65,6 @@ private inline fun mapFileAndUse(file: Path, consumer: (ByteBuffer, fileSize: In try { consumer(mappedBuffer, fileSize) } - catch (e: IOException) { - throw IOException(file.toString(), e) - } finally { if (mappedBuffer.isDirect) { // on Windows memory-mapped file cannot be deleted without clearing in-memory buffer first @@ -128,7 +130,7 @@ internal inline fun readCentralDirectory( private const val STORED: Byte = 0 private const val DEFLATED: Byte = 8 -internal fun getByteBuffer( +private fun getByteBuffer( buffer: ByteBuffer, compressedSize: Int, uncompressedSize: Int, @@ -161,10 +163,18 @@ internal fun getByteBuffer( inflater.setInput(inputBuffer) try { val result = byteBufferAllocator.allocate(uncompressedSize) + val oldPosition = result.position() while (result.hasRemaining()) { - check(inflater.inflate(result) != 0) { "Inflater wants input, but input was already set" } + val inflatedByteCount = inflater.inflate(result) + check(inflatedByteCount != 0) { + "Inflater wants input, but input was already set" + } + check(inflatedByteCount == uncompressedSize) { + "Inflater returned unexpected result: $inflatedByteCount instead of $uncompressedSize" + } } - result.rewind() + result.limit(result.position()) + result.position(oldPosition) return result } catch (e: DataFormatException) { diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityRepositoryModules.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityRepositoryModules.kt index 2193632da8a1..017390f5de38 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityRepositoryModules.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityRepositoryModules.kt @@ -696,7 +696,9 @@ private suspend fun copyAnt(pluginDir: Path, context: BuildContext): List ProjectLibraryEntry( diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/diskUsage.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/diskUsage.kt new file mode 100644 index 000000000000..ecf5a39eab3a --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/diskUsage.kt @@ -0,0 +1,204 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.intellij.build + +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap +import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap +import java.io.IOException +import java.nio.file.* +import java.nio.file.attribute.BasicFileAttributes + +internal class NoDiskSpaceLeftException(message: String, e: IOException) : RuntimeException(message, e) + +internal inline fun checkForNoDiskSpace(context: BuildContext, task: () -> T): T { + try { + return task() + } + catch (e: NoDiskSpaceLeftException) { + throw IOException(getDiskUsageDiagnostics(context.paths), e) + } +} + +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +private fun getDiskInfo(fileStore: FileStore, builder: StringBuilder) { + builder.appendLine("Disk info of ${fileStore.name()}") + builder.appendLine(" Total space: " + formatSize(fileStore.totalSpace)) + builder.appendLine(" Unallocated space: " + formatSize(fileStore.unallocatedSpace)) + builder.appendLine(" Usable space: " + formatSize(fileStore.usableSpace)) +} + +@Suppress("SpellCheckingInspection") +private fun formatSize(value: Long): String { + if (value < 1024) { + return "$value B" + } + val z = (63 - java.lang.Long.numberOfLeadingZeros(value)) / 10 + return String.format("%.1f %sB", value.toDouble() / (1L shl z * 10), " KMGTPE"[z]) +} + +private fun describe(dir: Path, builder: StringBuilder) { + if (Files.notExists(dir)) { + return + } + + builder.appendLine("Disk usage by $dir") + builder.append(getDiskInfo(Files.getFileStore(dir), builder)) + listDirectoryContent(dir, maxDepth = 3, builder) + builder.append('\n') +} + +private fun getDiskUsageDiagnostics(paths: BuildPaths): String { + val builder = StringBuilder() + describe(paths.distAllDir, builder) + if (!paths.tempDir.startsWith(paths.distAllDir)) { + describe(paths.tempDir, builder) + } + return builder.toString() +} + +internal object TestListing { + @JvmStatic + fun main(args: Array) { + val builder = StringBuilder() + listDirectoryContent(Path.of(args[0]), maxDepth = 3, result = builder) + println(builder) + } +} + +private fun listDirectoryContent(directoryPath: Path, @Suppress("SameParameterValue") maxDepth: Int = 1, result: StringBuilder): String { + if (Files.notExists(directoryPath)) { + return "Directory does not exist: $directoryPath" + } + + if (!Files.isDirectory(directoryPath)) { + return "$directoryPath is not a directory" + } + + // map to store directory paths and their sizes + val directorySizes = Object2LongOpenHashMap() + + // list to store entries for display + data class Entry( + @JvmField val path: Path, + @JvmField val size: Long, + @JvmField val depth: Int, + ) + + val entries = mutableListOf() + + Files.walkFileTree(directoryPath, object : SimpleFileVisitor() { + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { + // initialize size to 0 for all directories + directorySizes.put(dir, 0L) + + val depth = directoryPath.relativize(dir).nameCount + + // add directory to entries for display if within max depth + if (dir != directoryPath && depth <= maxDepth) { + entries.add(Entry(path = dir, size = -1, depth = depth)) + } + + // skip traversing deeper than needed for display purposes + if (depth > maxDepth) { + return FileVisitResult.SKIP_SUBTREE + } + + return FileVisitResult.CONTINUE + } + + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { + val size = attrs.size() + val depth = directoryPath.relativize(file).nameCount + + // add a file to entries for display if within max depth + if (depth <= maxDepth) { + entries.add(Entry(path = file, size = size, depth = depth)) + } + + // add size to all parent directories + var currentPath = file.parent + while (currentPath != null && currentPath.startsWith(directoryPath)) { + directorySizes.addTo(currentPath, size) + currentPath = currentPath.parent + } + + return FileVisitResult.CONTINUE + } + + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult { + val depth = directoryPath.relativize(file).nameCount + + if (depth <= maxDepth) { + entries.add(Entry(path = file, size = 0, depth = depth)) + } + + return FileVisitResult.CONTINUE + } + }) + + // format the output + result.appendLine("Content of: $directoryPath") + + // group entries by parent directory to track last items + val entriesByDepth = entries.groupByTo(Int2ObjectOpenHashMap()) { it.depth } + + // track the last entry at each depth level + val lastEntryByDepth = Int2ObjectOpenHashMap() + for (depth in 1..maxDepth) { + entriesByDepth.get(depth)?.let { entriesAtDepth -> + if (entriesAtDepth.isNotEmpty()) { + lastEntryByDepth.put(depth, entriesAtDepth.last().path) + } + } + } + + // Sort entries by path for tree-like structure + entries.sortWith(compareBy({ it.path.parent }, { -it.size }, { it.path })) + + // display entries with their sizes + for (entry in entries) { + val depth = entry.depth + val name = entry.path.fileName + val isLast = entry.path == lastEntryByDepth.get(depth) + + // Create ASCII guide indentation + val indent = StringBuilder() + for (i in 1 until depth) { + indent.append(if (lastEntryByDepth.get(i) != entry.path.subpath(0, i).toAbsolutePath()) "│ " else " ") + } + + // add connector for the current item + val connector = if (isLast) "└── " else "├── " + if (depth > 0) { + indent.append(connector) + } + + if (entry.size == -1L) { + val size = directorySizes.getLong(entry.path) + result.appendLine("$indent[DIR] $name: ${formatFileSize(size)}") + } + else { + result.appendLine("$indent$name: ${formatFileSize(entry.size)}") + } + } + // add total size + result.appendLine("Total size: ${formatFileSize(directorySizes.getLong(directoryPath))}") + + return result.toString() +} + +private fun formatFileSize(size: Long): String { + if (size < 1024) return "$size B" + + val units = arrayOf("B", "KB", "MB", "GB", "TB", "PB", "EB") + var value = size.toDouble() + var unitIndex = 0 + + while (value >= 1024 && unitIndex < units.size - 1) { + value /= 1024 + unitIndex++ + } + + return "%.2f %s".format(value, units[unitIndex]) +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildContextImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildContextImpl.kt index 3b2cc307daf8..9162ba435a03 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildContextImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildContextImpl.kt @@ -7,14 +7,41 @@ import com.intellij.platform.ijent.community.buildConstants.MULTI_ROUTING_FILE_S import com.intellij.platform.ijent.community.buildConstants.isMultiRoutingFileSystemEnabledForProduct import com.intellij.platform.runtime.product.ProductMode import com.intellij.util.containers.with -import com.intellij.util.text.SemVer import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span import kotlinx.collections.immutable.PersistentList import kotlinx.collections.immutable.persistentListOf -import kotlinx.coroutines.* -import org.jetbrains.intellij.build.* +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import org.jetbrains.intellij.build.ApplicationInfoProperties +import org.jetbrains.intellij.build.ApplicationInfoPropertiesImpl +import org.jetbrains.intellij.build.BuildContext +import org.jetbrains.intellij.build.BuildOptions +import org.jetbrains.intellij.build.BuiltinModulesFileData +import org.jetbrains.intellij.build.CompilationContext +import org.jetbrains.intellij.build.ContentModuleFilter +import org.jetbrains.intellij.build.DistFile +import org.jetbrains.intellij.build.FrontendModuleFilter +import org.jetbrains.intellij.build.JarPackagerDependencyHelper +import org.jetbrains.intellij.build.JvmArchitecture +import org.jetbrains.intellij.build.LinuxDistributionCustomizer +import org.jetbrains.intellij.build.MacDistributionCustomizer +import org.jetbrains.intellij.build.OsFamily +import org.jetbrains.intellij.build.PLATFORM_LOADER_JAR +import org.jetbrains.intellij.build.ProductProperties +import org.jetbrains.intellij.build.ProprietaryBuildTools +import org.jetbrains.intellij.build.Source +import org.jetbrains.intellij.build.WindowsDistributionCustomizer +import org.jetbrains.intellij.build.buildJar +import org.jetbrains.intellij.build.checkForNoDiskSpace +import org.jetbrains.intellij.build.computeAppInfoXml import org.jetbrains.intellij.build.impl.PlatformJarNames.PLATFORM_CORE_NIO_FS import org.jetbrains.intellij.build.impl.plugins.PluginAutoPublishList import org.jetbrains.intellij.build.io.runProcess @@ -420,7 +447,9 @@ class BuildContextImpl internal constructor( } override suspend fun produce(targetFile: Path) { - buildJar(targetFile = targetFile, sources = sources, compress = compress, notify = false) + checkForNoDiskSpace(this@BuildContextImpl) { + buildJar(targetFile = targetFile, sources = sources, compress = compress, notify = false) + } } }, ) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt index 238fa6bc2517..7b06d0aa4dc1 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt @@ -32,6 +32,7 @@ import org.jetbrains.intellij.build.UTIL_8_JAR import org.jetbrains.intellij.build.UTIL_JAR import org.jetbrains.intellij.build.ZipSource import org.jetbrains.intellij.build.buildJar +import org.jetbrains.intellij.build.checkForNoDiskSpace import org.jetbrains.intellij.build.computeHashForModuleOutput import org.jetbrains.intellij.build.computeModuleSourcesByContent import org.jetbrains.intellij.build.defaultLibrarySourcesNamesFilter @@ -911,7 +912,7 @@ private suspend fun buildJars( sources = sources, nativeFileHandler = nativeFileHandler, notify = false, - addDirEntries = asset.includedModules.any { helper.isTestPluginModule(it.key.moduleName, null) }, + addDirEntries = asset.includedModules.any { helper.isTestPluginModule(moduleName = it.key.moduleName, module = null) }, ) } } @@ -995,14 +996,16 @@ suspend fun buildJar(targetFile: Path, moduleNames: List, context: Build return } - buildJar( - targetFile = targetFile, - sources = moduleNames.mapNotNull { moduleName -> - val module = context.findRequiredModule(moduleName) - val output = context.getModuleOutputDir(module) - toSource(module = module, outputDir = output, excludes = commonModuleExcludes) - }, - ) + checkForNoDiskSpace(context) { + buildJar( + targetFile = targetFile, + sources = moduleNames.mapNotNull { moduleName -> + val module = context.findRequiredModule(moduleName) + val output = context.getModuleOutputDir(module) + toSource(module = module, outputDir = output, excludes = commonModuleExcludes) + }, + ) + } } private fun toSource(module: JpsModule, outputDir: Path, excludes: List): Source? { diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sign.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sign.kt index 1139660600b6..70e390b313ec 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sign.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sign.kt @@ -20,7 +20,6 @@ import kotlinx.collections.immutable.persistentMapOf import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.jetbrains.intellij.build.BuildContext @@ -50,15 +49,18 @@ import kotlin.io.path.extension import kotlin.io.path.name import kotlin.io.path.relativeTo -internal fun isMacLibrary(name: String): Boolean = - name.endsWith(".jnilib") || - name.endsWith(".dylib") || - name.endsWith(".so") || - name.endsWith(".tbd") +internal fun isMacLibrary(name: String): Boolean { + return name.endsWith(".jnilib") || + name.endsWith(".dylib") || + name.endsWith(".so") || + name.endsWith(".tbd") +} -internal fun CoroutineScope.recursivelySignMacBinaries(root: Path, - context: BuildContext, - executableFileMatchers: Collection = emptyList()) { +internal fun CoroutineScope.recursivelySignMacBinaries( + root: Path, + context: BuildContext, + executableFileMatchers: Collection = emptyList(), +) { val archives = mutableListOf() val binaries = mutableListOf() @@ -92,7 +94,7 @@ internal fun CoroutineScope.recursivelySignMacBinaries(root: Path, } private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context: BuildContext) { - val filesToBeSigned = mutableMapOf() + val filesToBeSigned = LinkedHashMap() suspendAwareReadZipFile(zip) { name, dataSupplier -> if (!isMacLibrary(name)) { return@suspendAwareReadZipFile @@ -118,9 +120,7 @@ private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context return } - coroutineScope { - signMacBinaries(files = filesToBeSigned.values.toList(), context = context, checkPermissions = false) - } + signMacBinaries(files = filesToBeSigned.values.toList(), context = context, checkPermissions = false) copyZipReplacing(origin = zip, entries = filesToBeSigned, context = context) for (file in filesToBeSigned.values) { @@ -167,10 +167,12 @@ internal fun signingOptions(contentType: String, context: BuildContext): Persist ) } -internal suspend fun signMacBinaries(files: List, - context: BuildContext, - checkPermissions: Boolean = true, - additionalOptions: Map = emptyMap()) { +internal suspend fun signMacBinaries( + files: List, + context: BuildContext, + checkPermissions: Boolean = true, + additionalOptions: Map = emptyMap(), +) { if (files.isEmpty() || !context.isMacCodeSignEnabled) { return } @@ -227,24 +229,21 @@ internal suspend fun isSigned(path: Path): Boolean { } } -internal fun isMacBinary(byteChannel: SeekableByteChannel): Boolean = - detectFileType(byteChannel).first == FileType.MachO +internal fun isMacBinary(byteChannel: SeekableByteChannel): Boolean { + return detectFileType(byteChannel).first == FileType.MachO +} -private fun detectFileType(byteChannel: SeekableByteChannel): Pair> = - byteChannel.use { +private fun detectFileType(byteChannel: SeekableByteChannel): Pair> { + return byteChannel.use { it.DetectFileType() } +} /** * Assumes [isMacBinary]. */ internal suspend fun isSigned(byteChannel: SeekableByteChannel, binaryId: String): Boolean { - val verificationParams = SignatureVerificationParams( - signRootCertStore = null, - timestampRootCertStore = null, - buildChain = false, - withRevocationCheck = false - ) + val verificationParams = SignatureVerificationParams(signRootCertStore = null, timestampRootCertStore = null, buildChain = false, withRevocationCheck = false) val binaries = MachoArch(byteChannel).Extract() return binaries.all { binary -> val signatureData = try { @@ -278,7 +277,8 @@ internal suspend fun isSigned(byteChannel: SeekableByteChannel, binaryId: String } private class SignatureVerificationLog(val binaryId: String) : ILogger { - val span: Span = Span.current() + private val span: Span = Span.current() + fun addEvent(str: String, category: String) { span.addEvent(str) .setAttribute("binary", binaryId) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/jarCache/LocalDiskJarCacheManager.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/jarCache/LocalDiskJarCacheManager.kt index bb74f8776408..e1162d84a205 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/jarCache/LocalDiskJarCacheManager.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/jarCache/LocalDiskJarCacheManager.kt @@ -131,7 +131,7 @@ internal class LocalDiskJarCacheManager( if (!producer.useCacheAsTargetFile) { Files.createDirectories(targetFile.parent) - Files.copy(cacheFile, targetFile) + Files.createLink(targetFile, cacheFile) } Files.write(cacheMetadataFile, ProtoBuf.encodeToByteArray(JarCacheItem(sources = sourceCacheItems))) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/mergeJars.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/mergeJars.kt index 8b50fd187ed7..808c604394cd 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/mergeJars.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/mergeJars.kt @@ -16,12 +16,12 @@ import org.jetbrains.intellij.build.io.ZipFileWriter import org.jetbrains.intellij.build.io.archiveDir import org.jetbrains.intellij.build.io.suspendAwareReadZipFile import org.jetbrains.intellij.build.io.zipWriter +import java.io.IOException import java.nio.ByteBuffer import java.nio.file.Files import java.nio.file.Path import java.nio.file.PathMatcher import java.util.zip.Deflater -import kotlin.io.path.name private const val listOfEntitiesFileName = "META-INF/listOfEntities.txt" @@ -159,6 +159,14 @@ private suspend fun writeSource( filesToMerge = filesToMerge, ) } + catch (e: IOException) { + if (e.message?.contains("No space left on device") == true) { + throw NoDiskSpaceLeftException("No space left while including $sourceFile into $targetFile", e) + } + else { + throw IOException("Failed to include $sourceFile to $targetFile", e) + } + } finally { @Suppress("KotlinConstantConditions") if (sourceFile !== source.file) { @@ -209,8 +217,6 @@ private suspend fun handleZipSource( } } - // FileChannel is strongly required because only FileChannel provides `read(ByteBuffer dst, long position)` method - - // ability to read data without setting channel position, as setting channel position will require synchronization suspendAwareReadZipFile(sourceFile) { name, dataSupplier -> if (name == listOfEntitiesFileName) { filesToMerge.add(Charsets.UTF_8.decode(dataSupplier())) @@ -289,12 +295,12 @@ private fun checkCoverageAgentManifest( } val coveragePlatformAgentModuleName = "intellij.platform.coverage.agent" - if (!targetFile.name.contains(coveragePlatformAgentModuleName)) { + if (!targetFile.fileName.toString().contains(coveragePlatformAgentModuleName)) { return false } val agentPrefix = "intellij-coverage-agent" - if (!sourceFile.name.startsWith(agentPrefix)) { + if (!sourceFile.fileName.toString().startsWith(agentPrefix)) { return false } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/productRunner/DevModeProductRunner.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/productRunner/DevModeProductRunner.kt index efa603ec9247..fe093f9b2474 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/productRunner/DevModeProductRunner.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/productRunner/DevModeProductRunner.kt @@ -7,6 +7,7 @@ import com.intellij.platform.ijent.community.buildConstants.IJENT_BOOT_CLASSPATH import com.intellij.platform.ijent.community.buildConstants.isMultiRoutingFileSystemEnabledForProduct import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.VmProperties +import org.jetbrains.intellij.build.checkForNoDiskSpace import org.jetbrains.intellij.build.dev.BuildRequest import org.jetbrains.intellij.build.dev.buildProduct import org.jetbrains.intellij.build.dev.getIdeSystemProperties @@ -20,26 +21,28 @@ import kotlin.time.Duration internal suspend fun createDevModeProductRunner(context: BuildContext, additionalPluginModules: List = emptyList()): IntellijProductRunner { var newClassPath: Collection? = null val homeDir = context.paths.projectHome - val runDir = buildProduct( - request = BuildRequest( - //isUnpackedDist = context.productProperties.platformPrefix != "Gateway", - // https://youtrack.jetbrains.com/issue/IJPL-156115/devModeProductRunner-use-packed-dist-as-a-workaround-for-incorrect-product-info.json-entries-links-to-compilation-output - isUnpackedDist = false, - writeCoreClasspath = false, - platformPrefix = context.productProperties.platformPrefix ?: "idea", - additionalModules = additionalPluginModules, - projectDir = homeDir, - devRootDir = context.paths.tempDir.resolve("dev-run"), - jarCacheDir = homeDir.resolve("out/dev-run/jar-cache"), - productionClassOutput = context.classesOutputDirectory.resolve("production"), - platformClassPathConsumer = { _, classPath, _ -> - newClassPath = classPath - }, - buildOptionsTemplate = context.options, - ), - createProductProperties = { context.productProperties } - ) - return DevModeProductRunner(context = context, homePath = runDir, classPath = newClassPath!!.map { it.toString() }) + return checkForNoDiskSpace(context) { + val runDir = buildProduct( + request = BuildRequest( + //isUnpackedDist = context.productProperties.platformPrefix != "Gateway", + // https://youtrack.jetbrains.com/issue/IJPL-156115/devModeProductRunner-use-packed-dist-as-a-workaround-for-incorrect-product-info.json-entries-links-to-compilation-output + isUnpackedDist = false, + writeCoreClasspath = false, + platformPrefix = context.productProperties.platformPrefix ?: "idea", + additionalModules = additionalPluginModules, + projectDir = homeDir, + devRootDir = context.paths.tempDir.resolve("dev-run"), + jarCacheDir = homeDir.resolve("out/dev-run/jar-cache"), + productionClassOutput = context.classesOutputDirectory.resolve("production"), + platformClassPathConsumer = { _, classPath, _ -> + newClassPath = classPath + }, + buildOptionsTemplate = context.options, + ), + createProductProperties = { context.productProperties } + ) + DevModeProductRunner(context = context, homePath = runDir, classPath = newClassPath!!.map { it.toString() }) + } } private class DevModeProductRunner( diff --git a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/SlowZipTest.kt b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/SlowZipTest.kt new file mode 100644 index 000000000000..d50416f94bb0 --- /dev/null +++ b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/SlowZipTest.kt @@ -0,0 +1,24 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build + +import com.intellij.openapi.util.SystemInfoRt +import kotlinx.coroutines.runBlocking +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class SlowZipTest { + @Test + fun `read zip file with more than 65K entries`(@TempDir tempDir: Path) = runBlocking { + assumeTrue(SystemInfoRt.isUnix) + + val (list, archiveFile) = createLargeArchive(Short.MAX_VALUE * 2 + 20, tempDir) + checkZip(archiveFile) { zipFile -> + for (name in list) { + assertThat(zipFile.getResource(name)).isNotNull() + } + } + } +} \ No newline at end of file diff --git a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/ZipTest.kt b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/ZipTest.kt index 8eb7fa38f3f9..7e1f9ba5998a 100644 --- a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/ZipTest.kt +++ b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/ZipTest.kt @@ -1,7 +1,6 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build -import com.intellij.openapi.util.SystemInfoRt import com.intellij.util.io.toByteArray import com.intellij.util.io.write import com.intellij.util.lang.HashMapZipFile @@ -21,11 +20,11 @@ import org.jetbrains.intellij.build.io.PackageIndexBuilder import org.jetbrains.intellij.build.io.ZipArchiveOutputStream import org.jetbrains.intellij.build.io.ZipIndexWriter import org.jetbrains.intellij.build.io.compressedData +import org.jetbrains.intellij.build.io.readZipFile import org.jetbrains.intellij.build.io.zip import org.jetbrains.intellij.build.io.zipWithCompression import org.jetbrains.intellij.build.io.zipWithPackageIndex import org.jetbrains.intellij.build.io.zipWriter -import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import java.nio.ByteBuffer @@ -40,7 +39,6 @@ import java.util.zip.ZipEntry import java.util.zip.ZipInputStream import kotlin.random.Random - class ZipTest { @Test fun `interrupt thread`(@TempDir tempDir: Path) = runBlocking { @@ -70,18 +68,6 @@ class ZipTest { } } - @Test - fun `read zip file with more than 65K entries`(@TempDir tempDir: Path) = runBlocking { - assumeTrue(SystemInfoRt.isUnix) - - val (list, archiveFile) = createLargeArchive(Short.MAX_VALUE * 2 + 20, tempDir) - checkZip(archiveFile) { zipFile -> - for (name in list) { - assertThat(zipFile.getResource(name)).isNotNull() - } - } - } - @Test fun `custom prefix`(@TempDir tempDir: Path) { val random = Random(42) @@ -488,6 +474,14 @@ class ZipTest { val archiveFile = tempDir.resolve("archive.zip") zipWithCompression(archiveFile, mapOf(dir to "")) + readZipFile(archiveFile) { name, dataProvider -> + for (item in list) { + if (name == item.name) { + assertThat(dataProvider().remaining()).isEqualTo(item.size.toLong()) + } + } + } + java.util.zip.ZipFile(archiveFile.toFile()).use { jdkZipFile -> for (item in list) { val entry = jdkZipFile.getEntry(item.name) @@ -600,6 +594,9 @@ class ZipTest { // check both IKV- and non-IKV variants of an immutable zip file internal fun checkZip(file: Path, checker: (ZipFile) -> Unit) { + readZipFile(file) { name, dataProvider -> + dataProvider() + } HashMapZipFile.load(file).use { zipFile -> checker(zipFile) } @@ -678,7 +675,7 @@ internal class TestEntryItem( @JvmField val name: String, ) -private suspend fun createLargeArchive(size: Int, tempDir: Path, minFileSize: Int = 0, maxFileSize: Int = 32): Pair, Path> { +internal suspend fun createLargeArchive(size: Int, tempDir: Path, minFileSize: Int = 0, maxFileSize: Int = 32): Pair, Path> { val (dir, list) = createDirOnDisk(tempDir, size, minFileSize, maxFileSize) val archiveFile = tempDir.resolve("archive.zip") zipWithPackageIndex(archiveFile, dir) diff --git a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt index 8e173903de03..9cd9a45c6e22 100644 --- a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt +++ b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl.maven import com.intellij.testFramework.utils.io.createFile @@ -13,8 +13,8 @@ import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files import java.nio.file.Path -import kotlin.io.path.exists import kotlin.io.path.name import kotlin.io.path.writeText @@ -73,7 +73,7 @@ class MavenCentralPublicationTest { val files = createDistributionFiles().map { "${coordinates.directoryPath}/${it.name}" } publication.execute() val bundle = workDir.resolve("bundle.zip") - assert(bundle.exists()) + assert(Files.exists(bundle)) {} val entries = buildList { suspendAwareReadZipFile(bundle) { entry, _ -> add(entry)