investigate "No space left on device" - add more diagnostic (as we have for integration tests)

GitOrigin-RevId: f0e60bc7e729928a41bd108d15a2291ddc5c04f2
This commit is contained in:
Vladimir Krivosheev
2025-04-02 10:54:51 +00:00
committed by intellij-monorepo-bot
parent 463d04c15d
commit 248d320d6e
18 changed files with 396 additions and 107 deletions
+1
View File
@@ -17,6 +17,7 @@ def get_jvm_flags(flags):
# kotlin compiler
"-Dkotlin.environment.keepalive=true",
"-Didea.io.use.nio2=true",
"-Dio.netty.allocator.useCachedMagazinesForNonEventLoopThreads=true",
# https://github.com/netty/netty/issues/11532
"-Dio.netty.tryReflectionSetAccessible=true",
# see TargetConfigurationDigestProperty.KOTLIN_VERSION - we invalidate cache if kotlinc version changed
+3
View File
@@ -21,6 +21,9 @@ java_binary(
jvm_flags = [
"-Djava.awt.headless=true",
"-Dapple.awt.UIElement=true",
"-Dio.netty.allocator.useCachedMagazinesForNonEventLoopThreads=true",
# https://github.com/netty/netty/issues/11532
"-Dio.netty.tryReflectionSetAccessible=true",
],
visibility = ["//visibility:public"],
)
@@ -2,7 +2,6 @@
package org.jetbrains.intellij.build.io
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.Companion.FLUSH_THRESHOLD
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.Companion.INITIAL_BUFFER_CAPACITY
import java.io.IOException
@@ -60,7 +59,7 @@ class ZipArchiveOutputStream(
}
private var finished = false
private val buffer = ByteBufAllocator.DEFAULT.directBuffer(INITIAL_BUFFER_CAPACITY)
private val buffer = byteBufferAllocator.directBuffer(INITIAL_BUFFER_CAPACITY)
private var channelPosition = 0L
@Suppress("DuplicatedCode")
@@ -317,7 +316,7 @@ class ZipArchiveOutputStream(
if (headerAndDataSize > INITIAL_BUFFER_CAPACITY) {
// instead of resizing the current buffer, it's preferable to obtain a buffer of the required size from a pool
buffer = ByteBufAllocator.DEFAULT.directBuffer(headerAndDataSize)
buffer = byteBufferAllocator.directBuffer(headerAndDataSize)
releaseBuffer = true
}
+1 -2
View File
@@ -3,7 +3,6 @@
package org.jetbrains.intellij.build.io
import io.netty.buffer.ByteBufAllocator
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.CompressedSizeAndCrc
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
@@ -131,7 +130,7 @@ private fun compressAndWriteFile(
private fun doDeflate(chunkSize: Int, deflater: Deflater, writer: (ByteBuffer) -> Unit): Int {
var compressedSize = 0
ByteBufAllocator.DEFAULT.directBuffer(chunkSize).use { nettyOutput ->
byteBufferAllocator.directBuffer(chunkSize).use { nettyOutput ->
val output = nettyOutput.internalNioBuffer(nettyOutput.writerIndex(), chunkSize)!!
val oldPosition = output.position()
+1 -2
View File
@@ -3,13 +3,12 @@ package org.jetbrains.intellij.build.io
import com.dynatrace.hash4j.hashing.Hashing
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import java.util.zip.ZipEntry
internal const val INDEX_FORMAT_VERSION: Byte = 4
class ZipIndexWriter(@JvmField val packageIndexBuilder: PackageIndexBuilder?) {
private var buffer: ByteBuf? = ByteBufAllocator.DEFAULT.directBuffer(64 * 1024)
private var buffer: ByteBuf? = byteBufferAllocator.directBuffer(64 * 1024)
private var entryCount = 0
+15 -5
View File
@@ -1,6 +1,7 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
import io.netty.buffer.AdaptiveByteBufAllocator
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import java.lang.invoke.MethodHandles
@@ -8,6 +9,19 @@ import java.lang.invoke.MethodType
import java.nio.ByteBuffer
import java.nio.ByteOrder
val byteBufferAllocator: ByteBufAllocator = run {
System.setProperty("io.netty.tryReflectionSetAccessible", "true")
if (System.getProperty("io.netty.allocator.useCachedMagazinesForNonEventLoopThreads") == "true" &&
System.getProperty("io.netty.allocator.type", "adaptive") == "adaptive") {
val allocator = ByteBufAllocator.DEFAULT
if (allocator is AdaptiveByteBufAllocator) {
return@run allocator
}
}
AdaptiveByteBufAllocator(true, true)
}
// not thread-safe, intended only for single thread for one time use
internal class ByteBufferAllocator() : AutoCloseable {
private var byteBuf: ByteBuf? = null
@@ -22,13 +36,9 @@ internal class ByteBufferAllocator() : AutoCloseable {
}
if (result == null) {
result = ByteBufAllocator.DEFAULT.directBuffer(roundUpInt(size, 65_536))
result = byteBufferAllocator.directBuffer(roundUpInt(size, 65_536))
byteBuf = result
}
else {
result.clear()
}
return result.internalNioBuffer(result.writerIndex(), size).order(ByteOrder.LITTLE_ENDIAN).also { nioByteBuffer = it }
}
+18 -8
View File
@@ -20,8 +20,13 @@ typealias EntryProcessor = (String, () -> ByteBuffer) -> Unit
fun readZipFile(file: Path, entryProcessor: EntryProcessor) {
// FileChannel is strongly required because only FileChannel provides `read(ByteBuffer dst, long position)` method -
// ability to read data without setting channel position, as setting channel position will require synchronization
mapFileAndUse(file) { buffer, fileSize ->
readZipEntries(buffer = buffer, fileSize = fileSize, entryProcessor = entryProcessor)
try {
mapFileAndUse(file) { buffer, fileSize ->
readZipEntries(buffer = buffer, fileSize = fileSize, entryProcessor = entryProcessor)
}
}
catch (e: IOException) {
throw IOException("Cannot read $file", e)
}
}
@@ -60,9 +65,6 @@ private inline fun mapFileAndUse(file: Path, consumer: (ByteBuffer, fileSize: In
try {
consumer(mappedBuffer, fileSize)
}
catch (e: IOException) {
throw IOException(file.toString(), e)
}
finally {
if (mappedBuffer.isDirect) {
// on Windows memory-mapped file cannot be deleted without clearing in-memory buffer first
@@ -128,7 +130,7 @@ internal inline fun readCentralDirectory(
private const val STORED: Byte = 0
private const val DEFLATED: Byte = 8
internal fun getByteBuffer(
private fun getByteBuffer(
buffer: ByteBuffer,
compressedSize: Int,
uncompressedSize: Int,
@@ -161,10 +163,18 @@ internal fun getByteBuffer(
inflater.setInput(inputBuffer)
try {
val result = byteBufferAllocator.allocate(uncompressedSize)
val oldPosition = result.position()
while (result.hasRemaining()) {
check(inflater.inflate(result) != 0) { "Inflater wants input, but input was already set" }
val inflatedByteCount = inflater.inflate(result)
check(inflatedByteCount != 0) {
"Inflater wants input, but input was already set"
}
check(inflatedByteCount == uncompressedSize) {
"Inflater returned unexpected result: $inflatedByteCount instead of $uncompressedSize"
}
}
result.rewind()
result.limit(result.position())
result.position(oldPosition)
return result
}
catch (e: DataFormatException) {
@@ -696,7 +696,9 @@ private suspend fun copyAnt(pluginDir: Path, context: BuildContext): List<Distri
sources.sort()
val antTargetFile = antDir.resolve("ant.jar")
buildJar(targetFile = antTargetFile, sources = sources)
checkForNoDiskSpace(context) {
buildJar(targetFile = antTargetFile, sources = sources)
}
sources.map { source ->
ProjectLibraryEntry(
@@ -0,0 +1,204 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.intellij.build
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap
import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap
import java.io.IOException
import java.nio.file.*
import java.nio.file.attribute.BasicFileAttributes
internal class NoDiskSpaceLeftException(message: String, e: IOException) : RuntimeException(message, e)
internal inline fun <T> checkForNoDiskSpace(context: BuildContext, task: () -> T): T {
try {
return task()
}
catch (e: NoDiskSpaceLeftException) {
throw IOException(getDiskUsageDiagnostics(context.paths), e)
}
}
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
private fun getDiskInfo(fileStore: FileStore, builder: StringBuilder) {
builder.appendLine("Disk info of ${fileStore.name()}")
builder.appendLine(" Total space: " + formatSize(fileStore.totalSpace))
builder.appendLine(" Unallocated space: " + formatSize(fileStore.unallocatedSpace))
builder.appendLine(" Usable space: " + formatSize(fileStore.usableSpace))
}
@Suppress("SpellCheckingInspection")
private fun formatSize(value: Long): String {
if (value < 1024) {
return "$value B"
}
val z = (63 - java.lang.Long.numberOfLeadingZeros(value)) / 10
return String.format("%.1f %sB", value.toDouble() / (1L shl z * 10), " KMGTPE"[z])
}
private fun describe(dir: Path, builder: StringBuilder) {
if (Files.notExists(dir)) {
return
}
builder.appendLine("Disk usage by $dir")
builder.append(getDiskInfo(Files.getFileStore(dir), builder))
listDirectoryContent(dir, maxDepth = 3, builder)
builder.append('\n')
}
private fun getDiskUsageDiagnostics(paths: BuildPaths): String {
val builder = StringBuilder()
describe(paths.distAllDir, builder)
if (!paths.tempDir.startsWith(paths.distAllDir)) {
describe(paths.tempDir, builder)
}
return builder.toString()
}
internal object TestListing {
@JvmStatic
fun main(args: Array<String>) {
val builder = StringBuilder()
listDirectoryContent(Path.of(args[0]), maxDepth = 3, result = builder)
println(builder)
}
}
private fun listDirectoryContent(directoryPath: Path, @Suppress("SameParameterValue") maxDepth: Int = 1, result: StringBuilder): String {
if (Files.notExists(directoryPath)) {
return "Directory does not exist: $directoryPath"
}
if (!Files.isDirectory(directoryPath)) {
return "$directoryPath is not a directory"
}
// map to store directory paths and their sizes
val directorySizes = Object2LongOpenHashMap<Path>()
// list to store entries for display
data class Entry(
@JvmField val path: Path,
@JvmField val size: Long,
@JvmField val depth: Int,
)
val entries = mutableListOf<Entry>()
Files.walkFileTree(directoryPath, object : SimpleFileVisitor<Path>() {
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
// initialize size to 0 for all directories
directorySizes.put(dir, 0L)
val depth = directoryPath.relativize(dir).nameCount
// add directory to entries for display if within max depth
if (dir != directoryPath && depth <= maxDepth) {
entries.add(Entry(path = dir, size = -1, depth = depth))
}
// skip traversing deeper than needed for display purposes
if (depth > maxDepth) {
return FileVisitResult.SKIP_SUBTREE
}
return FileVisitResult.CONTINUE
}
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
val size = attrs.size()
val depth = directoryPath.relativize(file).nameCount
// add a file to entries for display if within max depth
if (depth <= maxDepth) {
entries.add(Entry(path = file, size = size, depth = depth))
}
// add size to all parent directories
var currentPath = file.parent
while (currentPath != null && currentPath.startsWith(directoryPath)) {
directorySizes.addTo(currentPath, size)
currentPath = currentPath.parent
}
return FileVisitResult.CONTINUE
}
override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult {
val depth = directoryPath.relativize(file).nameCount
if (depth <= maxDepth) {
entries.add(Entry(path = file, size = 0, depth = depth))
}
return FileVisitResult.CONTINUE
}
})
// format the output
result.appendLine("Content of: $directoryPath")
// group entries by parent directory to track last items
val entriesByDepth = entries.groupByTo(Int2ObjectOpenHashMap()) { it.depth }
// track the last entry at each depth level
val lastEntryByDepth = Int2ObjectOpenHashMap<Path>()
for (depth in 1..maxDepth) {
entriesByDepth.get(depth)?.let { entriesAtDepth ->
if (entriesAtDepth.isNotEmpty()) {
lastEntryByDepth.put(depth, entriesAtDepth.last().path)
}
}
}
// Sort entries by path for tree-like structure
entries.sortWith(compareBy({ it.path.parent }, { -it.size }, { it.path }))
// display entries with their sizes
for (entry in entries) {
val depth = entry.depth
val name = entry.path.fileName
val isLast = entry.path == lastEntryByDepth.get(depth)
// Create ASCII guide indentation
val indent = StringBuilder()
for (i in 1 until depth) {
indent.append(if (lastEntryByDepth.get(i) != entry.path.subpath(0, i).toAbsolutePath()) "│ " else " ")
}
// add connector for the current item
val connector = if (isLast) "└── " else "├── "
if (depth > 0) {
indent.append(connector)
}
if (entry.size == -1L) {
val size = directorySizes.getLong(entry.path)
result.appendLine("$indent[DIR] $name: ${formatFileSize(size)}")
}
else {
result.appendLine("$indent$name: ${formatFileSize(entry.size)}")
}
}
// add total size
result.appendLine("Total size: ${formatFileSize(directorySizes.getLong(directoryPath))}")
return result.toString()
}
private fun formatFileSize(size: Long): String {
if (size < 1024) return "$size B"
val units = arrayOf("B", "KB", "MB", "GB", "TB", "PB", "EB")
var value = size.toDouble()
var unitIndex = 0
while (value >= 1024 && unitIndex < units.size - 1) {
value /= 1024
unitIndex++
}
return "%.2f %s".format(value, units[unitIndex])
}
@@ -7,14 +7,41 @@ import com.intellij.platform.ijent.community.buildConstants.MULTI_ROUTING_FILE_S
import com.intellij.platform.ijent.community.buildConstants.isMultiRoutingFileSystemEnabledForProduct
import com.intellij.platform.runtime.product.ProductMode
import com.intellij.util.containers.with
import com.intellij.util.text.SemVer
import io.opentelemetry.api.common.AttributeKey
import io.opentelemetry.api.common.Attributes
import io.opentelemetry.api.trace.Span
import kotlinx.collections.immutable.PersistentList
import kotlinx.collections.immutable.persistentListOf
import kotlinx.coroutines.*
import org.jetbrains.intellij.build.*
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import org.jetbrains.intellij.build.ApplicationInfoProperties
import org.jetbrains.intellij.build.ApplicationInfoPropertiesImpl
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.BuiltinModulesFileData
import org.jetbrains.intellij.build.CompilationContext
import org.jetbrains.intellij.build.ContentModuleFilter
import org.jetbrains.intellij.build.DistFile
import org.jetbrains.intellij.build.FrontendModuleFilter
import org.jetbrains.intellij.build.JarPackagerDependencyHelper
import org.jetbrains.intellij.build.JvmArchitecture
import org.jetbrains.intellij.build.LinuxDistributionCustomizer
import org.jetbrains.intellij.build.MacDistributionCustomizer
import org.jetbrains.intellij.build.OsFamily
import org.jetbrains.intellij.build.PLATFORM_LOADER_JAR
import org.jetbrains.intellij.build.ProductProperties
import org.jetbrains.intellij.build.ProprietaryBuildTools
import org.jetbrains.intellij.build.Source
import org.jetbrains.intellij.build.WindowsDistributionCustomizer
import org.jetbrains.intellij.build.buildJar
import org.jetbrains.intellij.build.checkForNoDiskSpace
import org.jetbrains.intellij.build.computeAppInfoXml
import org.jetbrains.intellij.build.impl.PlatformJarNames.PLATFORM_CORE_NIO_FS
import org.jetbrains.intellij.build.impl.plugins.PluginAutoPublishList
import org.jetbrains.intellij.build.io.runProcess
@@ -420,7 +447,9 @@ class BuildContextImpl internal constructor(
}
override suspend fun produce(targetFile: Path) {
buildJar(targetFile = targetFile, sources = sources, compress = compress, notify = false)
checkForNoDiskSpace(this@BuildContextImpl) {
buildJar(targetFile = targetFile, sources = sources, compress = compress, notify = false)
}
}
},
)
@@ -32,6 +32,7 @@ import org.jetbrains.intellij.build.UTIL_8_JAR
import org.jetbrains.intellij.build.UTIL_JAR
import org.jetbrains.intellij.build.ZipSource
import org.jetbrains.intellij.build.buildJar
import org.jetbrains.intellij.build.checkForNoDiskSpace
import org.jetbrains.intellij.build.computeHashForModuleOutput
import org.jetbrains.intellij.build.computeModuleSourcesByContent
import org.jetbrains.intellij.build.defaultLibrarySourcesNamesFilter
@@ -911,7 +912,7 @@ private suspend fun buildJars(
sources = sources,
nativeFileHandler = nativeFileHandler,
notify = false,
addDirEntries = asset.includedModules.any { helper.isTestPluginModule(it.key.moduleName, null) },
addDirEntries = asset.includedModules.any { helper.isTestPluginModule(moduleName = it.key.moduleName, module = null) },
)
}
}
@@ -995,14 +996,16 @@ suspend fun buildJar(targetFile: Path, moduleNames: List<String>, context: Build
return
}
buildJar(
targetFile = targetFile,
sources = moduleNames.mapNotNull { moduleName ->
val module = context.findRequiredModule(moduleName)
val output = context.getModuleOutputDir(module)
toSource(module = module, outputDir = output, excludes = commonModuleExcludes)
},
)
checkForNoDiskSpace(context) {
buildJar(
targetFile = targetFile,
sources = moduleNames.mapNotNull { moduleName ->
val module = context.findRequiredModule(moduleName)
val output = context.getModuleOutputDir(module)
toSource(module = module, outputDir = output, excludes = commonModuleExcludes)
},
)
}
}
private fun toSource(module: JpsModule, outputDir: Path, excludes: List<PathMatcher>): Source? {
@@ -20,7 +20,6 @@ import kotlinx.collections.immutable.persistentMapOf
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.BuildContext
@@ -50,15 +49,18 @@ import kotlin.io.path.extension
import kotlin.io.path.name
import kotlin.io.path.relativeTo
internal fun isMacLibrary(name: String): Boolean =
name.endsWith(".jnilib") ||
name.endsWith(".dylib") ||
name.endsWith(".so") ||
name.endsWith(".tbd")
internal fun isMacLibrary(name: String): Boolean {
return name.endsWith(".jnilib") ||
name.endsWith(".dylib") ||
name.endsWith(".so") ||
name.endsWith(".tbd")
}
internal fun CoroutineScope.recursivelySignMacBinaries(root: Path,
context: BuildContext,
executableFileMatchers: Collection<PathMatcher> = emptyList()) {
internal fun CoroutineScope.recursivelySignMacBinaries(
root: Path,
context: BuildContext,
executableFileMatchers: Collection<PathMatcher> = emptyList(),
) {
val archives = mutableListOf<Path>()
val binaries = mutableListOf<Path>()
@@ -92,7 +94,7 @@ internal fun CoroutineScope.recursivelySignMacBinaries(root: Path,
}
private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context: BuildContext) {
val filesToBeSigned = mutableMapOf<String, Path>()
val filesToBeSigned = LinkedHashMap<String, Path>()
suspendAwareReadZipFile(zip) { name, dataSupplier ->
if (!isMacLibrary(name)) {
return@suspendAwareReadZipFile
@@ -118,9 +120,7 @@ private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context
return
}
coroutineScope {
signMacBinaries(files = filesToBeSigned.values.toList(), context = context, checkPermissions = false)
}
signMacBinaries(files = filesToBeSigned.values.toList(), context = context, checkPermissions = false)
copyZipReplacing(origin = zip, entries = filesToBeSigned, context = context)
for (file in filesToBeSigned.values) {
@@ -167,10 +167,12 @@ internal fun signingOptions(contentType: String, context: BuildContext): Persist
)
}
internal suspend fun signMacBinaries(files: List<Path>,
context: BuildContext,
checkPermissions: Boolean = true,
additionalOptions: Map<String, String> = emptyMap()) {
internal suspend fun signMacBinaries(
files: List<Path>,
context: BuildContext,
checkPermissions: Boolean = true,
additionalOptions: Map<String, String> = emptyMap(),
) {
if (files.isEmpty() || !context.isMacCodeSignEnabled) {
return
}
@@ -227,24 +229,21 @@ internal suspend fun isSigned(path: Path): Boolean {
}
}
internal fun isMacBinary(byteChannel: SeekableByteChannel): Boolean =
detectFileType(byteChannel).first == FileType.MachO
internal fun isMacBinary(byteChannel: SeekableByteChannel): Boolean {
return detectFileType(byteChannel).first == FileType.MachO
}
private fun detectFileType(byteChannel: SeekableByteChannel): Pair<FileType, EnumSet<FileProperties>> =
byteChannel.use {
private fun detectFileType(byteChannel: SeekableByteChannel): Pair<FileType, EnumSet<FileProperties>> {
return byteChannel.use {
it.DetectFileType()
}
}
/**
* Assumes [isMacBinary].
*/
internal suspend fun isSigned(byteChannel: SeekableByteChannel, binaryId: String): Boolean {
val verificationParams = SignatureVerificationParams(
signRootCertStore = null,
timestampRootCertStore = null,
buildChain = false,
withRevocationCheck = false
)
val verificationParams = SignatureVerificationParams(signRootCertStore = null, timestampRootCertStore = null, buildChain = false, withRevocationCheck = false)
val binaries = MachoArch(byteChannel).Extract()
return binaries.all { binary ->
val signatureData = try {
@@ -278,7 +277,8 @@ internal suspend fun isSigned(byteChannel: SeekableByteChannel, binaryId: String
}
private class SignatureVerificationLog(val binaryId: String) : ILogger {
val span: Span = Span.current()
private val span: Span = Span.current()
fun addEvent(str: String, category: String) {
span.addEvent(str)
.setAttribute("binary", binaryId)
@@ -131,7 +131,7 @@ internal class LocalDiskJarCacheManager(
if (!producer.useCacheAsTargetFile) {
Files.createDirectories(targetFile.parent)
Files.copy(cacheFile, targetFile)
Files.createLink(targetFile, cacheFile)
}
Files.write(cacheMetadataFile, ProtoBuf.encodeToByteArray(JarCacheItem(sources = sourceCacheItems)))
@@ -16,12 +16,12 @@ import org.jetbrains.intellij.build.io.ZipFileWriter
import org.jetbrains.intellij.build.io.archiveDir
import org.jetbrains.intellij.build.io.suspendAwareReadZipFile
import org.jetbrains.intellij.build.io.zipWriter
import java.io.IOException
import java.nio.ByteBuffer
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.PathMatcher
import java.util.zip.Deflater
import kotlin.io.path.name
private const val listOfEntitiesFileName = "META-INF/listOfEntities.txt"
@@ -159,6 +159,14 @@ private suspend fun writeSource(
filesToMerge = filesToMerge,
)
}
catch (e: IOException) {
if (e.message?.contains("No space left on device") == true) {
throw NoDiskSpaceLeftException("No space left while including $sourceFile into $targetFile", e)
}
else {
throw IOException("Failed to include $sourceFile to $targetFile", e)
}
}
finally {
@Suppress("KotlinConstantConditions")
if (sourceFile !== source.file) {
@@ -209,8 +217,6 @@ private suspend fun handleZipSource(
}
}
// FileChannel is strongly required because only FileChannel provides `read(ByteBuffer dst, long position)` method -
// ability to read data without setting channel position, as setting channel position will require synchronization
suspendAwareReadZipFile(sourceFile) { name, dataSupplier ->
if (name == listOfEntitiesFileName) {
filesToMerge.add(Charsets.UTF_8.decode(dataSupplier()))
@@ -289,12 +295,12 @@ private fun checkCoverageAgentManifest(
}
val coveragePlatformAgentModuleName = "intellij.platform.coverage.agent"
if (!targetFile.name.contains(coveragePlatformAgentModuleName)) {
if (!targetFile.fileName.toString().contains(coveragePlatformAgentModuleName)) {
return false
}
val agentPrefix = "intellij-coverage-agent"
if (!sourceFile.name.startsWith(agentPrefix)) {
if (!sourceFile.fileName.toString().startsWith(agentPrefix)) {
return false
}
@@ -7,6 +7,7 @@ import com.intellij.platform.ijent.community.buildConstants.IJENT_BOOT_CLASSPATH
import com.intellij.platform.ijent.community.buildConstants.isMultiRoutingFileSystemEnabledForProduct
import org.jetbrains.intellij.build.BuildContext
import org.jetbrains.intellij.build.VmProperties
import org.jetbrains.intellij.build.checkForNoDiskSpace
import org.jetbrains.intellij.build.dev.BuildRequest
import org.jetbrains.intellij.build.dev.buildProduct
import org.jetbrains.intellij.build.dev.getIdeSystemProperties
@@ -20,26 +21,28 @@ import kotlin.time.Duration
internal suspend fun createDevModeProductRunner(context: BuildContext, additionalPluginModules: List<String> = emptyList()): IntellijProductRunner {
var newClassPath: Collection<Path>? = null
val homeDir = context.paths.projectHome
val runDir = buildProduct(
request = BuildRequest(
//isUnpackedDist = context.productProperties.platformPrefix != "Gateway",
// https://youtrack.jetbrains.com/issue/IJPL-156115/devModeProductRunner-use-packed-dist-as-a-workaround-for-incorrect-product-info.json-entries-links-to-compilation-output
isUnpackedDist = false,
writeCoreClasspath = false,
platformPrefix = context.productProperties.platformPrefix ?: "idea",
additionalModules = additionalPluginModules,
projectDir = homeDir,
devRootDir = context.paths.tempDir.resolve("dev-run"),
jarCacheDir = homeDir.resolve("out/dev-run/jar-cache"),
productionClassOutput = context.classesOutputDirectory.resolve("production"),
platformClassPathConsumer = { _, classPath, _ ->
newClassPath = classPath
},
buildOptionsTemplate = context.options,
),
createProductProperties = { context.productProperties }
)
return DevModeProductRunner(context = context, homePath = runDir, classPath = newClassPath!!.map { it.toString() })
return checkForNoDiskSpace(context) {
val runDir = buildProduct(
request = BuildRequest(
//isUnpackedDist = context.productProperties.platformPrefix != "Gateway",
// https://youtrack.jetbrains.com/issue/IJPL-156115/devModeProductRunner-use-packed-dist-as-a-workaround-for-incorrect-product-info.json-entries-links-to-compilation-output
isUnpackedDist = false,
writeCoreClasspath = false,
platformPrefix = context.productProperties.platformPrefix ?: "idea",
additionalModules = additionalPluginModules,
projectDir = homeDir,
devRootDir = context.paths.tempDir.resolve("dev-run"),
jarCacheDir = homeDir.resolve("out/dev-run/jar-cache"),
productionClassOutput = context.classesOutputDirectory.resolve("production"),
platformClassPathConsumer = { _, classPath, _ ->
newClassPath = classPath
},
buildOptionsTemplate = context.options,
),
createProductProperties = { context.productProperties }
)
DevModeProductRunner(context = context, homePath = runDir, classPath = newClassPath!!.map { it.toString() })
}
}
private class DevModeProductRunner(
@@ -0,0 +1,24 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import com.intellij.openapi.util.SystemInfoRt
import kotlinx.coroutines.runBlocking
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Assumptions.assumeTrue
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import java.nio.file.Path
class SlowZipTest {
@Test
fun `read zip file with more than 65K entries`(@TempDir tempDir: Path) = runBlocking {
assumeTrue(SystemInfoRt.isUnix)
val (list, archiveFile) = createLargeArchive(Short.MAX_VALUE * 2 + 20, tempDir)
checkZip(archiveFile) { zipFile ->
for (name in list) {
assertThat(zipFile.getResource(name)).isNotNull()
}
}
}
}
@@ -1,7 +1,6 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import com.intellij.openapi.util.SystemInfoRt
import com.intellij.util.io.toByteArray
import com.intellij.util.io.write
import com.intellij.util.lang.HashMapZipFile
@@ -21,11 +20,11 @@ import org.jetbrains.intellij.build.io.PackageIndexBuilder
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream
import org.jetbrains.intellij.build.io.ZipIndexWriter
import org.jetbrains.intellij.build.io.compressedData
import org.jetbrains.intellij.build.io.readZipFile
import org.jetbrains.intellij.build.io.zip
import org.jetbrains.intellij.build.io.zipWithCompression
import org.jetbrains.intellij.build.io.zipWithPackageIndex
import org.jetbrains.intellij.build.io.zipWriter
import org.junit.jupiter.api.Assumptions.assumeTrue
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import java.nio.ByteBuffer
@@ -40,7 +39,6 @@ import java.util.zip.ZipEntry
import java.util.zip.ZipInputStream
import kotlin.random.Random
class ZipTest {
@Test
fun `interrupt thread`(@TempDir tempDir: Path) = runBlocking {
@@ -70,18 +68,6 @@ class ZipTest {
}
}
@Test
fun `read zip file with more than 65K entries`(@TempDir tempDir: Path) = runBlocking {
assumeTrue(SystemInfoRt.isUnix)
val (list, archiveFile) = createLargeArchive(Short.MAX_VALUE * 2 + 20, tempDir)
checkZip(archiveFile) { zipFile ->
for (name in list) {
assertThat(zipFile.getResource(name)).isNotNull()
}
}
}
@Test
fun `custom prefix`(@TempDir tempDir: Path) {
val random = Random(42)
@@ -488,6 +474,14 @@ class ZipTest {
val archiveFile = tempDir.resolve("archive.zip")
zipWithCompression(archiveFile, mapOf(dir to ""))
readZipFile(archiveFile) { name, dataProvider ->
for (item in list) {
if (name == item.name) {
assertThat(dataProvider().remaining()).isEqualTo(item.size.toLong())
}
}
}
java.util.zip.ZipFile(archiveFile.toFile()).use { jdkZipFile ->
for (item in list) {
val entry = jdkZipFile.getEntry(item.name)
@@ -600,6 +594,9 @@ class ZipTest {
// check both IKV- and non-IKV variants of an immutable zip file
internal fun checkZip(file: Path, checker: (ZipFile) -> Unit) {
readZipFile(file) { name, dataProvider ->
dataProvider()
}
HashMapZipFile.load(file).use { zipFile ->
checker(zipFile)
}
@@ -678,7 +675,7 @@ internal class TestEntryItem(
@JvmField val name: String,
)
private suspend fun createLargeArchive(size: Int, tempDir: Path, minFileSize: Int = 0, maxFileSize: Int = 32): Pair<List<String>, Path> {
internal suspend fun createLargeArchive(size: Int, tempDir: Path, minFileSize: Int = 0, maxFileSize: Int = 32): Pair<List<String>, Path> {
val (dir, list) = createDirOnDisk(tempDir, size, minFileSize, maxFileSize)
val archiveFile = tempDir.resolve("archive.zip")
zipWithPackageIndex(archiveFile, dir)
@@ -1,4 +1,4 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.impl.maven
import com.intellij.testFramework.utils.io.createFile
@@ -13,8 +13,8 @@ import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.io.TempDir
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.exists
import kotlin.io.path.name
import kotlin.io.path.writeText
@@ -73,7 +73,7 @@ class MavenCentralPublicationTest {
val files = createDistributionFiles().map { "${coordinates.directoryPath}/${it.name}" }
publication.execute()
val bundle = workDir.resolve("bundle.zip")
assert(bundle.exists())
assert(Files.exists(bundle)) {}
val entries = buildList {
suspendAwareReadZipFile(bundle) { entry, _ ->
add(entry)