mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-09-27 10:03:11 +07:00
investigate "No space left on device" - add more diagnostic (as we have for integration tests)
GitOrigin-RevId: f0e60bc7e729928a41bd108d15a2291ddc5c04f2
This commit is contained in:
committed by
intellij-monorepo-bot
parent
463d04c15d
commit
248d320d6e
@@ -17,6 +17,7 @@ def get_jvm_flags(flags):
|
||||
# kotlin compiler
|
||||
"-Dkotlin.environment.keepalive=true",
|
||||
"-Didea.io.use.nio2=true",
|
||||
"-Dio.netty.allocator.useCachedMagazinesForNonEventLoopThreads=true",
|
||||
# https://github.com/netty/netty/issues/11532
|
||||
"-Dio.netty.tryReflectionSetAccessible=true",
|
||||
# see TargetConfigurationDigestProperty.KOTLIN_VERSION - we invalidate cache if kotlinc version changed
|
||||
|
||||
@@ -21,6 +21,9 @@ java_binary(
|
||||
jvm_flags = [
|
||||
"-Djava.awt.headless=true",
|
||||
"-Dapple.awt.UIElement=true",
|
||||
"-Dio.netty.allocator.useCachedMagazinesForNonEventLoopThreads=true",
|
||||
# https://github.com/netty/netty/issues/11532
|
||||
"-Dio.netty.tryReflectionSetAccessible=true",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import io.netty.buffer.ByteBuf
|
||||
import io.netty.buffer.ByteBufAllocator
|
||||
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.Companion.FLUSH_THRESHOLD
|
||||
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.Companion.INITIAL_BUFFER_CAPACITY
|
||||
import java.io.IOException
|
||||
@@ -60,7 +59,7 @@ class ZipArchiveOutputStream(
|
||||
}
|
||||
|
||||
private var finished = false
|
||||
private val buffer = ByteBufAllocator.DEFAULT.directBuffer(INITIAL_BUFFER_CAPACITY)
|
||||
private val buffer = byteBufferAllocator.directBuffer(INITIAL_BUFFER_CAPACITY)
|
||||
private var channelPosition = 0L
|
||||
|
||||
@Suppress("DuplicatedCode")
|
||||
@@ -317,7 +316,7 @@ class ZipArchiveOutputStream(
|
||||
|
||||
if (headerAndDataSize > INITIAL_BUFFER_CAPACITY) {
|
||||
// instead of resizing the current buffer, it's preferable to obtain a buffer of the required size from a pool
|
||||
buffer = ByteBufAllocator.DEFAULT.directBuffer(headerAndDataSize)
|
||||
buffer = byteBufferAllocator.directBuffer(headerAndDataSize)
|
||||
releaseBuffer = true
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import io.netty.buffer.ByteBufAllocator
|
||||
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream.CompressedSizeAndCrc
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.channels.FileChannel
|
||||
@@ -131,7 +130,7 @@ private fun compressAndWriteFile(
|
||||
|
||||
private fun doDeflate(chunkSize: Int, deflater: Deflater, writer: (ByteBuffer) -> Unit): Int {
|
||||
var compressedSize = 0
|
||||
ByteBufAllocator.DEFAULT.directBuffer(chunkSize).use { nettyOutput ->
|
||||
byteBufferAllocator.directBuffer(chunkSize).use { nettyOutput ->
|
||||
val output = nettyOutput.internalNioBuffer(nettyOutput.writerIndex(), chunkSize)!!
|
||||
|
||||
val oldPosition = output.position()
|
||||
|
||||
@@ -3,13 +3,12 @@ package org.jetbrains.intellij.build.io
|
||||
|
||||
import com.dynatrace.hash4j.hashing.Hashing
|
||||
import io.netty.buffer.ByteBuf
|
||||
import io.netty.buffer.ByteBufAllocator
|
||||
import java.util.zip.ZipEntry
|
||||
|
||||
internal const val INDEX_FORMAT_VERSION: Byte = 4
|
||||
|
||||
class ZipIndexWriter(@JvmField val packageIndexBuilder: PackageIndexBuilder?) {
|
||||
private var buffer: ByteBuf? = ByteBufAllocator.DEFAULT.directBuffer(64 * 1024)
|
||||
private var buffer: ByteBuf? = byteBufferAllocator.directBuffer(64 * 1024)
|
||||
|
||||
private var entryCount = 0
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.io
|
||||
|
||||
import io.netty.buffer.AdaptiveByteBufAllocator
|
||||
import io.netty.buffer.ByteBuf
|
||||
import io.netty.buffer.ByteBufAllocator
|
||||
import java.lang.invoke.MethodHandles
|
||||
@@ -8,6 +9,19 @@ import java.lang.invoke.MethodType
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
|
||||
val byteBufferAllocator: ByteBufAllocator = run {
|
||||
System.setProperty("io.netty.tryReflectionSetAccessible", "true")
|
||||
|
||||
if (System.getProperty("io.netty.allocator.useCachedMagazinesForNonEventLoopThreads") == "true" &&
|
||||
System.getProperty("io.netty.allocator.type", "adaptive") == "adaptive") {
|
||||
val allocator = ByteBufAllocator.DEFAULT
|
||||
if (allocator is AdaptiveByteBufAllocator) {
|
||||
return@run allocator
|
||||
}
|
||||
}
|
||||
AdaptiveByteBufAllocator(true, true)
|
||||
}
|
||||
|
||||
// not thread-safe, intended only for single thread for one time use
|
||||
internal class ByteBufferAllocator() : AutoCloseable {
|
||||
private var byteBuf: ByteBuf? = null
|
||||
@@ -22,13 +36,9 @@ internal class ByteBufferAllocator() : AutoCloseable {
|
||||
}
|
||||
|
||||
if (result == null) {
|
||||
result = ByteBufAllocator.DEFAULT.directBuffer(roundUpInt(size, 65_536))
|
||||
result = byteBufferAllocator.directBuffer(roundUpInt(size, 65_536))
|
||||
byteBuf = result
|
||||
}
|
||||
else {
|
||||
result.clear()
|
||||
}
|
||||
|
||||
return result.internalNioBuffer(result.writerIndex(), size).order(ByteOrder.LITTLE_ENDIAN).also { nioByteBuffer = it }
|
||||
}
|
||||
|
||||
|
||||
@@ -20,8 +20,13 @@ typealias EntryProcessor = (String, () -> ByteBuffer) -> Unit
|
||||
fun readZipFile(file: Path, entryProcessor: EntryProcessor) {
|
||||
// FileChannel is strongly required because only FileChannel provides `read(ByteBuffer dst, long position)` method -
|
||||
// ability to read data without setting channel position, as setting channel position will require synchronization
|
||||
mapFileAndUse(file) { buffer, fileSize ->
|
||||
readZipEntries(buffer = buffer, fileSize = fileSize, entryProcessor = entryProcessor)
|
||||
try {
|
||||
mapFileAndUse(file) { buffer, fileSize ->
|
||||
readZipEntries(buffer = buffer, fileSize = fileSize, entryProcessor = entryProcessor)
|
||||
}
|
||||
}
|
||||
catch (e: IOException) {
|
||||
throw IOException("Cannot read $file", e)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,9 +65,6 @@ private inline fun mapFileAndUse(file: Path, consumer: (ByteBuffer, fileSize: In
|
||||
try {
|
||||
consumer(mappedBuffer, fileSize)
|
||||
}
|
||||
catch (e: IOException) {
|
||||
throw IOException(file.toString(), e)
|
||||
}
|
||||
finally {
|
||||
if (mappedBuffer.isDirect) {
|
||||
// on Windows memory-mapped file cannot be deleted without clearing in-memory buffer first
|
||||
@@ -128,7 +130,7 @@ internal inline fun readCentralDirectory(
|
||||
private const val STORED: Byte = 0
|
||||
private const val DEFLATED: Byte = 8
|
||||
|
||||
internal fun getByteBuffer(
|
||||
private fun getByteBuffer(
|
||||
buffer: ByteBuffer,
|
||||
compressedSize: Int,
|
||||
uncompressedSize: Int,
|
||||
@@ -161,10 +163,18 @@ internal fun getByteBuffer(
|
||||
inflater.setInput(inputBuffer)
|
||||
try {
|
||||
val result = byteBufferAllocator.allocate(uncompressedSize)
|
||||
val oldPosition = result.position()
|
||||
while (result.hasRemaining()) {
|
||||
check(inflater.inflate(result) != 0) { "Inflater wants input, but input was already set" }
|
||||
val inflatedByteCount = inflater.inflate(result)
|
||||
check(inflatedByteCount != 0) {
|
||||
"Inflater wants input, but input was already set"
|
||||
}
|
||||
check(inflatedByteCount == uncompressedSize) {
|
||||
"Inflater returned unexpected result: $inflatedByteCount instead of $uncompressedSize"
|
||||
}
|
||||
}
|
||||
result.rewind()
|
||||
result.limit(result.position())
|
||||
result.position(oldPosition)
|
||||
return result
|
||||
}
|
||||
catch (e: DataFormatException) {
|
||||
|
||||
+3
-1
@@ -696,7 +696,9 @@ private suspend fun copyAnt(pluginDir: Path, context: BuildContext): List<Distri
|
||||
sources.sort()
|
||||
|
||||
val antTargetFile = antDir.resolve("ant.jar")
|
||||
buildJar(targetFile = antTargetFile, sources = sources)
|
||||
checkForNoDiskSpace(context) {
|
||||
buildJar(targetFile = antTargetFile, sources = sources)
|
||||
}
|
||||
|
||||
sources.map { source ->
|
||||
ProjectLibraryEntry(
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
@file:Suppress("ReplaceGetOrSet")
|
||||
|
||||
package org.jetbrains.intellij.build
|
||||
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap
|
||||
import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap
|
||||
import java.io.IOException
|
||||
import java.nio.file.*
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
|
||||
internal class NoDiskSpaceLeftException(message: String, e: IOException) : RuntimeException(message, e)
|
||||
|
||||
internal inline fun <T> checkForNoDiskSpace(context: BuildContext, task: () -> T): T {
|
||||
try {
|
||||
return task()
|
||||
}
|
||||
catch (e: NoDiskSpaceLeftException) {
|
||||
throw IOException(getDiskUsageDiagnostics(context.paths), e)
|
||||
}
|
||||
}
|
||||
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
private fun getDiskInfo(fileStore: FileStore, builder: StringBuilder) {
|
||||
builder.appendLine("Disk info of ${fileStore.name()}")
|
||||
builder.appendLine(" Total space: " + formatSize(fileStore.totalSpace))
|
||||
builder.appendLine(" Unallocated space: " + formatSize(fileStore.unallocatedSpace))
|
||||
builder.appendLine(" Usable space: " + formatSize(fileStore.usableSpace))
|
||||
}
|
||||
|
||||
@Suppress("SpellCheckingInspection")
|
||||
private fun formatSize(value: Long): String {
|
||||
if (value < 1024) {
|
||||
return "$value B"
|
||||
}
|
||||
val z = (63 - java.lang.Long.numberOfLeadingZeros(value)) / 10
|
||||
return String.format("%.1f %sB", value.toDouble() / (1L shl z * 10), " KMGTPE"[z])
|
||||
}
|
||||
|
||||
private fun describe(dir: Path, builder: StringBuilder) {
|
||||
if (Files.notExists(dir)) {
|
||||
return
|
||||
}
|
||||
|
||||
builder.appendLine("Disk usage by $dir")
|
||||
builder.append(getDiskInfo(Files.getFileStore(dir), builder))
|
||||
listDirectoryContent(dir, maxDepth = 3, builder)
|
||||
builder.append('\n')
|
||||
}
|
||||
|
||||
private fun getDiskUsageDiagnostics(paths: BuildPaths): String {
|
||||
val builder = StringBuilder()
|
||||
describe(paths.distAllDir, builder)
|
||||
if (!paths.tempDir.startsWith(paths.distAllDir)) {
|
||||
describe(paths.tempDir, builder)
|
||||
}
|
||||
return builder.toString()
|
||||
}
|
||||
|
||||
internal object TestListing {
|
||||
@JvmStatic
|
||||
fun main(args: Array<String>) {
|
||||
val builder = StringBuilder()
|
||||
listDirectoryContent(Path.of(args[0]), maxDepth = 3, result = builder)
|
||||
println(builder)
|
||||
}
|
||||
}
|
||||
|
||||
private fun listDirectoryContent(directoryPath: Path, @Suppress("SameParameterValue") maxDepth: Int = 1, result: StringBuilder): String {
|
||||
if (Files.notExists(directoryPath)) {
|
||||
return "Directory does not exist: $directoryPath"
|
||||
}
|
||||
|
||||
if (!Files.isDirectory(directoryPath)) {
|
||||
return "$directoryPath is not a directory"
|
||||
}
|
||||
|
||||
// map to store directory paths and their sizes
|
||||
val directorySizes = Object2LongOpenHashMap<Path>()
|
||||
|
||||
// list to store entries for display
|
||||
data class Entry(
|
||||
@JvmField val path: Path,
|
||||
@JvmField val size: Long,
|
||||
@JvmField val depth: Int,
|
||||
)
|
||||
|
||||
val entries = mutableListOf<Entry>()
|
||||
|
||||
Files.walkFileTree(directoryPath, object : SimpleFileVisitor<Path>() {
|
||||
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
|
||||
// initialize size to 0 for all directories
|
||||
directorySizes.put(dir, 0L)
|
||||
|
||||
val depth = directoryPath.relativize(dir).nameCount
|
||||
|
||||
// add directory to entries for display if within max depth
|
||||
if (dir != directoryPath && depth <= maxDepth) {
|
||||
entries.add(Entry(path = dir, size = -1, depth = depth))
|
||||
}
|
||||
|
||||
// skip traversing deeper than needed for display purposes
|
||||
if (depth > maxDepth) {
|
||||
return FileVisitResult.SKIP_SUBTREE
|
||||
}
|
||||
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
|
||||
val size = attrs.size()
|
||||
val depth = directoryPath.relativize(file).nameCount
|
||||
|
||||
// add a file to entries for display if within max depth
|
||||
if (depth <= maxDepth) {
|
||||
entries.add(Entry(path = file, size = size, depth = depth))
|
||||
}
|
||||
|
||||
// add size to all parent directories
|
||||
var currentPath = file.parent
|
||||
while (currentPath != null && currentPath.startsWith(directoryPath)) {
|
||||
directorySizes.addTo(currentPath, size)
|
||||
currentPath = currentPath.parent
|
||||
}
|
||||
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult {
|
||||
val depth = directoryPath.relativize(file).nameCount
|
||||
|
||||
if (depth <= maxDepth) {
|
||||
entries.add(Entry(path = file, size = 0, depth = depth))
|
||||
}
|
||||
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
})
|
||||
|
||||
// format the output
|
||||
result.appendLine("Content of: $directoryPath")
|
||||
|
||||
// group entries by parent directory to track last items
|
||||
val entriesByDepth = entries.groupByTo(Int2ObjectOpenHashMap()) { it.depth }
|
||||
|
||||
// track the last entry at each depth level
|
||||
val lastEntryByDepth = Int2ObjectOpenHashMap<Path>()
|
||||
for (depth in 1..maxDepth) {
|
||||
entriesByDepth.get(depth)?.let { entriesAtDepth ->
|
||||
if (entriesAtDepth.isNotEmpty()) {
|
||||
lastEntryByDepth.put(depth, entriesAtDepth.last().path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort entries by path for tree-like structure
|
||||
entries.sortWith(compareBy({ it.path.parent }, { -it.size }, { it.path }))
|
||||
|
||||
// display entries with their sizes
|
||||
for (entry in entries) {
|
||||
val depth = entry.depth
|
||||
val name = entry.path.fileName
|
||||
val isLast = entry.path == lastEntryByDepth.get(depth)
|
||||
|
||||
// Create ASCII guide indentation
|
||||
val indent = StringBuilder()
|
||||
for (i in 1 until depth) {
|
||||
indent.append(if (lastEntryByDepth.get(i) != entry.path.subpath(0, i).toAbsolutePath()) "│ " else " ")
|
||||
}
|
||||
|
||||
// add connector for the current item
|
||||
val connector = if (isLast) "└── " else "├── "
|
||||
if (depth > 0) {
|
||||
indent.append(connector)
|
||||
}
|
||||
|
||||
if (entry.size == -1L) {
|
||||
val size = directorySizes.getLong(entry.path)
|
||||
result.appendLine("$indent[DIR] $name: ${formatFileSize(size)}")
|
||||
}
|
||||
else {
|
||||
result.appendLine("$indent$name: ${formatFileSize(entry.size)}")
|
||||
}
|
||||
}
|
||||
// add total size
|
||||
result.appendLine("Total size: ${formatFileSize(directorySizes.getLong(directoryPath))}")
|
||||
|
||||
return result.toString()
|
||||
}
|
||||
|
||||
private fun formatFileSize(size: Long): String {
|
||||
if (size < 1024) return "$size B"
|
||||
|
||||
val units = arrayOf("B", "KB", "MB", "GB", "TB", "PB", "EB")
|
||||
var value = size.toDouble()
|
||||
var unitIndex = 0
|
||||
|
||||
while (value >= 1024 && unitIndex < units.size - 1) {
|
||||
value /= 1024
|
||||
unitIndex++
|
||||
}
|
||||
|
||||
return "%.2f %s".format(value, units[unitIndex])
|
||||
}
|
||||
@@ -7,14 +7,41 @@ import com.intellij.platform.ijent.community.buildConstants.MULTI_ROUTING_FILE_S
|
||||
import com.intellij.platform.ijent.community.buildConstants.isMultiRoutingFileSystemEnabledForProduct
|
||||
import com.intellij.platform.runtime.product.ProductMode
|
||||
import com.intellij.util.containers.with
|
||||
import com.intellij.util.text.SemVer
|
||||
import io.opentelemetry.api.common.AttributeKey
|
||||
import io.opentelemetry.api.common.Attributes
|
||||
import io.opentelemetry.api.trace.Span
|
||||
import kotlinx.collections.immutable.PersistentList
|
||||
import kotlinx.collections.immutable.persistentListOf
|
||||
import kotlinx.coroutines.*
|
||||
import org.jetbrains.intellij.build.*
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.CoroutineName
|
||||
import kotlinx.coroutines.CoroutineStart
|
||||
import kotlinx.coroutines.Deferred
|
||||
import kotlinx.coroutines.DelicateCoroutinesApi
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.GlobalScope
|
||||
import kotlinx.coroutines.async
|
||||
import org.jetbrains.intellij.build.ApplicationInfoProperties
|
||||
import org.jetbrains.intellij.build.ApplicationInfoPropertiesImpl
|
||||
import org.jetbrains.intellij.build.BuildContext
|
||||
import org.jetbrains.intellij.build.BuildOptions
|
||||
import org.jetbrains.intellij.build.BuiltinModulesFileData
|
||||
import org.jetbrains.intellij.build.CompilationContext
|
||||
import org.jetbrains.intellij.build.ContentModuleFilter
|
||||
import org.jetbrains.intellij.build.DistFile
|
||||
import org.jetbrains.intellij.build.FrontendModuleFilter
|
||||
import org.jetbrains.intellij.build.JarPackagerDependencyHelper
|
||||
import org.jetbrains.intellij.build.JvmArchitecture
|
||||
import org.jetbrains.intellij.build.LinuxDistributionCustomizer
|
||||
import org.jetbrains.intellij.build.MacDistributionCustomizer
|
||||
import org.jetbrains.intellij.build.OsFamily
|
||||
import org.jetbrains.intellij.build.PLATFORM_LOADER_JAR
|
||||
import org.jetbrains.intellij.build.ProductProperties
|
||||
import org.jetbrains.intellij.build.ProprietaryBuildTools
|
||||
import org.jetbrains.intellij.build.Source
|
||||
import org.jetbrains.intellij.build.WindowsDistributionCustomizer
|
||||
import org.jetbrains.intellij.build.buildJar
|
||||
import org.jetbrains.intellij.build.checkForNoDiskSpace
|
||||
import org.jetbrains.intellij.build.computeAppInfoXml
|
||||
import org.jetbrains.intellij.build.impl.PlatformJarNames.PLATFORM_CORE_NIO_FS
|
||||
import org.jetbrains.intellij.build.impl.plugins.PluginAutoPublishList
|
||||
import org.jetbrains.intellij.build.io.runProcess
|
||||
@@ -420,7 +447,9 @@ class BuildContextImpl internal constructor(
|
||||
}
|
||||
|
||||
override suspend fun produce(targetFile: Path) {
|
||||
buildJar(targetFile = targetFile, sources = sources, compress = compress, notify = false)
|
||||
checkForNoDiskSpace(this@BuildContextImpl) {
|
||||
buildJar(targetFile = targetFile, sources = sources, compress = compress, notify = false)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
@@ -32,6 +32,7 @@ import org.jetbrains.intellij.build.UTIL_8_JAR
|
||||
import org.jetbrains.intellij.build.UTIL_JAR
|
||||
import org.jetbrains.intellij.build.ZipSource
|
||||
import org.jetbrains.intellij.build.buildJar
|
||||
import org.jetbrains.intellij.build.checkForNoDiskSpace
|
||||
import org.jetbrains.intellij.build.computeHashForModuleOutput
|
||||
import org.jetbrains.intellij.build.computeModuleSourcesByContent
|
||||
import org.jetbrains.intellij.build.defaultLibrarySourcesNamesFilter
|
||||
@@ -911,7 +912,7 @@ private suspend fun buildJars(
|
||||
sources = sources,
|
||||
nativeFileHandler = nativeFileHandler,
|
||||
notify = false,
|
||||
addDirEntries = asset.includedModules.any { helper.isTestPluginModule(it.key.moduleName, null) },
|
||||
addDirEntries = asset.includedModules.any { helper.isTestPluginModule(moduleName = it.key.moduleName, module = null) },
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -995,14 +996,16 @@ suspend fun buildJar(targetFile: Path, moduleNames: List<String>, context: Build
|
||||
return
|
||||
}
|
||||
|
||||
buildJar(
|
||||
targetFile = targetFile,
|
||||
sources = moduleNames.mapNotNull { moduleName ->
|
||||
val module = context.findRequiredModule(moduleName)
|
||||
val output = context.getModuleOutputDir(module)
|
||||
toSource(module = module, outputDir = output, excludes = commonModuleExcludes)
|
||||
},
|
||||
)
|
||||
checkForNoDiskSpace(context) {
|
||||
buildJar(
|
||||
targetFile = targetFile,
|
||||
sources = moduleNames.mapNotNull { moduleName ->
|
||||
val module = context.findRequiredModule(moduleName)
|
||||
val output = context.getModuleOutputDir(module)
|
||||
toSource(module = module, outputDir = output, excludes = commonModuleExcludes)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun toSource(module: JpsModule, outputDir: Path, excludes: List<PathMatcher>): Source? {
|
||||
|
||||
@@ -20,7 +20,6 @@ import kotlinx.collections.immutable.persistentMapOf
|
||||
import kotlinx.coroutines.CoroutineName
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import org.jetbrains.intellij.build.BuildContext
|
||||
@@ -50,15 +49,18 @@ import kotlin.io.path.extension
|
||||
import kotlin.io.path.name
|
||||
import kotlin.io.path.relativeTo
|
||||
|
||||
internal fun isMacLibrary(name: String): Boolean =
|
||||
name.endsWith(".jnilib") ||
|
||||
name.endsWith(".dylib") ||
|
||||
name.endsWith(".so") ||
|
||||
name.endsWith(".tbd")
|
||||
internal fun isMacLibrary(name: String): Boolean {
|
||||
return name.endsWith(".jnilib") ||
|
||||
name.endsWith(".dylib") ||
|
||||
name.endsWith(".so") ||
|
||||
name.endsWith(".tbd")
|
||||
}
|
||||
|
||||
internal fun CoroutineScope.recursivelySignMacBinaries(root: Path,
|
||||
context: BuildContext,
|
||||
executableFileMatchers: Collection<PathMatcher> = emptyList()) {
|
||||
internal fun CoroutineScope.recursivelySignMacBinaries(
|
||||
root: Path,
|
||||
context: BuildContext,
|
||||
executableFileMatchers: Collection<PathMatcher> = emptyList(),
|
||||
) {
|
||||
val archives = mutableListOf<Path>()
|
||||
val binaries = mutableListOf<Path>()
|
||||
|
||||
@@ -92,7 +94,7 @@ internal fun CoroutineScope.recursivelySignMacBinaries(root: Path,
|
||||
}
|
||||
|
||||
private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context: BuildContext) {
|
||||
val filesToBeSigned = mutableMapOf<String, Path>()
|
||||
val filesToBeSigned = LinkedHashMap<String, Path>()
|
||||
suspendAwareReadZipFile(zip) { name, dataSupplier ->
|
||||
if (!isMacLibrary(name)) {
|
||||
return@suspendAwareReadZipFile
|
||||
@@ -118,9 +120,7 @@ private suspend fun signAndRepackZipIfMacSignaturesAreMissing(zip: Path, context
|
||||
return
|
||||
}
|
||||
|
||||
coroutineScope {
|
||||
signMacBinaries(files = filesToBeSigned.values.toList(), context = context, checkPermissions = false)
|
||||
}
|
||||
signMacBinaries(files = filesToBeSigned.values.toList(), context = context, checkPermissions = false)
|
||||
|
||||
copyZipReplacing(origin = zip, entries = filesToBeSigned, context = context)
|
||||
for (file in filesToBeSigned.values) {
|
||||
@@ -167,10 +167,12 @@ internal fun signingOptions(contentType: String, context: BuildContext): Persist
|
||||
)
|
||||
}
|
||||
|
||||
internal suspend fun signMacBinaries(files: List<Path>,
|
||||
context: BuildContext,
|
||||
checkPermissions: Boolean = true,
|
||||
additionalOptions: Map<String, String> = emptyMap()) {
|
||||
internal suspend fun signMacBinaries(
|
||||
files: List<Path>,
|
||||
context: BuildContext,
|
||||
checkPermissions: Boolean = true,
|
||||
additionalOptions: Map<String, String> = emptyMap(),
|
||||
) {
|
||||
if (files.isEmpty() || !context.isMacCodeSignEnabled) {
|
||||
return
|
||||
}
|
||||
@@ -227,24 +229,21 @@ internal suspend fun isSigned(path: Path): Boolean {
|
||||
}
|
||||
}
|
||||
|
||||
internal fun isMacBinary(byteChannel: SeekableByteChannel): Boolean =
|
||||
detectFileType(byteChannel).first == FileType.MachO
|
||||
internal fun isMacBinary(byteChannel: SeekableByteChannel): Boolean {
|
||||
return detectFileType(byteChannel).first == FileType.MachO
|
||||
}
|
||||
|
||||
private fun detectFileType(byteChannel: SeekableByteChannel): Pair<FileType, EnumSet<FileProperties>> =
|
||||
byteChannel.use {
|
||||
private fun detectFileType(byteChannel: SeekableByteChannel): Pair<FileType, EnumSet<FileProperties>> {
|
||||
return byteChannel.use {
|
||||
it.DetectFileType()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assumes [isMacBinary].
|
||||
*/
|
||||
internal suspend fun isSigned(byteChannel: SeekableByteChannel, binaryId: String): Boolean {
|
||||
val verificationParams = SignatureVerificationParams(
|
||||
signRootCertStore = null,
|
||||
timestampRootCertStore = null,
|
||||
buildChain = false,
|
||||
withRevocationCheck = false
|
||||
)
|
||||
val verificationParams = SignatureVerificationParams(signRootCertStore = null, timestampRootCertStore = null, buildChain = false, withRevocationCheck = false)
|
||||
val binaries = MachoArch(byteChannel).Extract()
|
||||
return binaries.all { binary ->
|
||||
val signatureData = try {
|
||||
@@ -278,7 +277,8 @@ internal suspend fun isSigned(byteChannel: SeekableByteChannel, binaryId: String
|
||||
}
|
||||
|
||||
private class SignatureVerificationLog(val binaryId: String) : ILogger {
|
||||
val span: Span = Span.current()
|
||||
private val span: Span = Span.current()
|
||||
|
||||
fun addEvent(str: String, category: String) {
|
||||
span.addEvent(str)
|
||||
.setAttribute("binary", binaryId)
|
||||
|
||||
+1
-1
@@ -131,7 +131,7 @@ internal class LocalDiskJarCacheManager(
|
||||
|
||||
if (!producer.useCacheAsTargetFile) {
|
||||
Files.createDirectories(targetFile.parent)
|
||||
Files.copy(cacheFile, targetFile)
|
||||
Files.createLink(targetFile, cacheFile)
|
||||
}
|
||||
|
||||
Files.write(cacheMetadataFile, ProtoBuf.encodeToByteArray(JarCacheItem(sources = sourceCacheItems)))
|
||||
|
||||
@@ -16,12 +16,12 @@ import org.jetbrains.intellij.build.io.ZipFileWriter
|
||||
import org.jetbrains.intellij.build.io.archiveDir
|
||||
import org.jetbrains.intellij.build.io.suspendAwareReadZipFile
|
||||
import org.jetbrains.intellij.build.io.zipWriter
|
||||
import java.io.IOException
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.PathMatcher
|
||||
import java.util.zip.Deflater
|
||||
import kotlin.io.path.name
|
||||
|
||||
private const val listOfEntitiesFileName = "META-INF/listOfEntities.txt"
|
||||
|
||||
@@ -159,6 +159,14 @@ private suspend fun writeSource(
|
||||
filesToMerge = filesToMerge,
|
||||
)
|
||||
}
|
||||
catch (e: IOException) {
|
||||
if (e.message?.contains("No space left on device") == true) {
|
||||
throw NoDiskSpaceLeftException("No space left while including $sourceFile into $targetFile", e)
|
||||
}
|
||||
else {
|
||||
throw IOException("Failed to include $sourceFile to $targetFile", e)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
@Suppress("KotlinConstantConditions")
|
||||
if (sourceFile !== source.file) {
|
||||
@@ -209,8 +217,6 @@ private suspend fun handleZipSource(
|
||||
}
|
||||
}
|
||||
|
||||
// FileChannel is strongly required because only FileChannel provides `read(ByteBuffer dst, long position)` method -
|
||||
// ability to read data without setting channel position, as setting channel position will require synchronization
|
||||
suspendAwareReadZipFile(sourceFile) { name, dataSupplier ->
|
||||
if (name == listOfEntitiesFileName) {
|
||||
filesToMerge.add(Charsets.UTF_8.decode(dataSupplier()))
|
||||
@@ -289,12 +295,12 @@ private fun checkCoverageAgentManifest(
|
||||
}
|
||||
|
||||
val coveragePlatformAgentModuleName = "intellij.platform.coverage.agent"
|
||||
if (!targetFile.name.contains(coveragePlatformAgentModuleName)) {
|
||||
if (!targetFile.fileName.toString().contains(coveragePlatformAgentModuleName)) {
|
||||
return false
|
||||
}
|
||||
|
||||
val agentPrefix = "intellij-coverage-agent"
|
||||
if (!sourceFile.name.startsWith(agentPrefix)) {
|
||||
if (!sourceFile.fileName.toString().startsWith(agentPrefix)) {
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
+23
-20
@@ -7,6 +7,7 @@ import com.intellij.platform.ijent.community.buildConstants.IJENT_BOOT_CLASSPATH
|
||||
import com.intellij.platform.ijent.community.buildConstants.isMultiRoutingFileSystemEnabledForProduct
|
||||
import org.jetbrains.intellij.build.BuildContext
|
||||
import org.jetbrains.intellij.build.VmProperties
|
||||
import org.jetbrains.intellij.build.checkForNoDiskSpace
|
||||
import org.jetbrains.intellij.build.dev.BuildRequest
|
||||
import org.jetbrains.intellij.build.dev.buildProduct
|
||||
import org.jetbrains.intellij.build.dev.getIdeSystemProperties
|
||||
@@ -20,26 +21,28 @@ import kotlin.time.Duration
|
||||
internal suspend fun createDevModeProductRunner(context: BuildContext, additionalPluginModules: List<String> = emptyList()): IntellijProductRunner {
|
||||
var newClassPath: Collection<Path>? = null
|
||||
val homeDir = context.paths.projectHome
|
||||
val runDir = buildProduct(
|
||||
request = BuildRequest(
|
||||
//isUnpackedDist = context.productProperties.platformPrefix != "Gateway",
|
||||
// https://youtrack.jetbrains.com/issue/IJPL-156115/devModeProductRunner-use-packed-dist-as-a-workaround-for-incorrect-product-info.json-entries-links-to-compilation-output
|
||||
isUnpackedDist = false,
|
||||
writeCoreClasspath = false,
|
||||
platformPrefix = context.productProperties.platformPrefix ?: "idea",
|
||||
additionalModules = additionalPluginModules,
|
||||
projectDir = homeDir,
|
||||
devRootDir = context.paths.tempDir.resolve("dev-run"),
|
||||
jarCacheDir = homeDir.resolve("out/dev-run/jar-cache"),
|
||||
productionClassOutput = context.classesOutputDirectory.resolve("production"),
|
||||
platformClassPathConsumer = { _, classPath, _ ->
|
||||
newClassPath = classPath
|
||||
},
|
||||
buildOptionsTemplate = context.options,
|
||||
),
|
||||
createProductProperties = { context.productProperties }
|
||||
)
|
||||
return DevModeProductRunner(context = context, homePath = runDir, classPath = newClassPath!!.map { it.toString() })
|
||||
return checkForNoDiskSpace(context) {
|
||||
val runDir = buildProduct(
|
||||
request = BuildRequest(
|
||||
//isUnpackedDist = context.productProperties.platformPrefix != "Gateway",
|
||||
// https://youtrack.jetbrains.com/issue/IJPL-156115/devModeProductRunner-use-packed-dist-as-a-workaround-for-incorrect-product-info.json-entries-links-to-compilation-output
|
||||
isUnpackedDist = false,
|
||||
writeCoreClasspath = false,
|
||||
platformPrefix = context.productProperties.platformPrefix ?: "idea",
|
||||
additionalModules = additionalPluginModules,
|
||||
projectDir = homeDir,
|
||||
devRootDir = context.paths.tempDir.resolve("dev-run"),
|
||||
jarCacheDir = homeDir.resolve("out/dev-run/jar-cache"),
|
||||
productionClassOutput = context.classesOutputDirectory.resolve("production"),
|
||||
platformClassPathConsumer = { _, classPath, _ ->
|
||||
newClassPath = classPath
|
||||
},
|
||||
buildOptionsTemplate = context.options,
|
||||
),
|
||||
createProductProperties = { context.productProperties }
|
||||
)
|
||||
DevModeProductRunner(context = context, homePath = runDir, classPath = newClassPath!!.map { it.toString() })
|
||||
}
|
||||
}
|
||||
|
||||
private class DevModeProductRunner(
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build
|
||||
|
||||
import com.intellij.openapi.util.SystemInfoRt
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.assertj.core.api.Assertions.assertThat
|
||||
import org.junit.jupiter.api.Assumptions.assumeTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
import java.nio.file.Path
|
||||
|
||||
class SlowZipTest {
|
||||
@Test
|
||||
fun `read zip file with more than 65K entries`(@TempDir tempDir: Path) = runBlocking {
|
||||
assumeTrue(SystemInfoRt.isUnix)
|
||||
|
||||
val (list, archiveFile) = createLargeArchive(Short.MAX_VALUE * 2 + 20, tempDir)
|
||||
checkZip(archiveFile) { zipFile ->
|
||||
for (name in list) {
|
||||
assertThat(zipFile.getResource(name)).isNotNull()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build
|
||||
|
||||
import com.intellij.openapi.util.SystemInfoRt
|
||||
import com.intellij.util.io.toByteArray
|
||||
import com.intellij.util.io.write
|
||||
import com.intellij.util.lang.HashMapZipFile
|
||||
@@ -21,11 +20,11 @@ import org.jetbrains.intellij.build.io.PackageIndexBuilder
|
||||
import org.jetbrains.intellij.build.io.ZipArchiveOutputStream
|
||||
import org.jetbrains.intellij.build.io.ZipIndexWriter
|
||||
import org.jetbrains.intellij.build.io.compressedData
|
||||
import org.jetbrains.intellij.build.io.readZipFile
|
||||
import org.jetbrains.intellij.build.io.zip
|
||||
import org.jetbrains.intellij.build.io.zipWithCompression
|
||||
import org.jetbrains.intellij.build.io.zipWithPackageIndex
|
||||
import org.jetbrains.intellij.build.io.zipWriter
|
||||
import org.junit.jupiter.api.Assumptions.assumeTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
import java.nio.ByteBuffer
|
||||
@@ -40,7 +39,6 @@ import java.util.zip.ZipEntry
|
||||
import java.util.zip.ZipInputStream
|
||||
import kotlin.random.Random
|
||||
|
||||
|
||||
class ZipTest {
|
||||
@Test
|
||||
fun `interrupt thread`(@TempDir tempDir: Path) = runBlocking {
|
||||
@@ -70,18 +68,6 @@ class ZipTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `read zip file with more than 65K entries`(@TempDir tempDir: Path) = runBlocking {
|
||||
assumeTrue(SystemInfoRt.isUnix)
|
||||
|
||||
val (list, archiveFile) = createLargeArchive(Short.MAX_VALUE * 2 + 20, tempDir)
|
||||
checkZip(archiveFile) { zipFile ->
|
||||
for (name in list) {
|
||||
assertThat(zipFile.getResource(name)).isNotNull()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `custom prefix`(@TempDir tempDir: Path) {
|
||||
val random = Random(42)
|
||||
@@ -488,6 +474,14 @@ class ZipTest {
|
||||
val archiveFile = tempDir.resolve("archive.zip")
|
||||
zipWithCompression(archiveFile, mapOf(dir to ""))
|
||||
|
||||
readZipFile(archiveFile) { name, dataProvider ->
|
||||
for (item in list) {
|
||||
if (name == item.name) {
|
||||
assertThat(dataProvider().remaining()).isEqualTo(item.size.toLong())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
java.util.zip.ZipFile(archiveFile.toFile()).use { jdkZipFile ->
|
||||
for (item in list) {
|
||||
val entry = jdkZipFile.getEntry(item.name)
|
||||
@@ -600,6 +594,9 @@ class ZipTest {
|
||||
|
||||
// check both IKV- and non-IKV variants of an immutable zip file
|
||||
internal fun checkZip(file: Path, checker: (ZipFile) -> Unit) {
|
||||
readZipFile(file) { name, dataProvider ->
|
||||
dataProvider()
|
||||
}
|
||||
HashMapZipFile.load(file).use { zipFile ->
|
||||
checker(zipFile)
|
||||
}
|
||||
@@ -678,7 +675,7 @@ internal class TestEntryItem(
|
||||
@JvmField val name: String,
|
||||
)
|
||||
|
||||
private suspend fun createLargeArchive(size: Int, tempDir: Path, minFileSize: Int = 0, maxFileSize: Int = 32): Pair<List<String>, Path> {
|
||||
internal suspend fun createLargeArchive(size: Int, tempDir: Path, minFileSize: Int = 0, maxFileSize: Int = 32): Pair<List<String>, Path> {
|
||||
val (dir, list) = createDirOnDisk(tempDir, size, minFileSize, maxFileSize)
|
||||
val archiveFile = tempDir.resolve("archive.zip")
|
||||
zipWithPackageIndex(archiveFile, dir)
|
||||
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package org.jetbrains.intellij.build.impl.maven
|
||||
|
||||
import com.intellij.testFramework.utils.io.createFile
|
||||
@@ -13,8 +13,8 @@ import org.junit.jupiter.api.Assertions
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.assertThrows
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.exists
|
||||
import kotlin.io.path.name
|
||||
import kotlin.io.path.writeText
|
||||
|
||||
@@ -73,7 +73,7 @@ class MavenCentralPublicationTest {
|
||||
val files = createDistributionFiles().map { "${coordinates.directoryPath}/${it.name}" }
|
||||
publication.execute()
|
||||
val bundle = workDir.resolve("bundle.zip")
|
||||
assert(bundle.exists())
|
||||
assert(Files.exists(bundle)) {}
|
||||
val entries = buildList {
|
||||
suspendAwareReadZipFile(bundle) { entry, _ ->
|
||||
add(entry)
|
||||
|
||||
Reference in New Issue
Block a user