From 2f2a86bc384269c77141db5205ec8a97219eddc6 Mon Sep 17 00:00:00 2001 From: Vladimir Lagunov Date: Mon, 29 Sep 2025 16:37:18 +0200 Subject: [PATCH] IJPL-201569 IJent: reimplement function for reading whole files to not require a special channel GitOrigin-RevId: 31c89d8e1ef66de5b59652813c8466c994802c53 --- .../platform/eel/impl/fs/EelFsResultImpl.kt | 15 +- .../platform/eel/impl/fs/EelReadFileImpl.kt | 253 ++++++++++++++++++ .../eel/impl/fs/LocalEelFileSystemApi.kt | 164 ++++++++---- .../platform/eel/provider/utils/eelUtils.kt | 3 + .../eel/fs/EelFileSystemApiHelpers.kt | 185 +++++++++++++ .../eel/fs/OpenForReadingArgsBuilder.kt | 84 ++++++ .../platform/eel/fs/ReadFileArgsBuilder.kt | 88 ++++++ .../platform/eel/channels/EelLowLevelApi.kt | 11 +- .../platform/eel/fs/EelFileSystemApi.kt | 133 +++++++-- .../impl/IjentFailSafeFileSystemPosixApi.kt | 16 +- .../community/impl/nio/IjentNioFileChannel.kt | 7 +- .../intellij/openapi/vfs/impl/local/util.kt | 26 +- .../junit5/eel/src/impl/EelTestPosixApi.kt | 4 - .../junit5/eel/src/impl/EelTestWindowsApi.kt | 4 - 14 files changed, 882 insertions(+), 111 deletions(-) create mode 100644 platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelReadFileImpl.kt create mode 100644 platform/eel/gen-builders/com/intellij/platform/eel/fs/OpenForReadingArgsBuilder.kt create mode 100644 platform/eel/gen-builders/com/intellij/platform/eel/fs/ReadFileArgsBuilder.kt diff --git a/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelFsResultImpl.kt b/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelFsResultImpl.kt index 0fb9dac8e846..dfb2067ea9ba 100644 --- a/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelFsResultImpl.kt +++ b/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelFsResultImpl.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.platform.eel.impl.fs import com.intellij.platform.eel.EelResult @@ -15,10 +15,6 @@ object EelFsResultImpl { data class DiskInfoImpl(override val totalSpace: ULong, override val availableSpace: ULong) : EelFileSystemApi.DiskInfo - data class FullBytesReadImpl(override val bytes: ByteArray) : EelFileSystemApi.FullReadResult.Bytes - data class FullBytesReadOverflown(override val bytes: ByteArray) : EelFileSystemApi.FullReadResult.BytesOverflown - data object Overflow : EelFileSystemApi.FullReadResult.Overflow - data class Other(override val where: EelPath, override val message: String) : EelFileSystemApi.FileReaderError.Other, EelFileSystemApi.FileWriterError.Other, @@ -40,7 +36,6 @@ object EelFsResultImpl { EelFileSystemPosixApi.CreateSymbolicLinkError.Other, EelOpenedFile.CloseError.Other, EelOpenedFile.Writer.TruncateError.Other, - EelFileSystemApi.FullReadError.Other, EelFileSystemApi.WalkDirectoryError.Other data class DoesNotExist(override val where: EelPath, override val message: String) : @@ -51,7 +46,6 @@ object EelFsResultImpl { EelFileSystemApi.SameFileError.DoesNotExist, EelFileSystemApi.StatError.DoesNotExist, EelFileSystemApi.CanonicalizeError.DoesNotExist, - EelFileSystemApi.FullReadError.DoesNotExist, EelFileSystemApi.DiskInfoError.PathDoesNotExists, EelFileSystemPosixApi.CreateDirectoryError.ParentNotFound, EelFileSystemApi.CopyError.SourceDoesNotExist, @@ -81,7 +75,6 @@ object EelFsResultImpl { EelFileSystemApi.CopyError.PermissionDenied, EelFileSystemApi.MoveError.PermissionDenied, EelFileSystemPosixApi.CreateSymbolicLinkError.PermissionDenied, - EelFileSystemApi.FullReadError.PermissionDenied, EelFileSystemApi.WalkDirectoryError.Other data class NotDirectory(override val where: EelPath, override val message: String) : @@ -106,8 +99,7 @@ object EelFsResultImpl { EelFileSystemApi.FileWriterError.NotFile, EelFileSystemApi.SameFileError.NotFile, EelFileSystemApi.StatError.NotFile, - EelFileSystemApi.MoveError.TargetIsDirectory, - EelFileSystemApi.FullReadError.NotFile + EelFileSystemApi.MoveError.TargetIsDirectory data class InvalidValue(override val where: EelPath, override val message: String) : EelOpenedFile.Reader.ReadError.InvalidValue, @@ -147,4 +139,7 @@ object EelFsResultImpl { data class OffsetTooBig(override val where: EelPath, override val message: String) : EelOpenedFile.Writer.TruncateError.OffsetTooBig + + data class FileBiggerThanRequested(override val where: EelPath, override val message: String) : + EelFileSystemApi.FileReaderError.FileBiggerThanRequested } \ No newline at end of file diff --git a/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelReadFileImpl.kt b/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelReadFileImpl.kt new file mode 100644 index 000000000000..8799da657abe --- /dev/null +++ b/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/EelReadFileImpl.kt @@ -0,0 +1,253 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.platform.eel.impl.fs + +import com.intellij.openapi.components.Service +import com.intellij.openapi.components.serviceAsync +import com.intellij.openapi.diagnostic.logger +import com.intellij.platform.eel.EelResult +import com.intellij.platform.eel.ReadResult +import com.intellij.platform.eel.channels.EelDelicateApi +import com.intellij.platform.eel.fs.EelFileSystemApi +import com.intellij.platform.eel.fs.EelOpenedFile +import com.intellij.platform.eel.fs.openForReading +import com.intellij.platform.eel.getOr +import com.intellij.platform.eel.getOrThrow +import com.intellij.platform.eel.path.EelPath +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import java.nio.ByteBuffer +import kotlin.math.max + +@OptIn(EelDelicateApi::class) +suspend fun EelFileSystemApi.readFileImpl(args: EelFileSystemApi.ReadFileArgs): EelResult = + readFileImpl( + path = args.path, + initialBuffer = args.buffer, + mayReturnSameBuffer = args.mayReturnSameBuffer, + totalLimit = args.limit?.toUInt() ?: UInt.MAX_VALUE, + failFastIfBeyondLimit = args.failFastIfBeyondLimit, + ) + +@OptIn(EelDelicateApi::class) +private suspend fun EelFileSystemApi.readFileImpl( + path: EelPath, + initialBuffer: ByteBuffer?, + mayReturnSameBuffer: Boolean, + totalLimit: UInt, + failFastIfBeyondLimit: Boolean, +): EelResult { + val isTotalLimitSet = totalLimit != UInt.MAX_VALUE + val totalLimit = totalLimit.coerceAtMost(Int.MAX_VALUE.toUInt()).toInt() + var buffer: ByteBuffer = initialBuffer ?: ByteBuffer.allocate(totalLimit.coerceAtMost(RECOMMENDED_BUFFER_SIZE)) + buffer.position(0).limit(totalLimit.coerceAtMost(buffer.capacity())) + + val reader = openForReading(path) + .autoCloseAfterLastChunk(true) + .readFirstChunkInto(buffer) + .apply { + if (failFastIfBeyondLimit) { + closeImmediatelyIfFileBiggerThan(totalLimit.toLong()) + } + } + .eelIt() + .getOr { + return it + } + + if (reader.isClosed == true) { + return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = true) + } + + try { + if (buffer.limit() >= totalLimit) { + val fullyRead = reader.isClosed ?: readOneByteToCheckEof(reader, buffer).getOr { + return it + } + + return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = fullyRead) + } + + if (reader.isClosed == true) { + return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = true) + } + + buffer.position(buffer.limit()) + + var fullyRead = false + + mainLoop@ while (true) { + while (buffer.position() < totalLimit) { + buffer.limit( + totalLimit + .coerceAtMost(buffer.limit() + RECOMMENDED_BUFFER_SIZE) + .coerceAtMost(buffer.capacity()) + ) + if (buffer.remaining() < MINIMAL_EFFECTIVE_BUFFER_SIZE && buffer.capacity() < totalLimit) { + break + } + + val readResult = reader.read(buffer).getOr { + return EelFsResultImpl.Error(it.error.mapError()) + } + + when (readResult) { + ReadResult.EOF -> { + fullyRead = true + buffer.flip() + break@mainLoop + } + ReadResult.NOT_EOF -> Unit + } + } + + buffer.flip() + if (buffer.limit() >= totalLimit) { + break + } + + val newBuffer = ByteBuffer.allocate(run { + val candidate = newLength(buffer.capacity()) + if (totalLimit < candidate + MINIMAL_EFFECTIVE_BUFFER_SIZE) totalLimit + else candidate + }) + newBuffer.put(buffer) + buffer = newBuffer + } + + return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = fullyRead) + } + finally { + serviceAsync().closeSometimeLater(reader) + } +} + +private suspend fun readOneByteToCheckEof( + reader: EelOpenedFile.Reader, + existingBuffer: ByteBuffer, +): EelResult { + val (buffer, restoreValues) = + if (existingBuffer.limit() == existingBuffer.capacity()) { + ByteBuffer.allocate(1) to false + } + else { + existingBuffer to true + } + + if (restoreValues) { + buffer.position(buffer.limit()) + buffer.limit(buffer.limit() + 1) + } + + val oldPosition = buffer.position() + reader.read(buffer).getOr { + return EelFsResultImpl.Error(it.error.mapError()) + } + val fullyRead = buffer.position() == oldPosition + + if (restoreValues) { + buffer.position(0) + buffer.limit(buffer.limit() - 1) + } + + return EelFsResultImpl.Ok(fullyRead) +} + +private fun prepareResult( + initialBuffer: ByteBuffer?, + buffer: ByteBuffer, + mayReturnSameBuffer: Boolean, + fullyRead: Boolean, +): EelFsResultImpl.Ok = + EelFsResultImpl.Ok(object : EelFileSystemApi.ReadFileResult { + override val bytes: ByteBuffer = + if (buffer === initialBuffer && !mayReturnSameBuffer) + ByteBuffer.allocate(buffer.limit()).put(buffer).flip() + else + buffer + + override val fullyRead: Boolean = fullyRead + }) + +/** + * The algorithm tries to not allocate buffers smaller than this threshold. + * + * It's useful to have such buffers, but the size doesn't matter. + * Here is a statistic about the sizes of all files in a personal checkout of the IntelliJ monorepo, + * including various build caches for Java and Rust projects. + * This constant is used only when a file can't be read with a single read operation. + * Hence, files smaller than 131,072 bytes are not considered. + * Hypothetically, fine-tuning of this constant can spare one read operation. + * Considering remainders of division by 131,072: if the distribution had peaks or falls, + * a special value of this constant could cover them. + * But there's only one peak: for value 0, which requires no specific optimization. + * Without 0, the distribution of this particular data is almost flat. + * + * Therefore, since any value passes, 16 * 1024 is chosen just because the author finds it attractive. + */ +private const val MINIMAL_EFFECTIVE_BUFFER_SIZE = 16 * 1024 + +/** + * 1. It's equal to `com.intellij.platform.ijent.spi.IjentSpiConstKt.RECOMMENDED_MAX_PACKET_SIZE`. + * Better to keep them in sync, at least for not allocating excessive buffers, + * but nothing should break down if they diverge. + * 2. It's big enough. + * Here is a statistic about the sizes of all files in a personal checkout of the IntelliJ monorepo, + * including various build caches for Java and Rust projects. + * * Total size = 182,106,998,688 + * * File count = 2,922,717 + * * Median size = 1369 + * * P90 = 11,579 + * * P95 = 30,480 + * * P97 = 80,104 + * * P98 = 138,401 + * * P999 = 8,388,608 + * * Max size = 27,239,038,774 -- if you're curious, it's one of `.git/objects/pack/pack-*.pack` + * + * This buffer is enough to read more than 97% of all files with a single read operation. + */ +private const val RECOMMENDED_BUFFER_SIZE = 131_072 + +/** A copy-paste of `jdk.internal.util.ArraysSupport.SOFT_MAX_ARRAY_LENGTH` */ +private const val SOFT_MAX_ARRAY_LENGTH = Int.MAX_VALUE - 8 + +/** A stripped copy-paste of `jdk.internal.util.ArraysSupport.newLength` */ +private fun newLength(oldLength: Int): Int { + val prefLength: Long = oldLength.toLong() + max(RECOMMENDED_BUFFER_SIZE, oldLength) + if (prefLength <= SOFT_MAX_ARRAY_LENGTH) { + return prefLength.toInt() + } + else { + // put code cold in a separate method + return hugeLength(oldLength) + } +} + +/** A copy-paste of `jdk.internal.util.ArraysSupport.hugeLength` */ +private fun hugeLength(oldLength: Int): Int { + val minLength: Long = oldLength.toLong() + RECOMMENDED_BUFFER_SIZE + if (minLength >= Int.MAX_VALUE) { // overflow + throw OutOfMemoryError( + "Required array length $oldLength + $RECOMMENDED_BUFFER_SIZE is too large") + } + return minLength.toInt().coerceAtLeast(SOFT_MAX_ARRAY_LENGTH) +} + +private fun EelOpenedFile.Reader.ReadError.mapError(): EelFileSystemApi.FileReaderError = + when (this) { + is EelOpenedFile.Reader.ReadError.UnknownFile, is EelOpenedFile.Reader.ReadError.InvalidValue -> EelFsResultImpl.Other(where, toString()) + is EelOpenedFile.Reader.ReadError.Other -> EelFsResultImpl.Other(where, message) + } + +@Service +private class AsyncCloser(private val coroutineScope: CoroutineScope) { + fun closeSometimeLater(reader: EelOpenedFile.Reader) { + coroutineScope.launch { + try { + reader.close().getOrThrow() + } + catch (err: Exception) { + logger().info("Failed to close $reader", err) + } + } + } +} \ No newline at end of file diff --git a/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/LocalEelFileSystemApi.kt b/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/LocalEelFileSystemApi.kt index 6b8a5444484b..5431991b17d3 100644 --- a/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/LocalEelFileSystemApi.kt +++ b/platform/eel-impl/src/com/intellij/platform/eel/impl/fs/LocalEelFileSystemApi.kt @@ -7,6 +7,7 @@ import com.intellij.platform.eel.EelResult import com.intellij.platform.eel.EelUserPosixInfo import com.intellij.platform.eel.EelUserWindowsInfo import com.intellij.platform.eel.ReadResult +import com.intellij.platform.eel.channels.EelDelicateApi import com.intellij.platform.eel.fs.* import com.intellij.platform.eel.fs.EelFileSystemApi.FileWriterCreationMode.* import com.intellij.platform.eel.path.EelPath @@ -30,12 +31,8 @@ import java.nio.file.attribute.* import java.time.Instant import java.time.ZoneId import java.time.ZonedDateTime -import kotlin.io.path.exists -import kotlin.io.path.fileAttributesView -import kotlin.io.path.fileStore -import kotlin.io.path.isDirectory -import kotlin.io.path.listDirectoryEntries -import kotlin.io.path.pathString +import java.util.concurrent.atomic.AtomicReference +import kotlin.io.path.* import kotlin.streams.asSequence abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : EelFileSystemApi { @@ -51,10 +48,16 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : } inline fun wrapIntoEelResult(body: () -> T): EelResult = + wrapIntoEelResult(null, body) + + inline fun wrapIntoEelResult(isClosed: AtomicReference?, body: () -> T): EelResult = try { - EelFsResultImpl.Ok(body()) + val result = EelFsResultImpl.Ok(body()) + isClosed?.updateAndGet { it ?: false } + result } catch (err: FileSystemException) { + isClosed?.set(true) val path = try { EelPath.parse(err.file.toString(), LocalEelDescriptor) @@ -122,15 +125,58 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : Files.isSameFile(source.toNioPath(), target.toNioPath()) } - override suspend fun openForReading(path: EelPath): EelResult< + @OptIn(EelDelicateApi::class) + override suspend fun openForReading(args: EelFileSystemApi.OpenForReadingArgs): EelResult< EelOpenedFile.Reader, EelFileSystemApi.FileReaderError - > = - wrapIntoEelResult { - val nioPath = path.toNioPath() - val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, setOf(StandardOpenOption.READ)) - LocalEelOpenedFileReader(this, byteChannel, path) + > { + val result = openForReadingImpl( + path = args.path, + autoCloseAfterLastChunk = args.autoCloseAfterLastChunk, + closeImmediatelyIfFileBiggerThan = args.closeImmediatelyIfFileBiggerThan, + ) + // Not filling the buffer. This optimization makes no sense for the local case. + args.readFirstChunkInto?.run { + position(0) + limit(0) } + return result + } + + @OptIn(EelDelicateApi::class) + private fun openForReadingImpl( + path: EelPath, + autoCloseAfterLastChunk: Boolean, + closeImmediatelyIfFileBiggerThan: Long?, + ): EelResult = wrapIntoEelResult { + val nioPath = path.toNioPath() + val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, setOf(StandardOpenOption.READ)) + val isClosed = AtomicReference(null) + + if (closeImmediatelyIfFileBiggerThan != null) { + if (byteChannel.size() > closeImmediatelyIfFileBiggerThan) { + isClosed.set(true) + try { + doClose(this@NioBasedEelFileSystemApi, byteChannel, path) + } + catch (_: IOException) { + // Ignored. + } + return EelFsResultImpl.Error(EelFsResultImpl.FileBiggerThanRequested(path, "The file is bigger than the requested size")) + } + } + + LocalEelOpenedFileReader( + eelFs = this, + byteChannel = byteChannel, + path_ = path, + isClosed_ = isClosed, + autoCloseAfterLastChunk = autoCloseAfterLastChunk, + ) + } + + override suspend fun readFile(args: EelFileSystemApi.ReadFileArgs): EelResult = + readFileImpl(args) override suspend fun openForWriting(options: EelFileSystemApi.WriteOptions): EelResult< EelOpenedFile.Writer, @@ -140,7 +186,7 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : val nioPath = path.toNioPath() val nioOptions = writeOptionsToNioOptions(options) val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, nioOptions) - LocalEelOpenedFileWriter(this, byteChannel, path) + LocalEelOpenedFileWriter(this, byteChannel, path, AtomicReference(null)) } private fun writeOptionsToNioOptions(options: EelFileSystemApi.WriteOptions): MutableSet { @@ -168,15 +214,16 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : val nioOptions = writeOptionsToNioOptions(options) nioOptions += StandardOpenOption.READ val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, nioOptions) - object : EelOpenedFile.ReaderWriter, EelOpenedFile.Writer by LocalEelOpenedFileWriter(this, byteChannel, path) { + val isClosed = AtomicReference(null) + object : EelOpenedFile.ReaderWriter, EelOpenedFile.Writer by LocalEelOpenedFileWriter(this, byteChannel, path, isClosed) { override suspend fun read(buf: ByteBuffer): EelResult = - doRead(this@NioBasedEelFileSystemApi, byteChannel, buf) + doRead(this@NioBasedEelFileSystemApi, byteChannel, buf, isClosed, autoCloseAfterLastChunk = false) override suspend fun read( buf: ByteBuffer, offset: Long, ): EelResult = - doRead(this@NioBasedEelFileSystemApi, byteChannel, offset, buf) + doRead(this@NioBasedEelFileSystemApi, byteChannel, offset, buf, isClosed) } } @@ -231,32 +278,40 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : EelPathUtils.walkingTransfer(sourceNioPath, targetNioPath, removeSource = true, copyAttributes = true) } } - } +} +@EelDelicateApi internal class LocalEelOpenedFileReader( private val eelFs: NioBasedEelFileSystemApi, private val byteChannel: SeekableByteChannel, private val path_: EelPath, + private val isClosed_: AtomicReference, + private val autoCloseAfterLastChunk: Boolean, ) : EelOpenedFile.Reader { + override val isClosed: Boolean? + get() = isClosed_.get() + override suspend fun read(buf: ByteBuffer): EelResult = - doRead(eelFs, byteChannel, buf) + doRead(eelFs, byteChannel, buf, isClosed_, autoCloseAfterLastChunk) override suspend fun read(buf: ByteBuffer, offset: Long): EelResult< ReadResult, EelOpenedFile.Reader.ReadError > = - doRead(eelFs, byteChannel, offset, buf) + doRead(eelFs, byteChannel, offset, buf, isClosed_) override val path: EelPath = path_ - override suspend fun close(): EelResult = - doClose(eelFs, byteChannel, path_) + override suspend fun close(): EelResult { + isClosed_.set(true) + return doClose(eelFs, byteChannel, path_) + } override suspend fun tell(): EelResult = - doTell(eelFs, byteChannel, path_) + doTell(eelFs, byteChannel, isClosed_) override suspend fun seek(offset: Long, whence: EelOpenedFile.SeekWhence): EelResult = - doSeek(eelFs, byteChannel, path_, whence, offset) + doSeek(eelFs, byteChannel, path_, whence, offset, isClosed_) override suspend fun stat(): EelResult = eelFs.stat(path_, EelFileSystemApi.SymlinkPolicy.RESOLVE_AND_FOLLOW) @@ -266,14 +321,19 @@ private class LocalEelOpenedFileWriter( private val eelFs: NioBasedEelFileSystemApi, private val byteChannel: SeekableByteChannel, private val path_: EelPath, + private val isClosed_: AtomicReference, ) : EelOpenedFile.Writer { + @EelDelicateApi + override val isClosed: Boolean? + get() = isClosed_.get() + override suspend fun write(buf: ByteBuffer): EelResult = - eelFs.wrapIntoEelResult { + eelFs.wrapIntoEelResult(isClosed_) { byteChannel.write(buf) } override suspend fun write(buf: ByteBuffer, pos: Long): EelResult = - eelFs.wrapIntoEelResult { + eelFs.wrapIntoEelResult(isClosed_) { val oldPosition = byteChannel.position() byteChannel.position(pos) val written = byteChannel.write(buf) @@ -286,20 +346,22 @@ private class LocalEelOpenedFileWriter( } override suspend fun truncate(size: Long): EelResult = - eelFs.wrapIntoEelResult { + eelFs.wrapIntoEelResult(isClosed_) { byteChannel.truncate(size) } override val path: EelPath = path_ - override suspend fun close(): EelResult = - doClose(eelFs, byteChannel, path_) + override suspend fun close(): EelResult { + isClosed_.set(true) + return doClose(eelFs, byteChannel, path_) + } override suspend fun tell(): EelResult = - doTell(eelFs, byteChannel, path_) + doTell(eelFs, byteChannel, isClosed_) override suspend fun seek(offset: Long, whence: EelOpenedFile.SeekWhence): EelResult = - doSeek(eelFs, byteChannel, path_, whence, offset) + doSeek(eelFs, byteChannel, path_, whence, offset, isClosed_) override suspend fun stat(): EelResult = eelFs.stat(path_, EelFileSystemApi.SymlinkPolicy.RESOLVE_AND_FOLLOW) @@ -309,11 +371,19 @@ private fun doRead( eelFs: NioBasedEelFileSystemApi, byteChannel: SeekableByteChannel, buf: ByteBuffer, + isClosed: AtomicReference, + autoCloseAfterLastChunk: Boolean, ): EelResult = - eelFs.wrapIntoEelResult { + eelFs.wrapIntoEelResult(isClosed) { val read = byteChannel.read(buf) - - ReadResult.fromNumberOfReadBytes(read) + val result = ReadResult.fromNumberOfReadBytes(read) + if (autoCloseAfterLastChunk) { + when (result) { + ReadResult.EOF -> isClosed.set(true) + ReadResult.NOT_EOF -> isClosed.set(false) + } + } + result } private fun doRead( @@ -321,8 +391,9 @@ private fun doRead( byteChannel: SeekableByteChannel, offset: Long, buf: ByteBuffer, + isClosed: AtomicReference, ): EelResult = - eelFs.wrapIntoEelResult { + eelFs.wrapIntoEelResult(isClosed) { val oldPosition = byteChannel.position() byteChannel.position(offset) val read = byteChannel.read(buf) @@ -337,7 +408,8 @@ private fun doSeek( path: EelPath, whence: EelOpenedFile.SeekWhence, offset: Long, -): EelResult = eelFs.wrapIntoEelResult { + isClosed: AtomicReference, +): EelResult = eelFs.wrapIntoEelResult(isClosed) { val newPosition = when (whence) { EelOpenedFile.SeekWhence.START -> offset EelOpenedFile.SeekWhence.CURRENT -> byteChannel.position() + offset @@ -347,8 +419,12 @@ private fun doSeek( newPosition } -private fun doTell(eelFs: NioBasedEelFileSystemApi, byteChannel: SeekableByteChannel, path: EelPath): EelResult = - eelFs.wrapIntoEelResult { +private fun doTell( + eelFs: NioBasedEelFileSystemApi, + byteChannel: SeekableByteChannel, + isClosed: AtomicReference, +): EelResult = + eelFs.wrapIntoEelResult(isClosed) { byteChannel.position().toLong() } @@ -424,10 +500,6 @@ abstract class PosixNioBasedEelFileSystemApi( Files.createSymbolicLink(linkPath.toNioPath(), targetPath) } - override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult { - TODO("Not yet implemented") - } - override suspend fun walkDirectory(options: EelFileSystemApi.WalkDirectoryOptions): Flow = flow { val rootDir = options.path.asNioPath() @@ -514,7 +586,7 @@ abstract class PosixNioBasedEelFileSystemApi( currentItem: Path, sourceAttrs: PosixFileAttributes, emptyFileHash: Long, - options: EelFileSystemApi.WalkDirectoryOptions + options: EelFileSystemApi.WalkDirectoryOptions, ): WalkDirectoryEntryResult? { var creationTime: ZonedDateTime? = null var lastModifiedTime: ZonedDateTime? = null @@ -656,10 +728,6 @@ abstract class WindowsNioBasedEelFileSystemApi( // TODO File permissions for windows. } - override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult { - TODO("Not yet implemented") - } - override suspend fun walkDirectory(options: EelFileSystemApi.WalkDirectoryOptions): Flow = flow { val rootDir = options.path.asNioPath() @@ -746,7 +814,7 @@ abstract class WindowsNioBasedEelFileSystemApi( currentItem: Path, sourceAttrs: DosFileAttributes, emptyFileHash: Long, - options: EelFileSystemApi.WalkDirectoryOptions + options: EelFileSystemApi.WalkDirectoryOptions, ): WalkDirectoryEntryResult? { var creationTime: ZonedDateTime? = null var lastModifiedTime: ZonedDateTime? = null diff --git a/platform/eel-provider/src/com/intellij/platform/eel/provider/utils/eelUtils.kt b/platform/eel-provider/src/com/intellij/platform/eel/provider/utils/eelUtils.kt index b931570a2901..aff899c6d3ef 100644 --- a/platform/eel-provider/src/com/intellij/platform/eel/provider/utils/eelUtils.kt +++ b/platform/eel-provider/src/com/intellij/platform/eel/provider/utils/eelUtils.kt @@ -2,6 +2,7 @@ package com.intellij.platform.eel.provider.utils import com.intellij.openapi.progress.runBlockingMaybeCancellable +import com.intellij.openapi.util.io.FileTooBigException import com.intellij.platform.eel.EelExecApi import com.intellij.platform.eel.EelPlatform import com.intellij.platform.eel.EelResult @@ -87,6 +88,8 @@ fun EelFsError.throwFileSystemException(): Nothing { -> throw IllegalArgumentException(message) is EelOpenedFile.Writer.WriteError.InvalidValue -> throw IllegalArgumentException(message) is EelFileSystemApi.DeleteError.UnresolvedLink -> throw FileSystemException(where.toString(), null, message) + is EelFileSystemApi.FileReaderError.FileBiggerThanRequested -> + throw FileSystemException(where.toString(), null, "File is too big").apply { initCause(FileTooBigException("File is too big")) } is EelFsError.Other -> FileSystemException(where.toString(), null, message.nullize()) } } diff --git a/platform/eel/gen-builders/com/intellij/platform/eel/fs/EelFileSystemApiHelpers.kt b/platform/eel/gen-builders/com/intellij/platform/eel/fs/EelFileSystemApiHelpers.kt index cef49bd07e0d..330ac9a719bb 100644 --- a/platform/eel/gen-builders/com/intellij/platform/eel/fs/EelFileSystemApiHelpers.kt +++ b/platform/eel/gen-builders/com/intellij/platform/eel/fs/EelFileSystemApiHelpers.kt @@ -7,6 +7,7 @@ package com.intellij.platform.eel.fs import com.intellij.platform.eel.EelResult import com.intellij.platform.eel.GeneratedBuilder import com.intellij.platform.eel.OwnedBuilder +import com.intellij.platform.eel.channels.EelDelicateApi import com.intellij.platform.eel.fs.EelFileInfo.Permissions import com.intellij.platform.eel.fs.EelFileSystemApi.FileChangeType import com.intellij.platform.eel.fs.EelFileSystemApi.FileWriterCreationMode @@ -23,6 +24,7 @@ import com.intellij.platform.eel.path.EelPath import kotlinx.coroutines.flow.Flow import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.CheckReturnValue +import java.nio.ByteBuffer @GeneratedBuilder.Result @@ -91,6 +93,39 @@ fun EelFileSystemApi.move( target = target, ) +/** + * Opens the file only for reading. + * + * In many cases [readFile] suits better than [openForReading]. + */ +@GeneratedBuilder.Result +@ApiStatus.Internal +fun EelFileSystemApi.openForReading( + path: EelPath, +): EelFileSystemApiHelpers.OpenForReading = + EelFileSystemApiHelpers.OpenForReading( + owner = this, + path = path, + ) + +/** + * Fully or partially reads the file. + * + * Although it's possible to implement file reading with [openForReading], + * this function is optimized and covered with tests. + * + * The returned [ReadFileResult.bytes] is prepared for reading. + */ +@GeneratedBuilder.Result +@ApiStatus.Internal +fun EelFileSystemApi.readFile( + path: EelPath, +): EelFileSystemApiHelpers.ReadFile = + EelFileSystemApiHelpers.ReadFile( + owner = this, + path = path, + ) + /** * Similar to stat(2) and lstat(2). [symlinkPolicy] has an impact only on [EelFileInfo.type] if [path] points on a symlink. */ @@ -494,6 +529,156 @@ object EelFileSystemApiHelpers { ) } + /** + * Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.openForReading]. + */ + @GeneratedBuilder.Result + @ApiStatus.Internal + class OpenForReading( + private val owner: EelFileSystemApi, + private var path: EelPath, + ) : OwnedBuilder> { + private var autoCloseAfterLastChunk: Boolean = false + + private var closeImmediatelyIfFileBiggerThan: Long? = null + + private var readFirstChunkInto: ByteBuffer? = null + + /** + * When specified, the implementation closes its internal file descriptor + * as soon as it internally reaches the end of the file. + * + * There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read] + * * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call. + * * By checking inexpensive but unreliable [EelOpenedFile.isClosed]. + */ + @EelDelicateApi + fun autoCloseAfterLastChunk(arg: Boolean): OpenForReading = apply { + this.autoCloseAfterLastChunk = arg + } + + /** + * An optimization suitable for reading into memory. + * It allows aborting the reading fast if the whole file content + * won't fit into some buffer. + * + * If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested]. + */ + @EelDelicateApi + fun closeImmediatelyIfFileBiggerThan(arg: Long?): OpenForReading = apply { + this.closeImmediatelyIfFileBiggerThan = arg + } + + fun path(arg: EelPath): OpenForReading = apply { + this.path = arg + } + + /** + * When specified, data from the file MAY be written into this buffer. + * [ByteBuffer.position] and [ByteBuffer.limit] are always changed. + * The buffer is prepared for reading after the call, + * so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading]. + * + * If some data is written into this buffer, + * the first call of [EelOpenedFile.Reader.read] reads the data following this buffer. + */ + @EelDelicateApi + fun readFirstChunkInto(arg: ByteBuffer?): OpenForReading = apply { + this.readFirstChunkInto = arg + } + + /** + * Complete the builder and call [com.intellij.platform.eel.fs.EelFileSystemApi.openForReading] + * with an instance of [com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs]. + */ + @CheckReturnValue + override suspend fun eelIt(): EelResult = + owner.openForReading( + OpenForReadingArgsImpl( + autoCloseAfterLastChunk = autoCloseAfterLastChunk, + closeImmediatelyIfFileBiggerThan = closeImmediatelyIfFileBiggerThan, + path = path, + readFirstChunkInto = readFirstChunkInto, + ) + ) + } + + /** + * Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.readFile]. + */ + @GeneratedBuilder.Result + @ApiStatus.Internal + class ReadFile( + private val owner: EelFileSystemApi, + private var path: EelPath, + ) : OwnedBuilder> { + private var buffer: ByteBuffer? = null + + private var failFastIfBeyondLimit: Boolean = false + + private var limit: Int? = null + + private var mayReturnSameBuffer: Boolean = true + + /** + * Use some specific buffer for reading files instead of creating a temporary buffer. + * + * The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call. + * + * The buffer is ready for reading, no need to call `flip`. + */ + @EelDelicateApi + fun buffer(arg: ByteBuffer?): ReadFile = apply { + this.buffer = arg + } + + /** + * If this flag is set, the implementation checks the file size before trying to read data, + * and if the file is certainly bigger than [limit], no data is read. + */ + @EelDelicateApi + fun failFastIfBeyondLimit(arg: Boolean): ReadFile = apply { + this.failFastIfBeyondLimit = arg + } + + /** + * Maximal number of bytes to read. + */ + fun limit(arg: Int?): ReadFile = apply { + this.limit = arg + } + + /** + * If this flag is set and [buffer] is specified, the implementation reads the whole + * file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer. + * + * If [buffer] is not specified, the value of the flag is ignored. + */ + @EelDelicateApi + fun mayReturnSameBuffer(arg: Boolean): ReadFile = apply { + this.mayReturnSameBuffer = arg + } + + fun path(arg: EelPath): ReadFile = apply { + this.path = arg + } + + /** + * Complete the builder and call [com.intellij.platform.eel.fs.EelFileSystemApi.readFile] + * with an instance of [com.intellij.platform.eel.fs.EelFileSystemApi.ReadFileArgs]. + */ + override suspend fun eelIt(): EelResult = + owner.readFile( + ReadFileArgsImpl( + buffer = buffer, + failFastIfBeyondLimit = failFastIfBeyondLimit, + limit = limit, + mayReturnSameBuffer = mayReturnSameBuffer, + path = path, + ) + ) + } + /** * Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.stat]. */ diff --git a/platform/eel/gen-builders/com/intellij/platform/eel/fs/OpenForReadingArgsBuilder.kt b/platform/eel/gen-builders/com/intellij/platform/eel/fs/OpenForReadingArgsBuilder.kt new file mode 100644 index 000000000000..b539454354b2 --- /dev/null +++ b/platform/eel/gen-builders/com/intellij/platform/eel/fs/OpenForReadingArgsBuilder.kt @@ -0,0 +1,84 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +/** + * This file is generated by [com.intellij.platform.eel.codegen.BuildersGeneratorTest]. + */ +package com.intellij.platform.eel.fs + +import com.intellij.platform.eel.GeneratedBuilder +import com.intellij.platform.eel.channels.EelDelicateApi +import com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs +import com.intellij.platform.eel.path.EelPath +import org.jetbrains.annotations.ApiStatus +import java.nio.ByteBuffer + + +@GeneratedBuilder.Result +@ApiStatus.Internal +class OpenForReadingArgsBuilder( + private var path: EelPath, +) { + private var autoCloseAfterLastChunk: Boolean = false + + private var closeImmediatelyIfFileBiggerThan: Long? = null + + private var readFirstChunkInto: ByteBuffer? = null + + /** + * When specified, the implementation closes its internal file descriptor + * as soon as it internally reaches the end of the file. + * + * There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read] + * * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call. + * * By checking inexpensive but unreliable [EelOpenedFile.isClosed]. + */ + @EelDelicateApi + fun autoCloseAfterLastChunk(arg: Boolean): OpenForReadingArgsBuilder = apply { + this.autoCloseAfterLastChunk = arg + } + + /** + * An optimization suitable for reading into memory. + * It allows aborting the reading fast if the whole file content + * won't fit into some buffer. + * + * If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested]. + */ + @EelDelicateApi + fun closeImmediatelyIfFileBiggerThan(arg: Long?): OpenForReadingArgsBuilder = apply { + this.closeImmediatelyIfFileBiggerThan = arg + } + + fun path(arg: EelPath): OpenForReadingArgsBuilder = apply { + this.path = arg + } + + /** + * When specified, data from the file MAY be written into this buffer. + * [ByteBuffer.position] and [ByteBuffer.limit] are always changed. + * The buffer is prepared for reading after the call, + * so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading]. + * + * If some data is written into this buffer, + * the first call of [EelOpenedFile.Reader.read] reads the data following this buffer. + */ + @EelDelicateApi + fun readFirstChunkInto(arg: ByteBuffer?): OpenForReadingArgsBuilder = apply { + this.readFirstChunkInto = arg + } + + fun build(): OpenForReadingArgs = + OpenForReadingArgsImpl( + autoCloseAfterLastChunk = autoCloseAfterLastChunk, + closeImmediatelyIfFileBiggerThan = closeImmediatelyIfFileBiggerThan, + path = path, + readFirstChunkInto = readFirstChunkInto, + ) +} + +@GeneratedBuilder.Result +internal class OpenForReadingArgsImpl( + override val autoCloseAfterLastChunk: Boolean, + override val closeImmediatelyIfFileBiggerThan: Long?, + override val path: EelPath, + override val readFirstChunkInto: ByteBuffer?, +) : OpenForReadingArgs \ No newline at end of file diff --git a/platform/eel/gen-builders/com/intellij/platform/eel/fs/ReadFileArgsBuilder.kt b/platform/eel/gen-builders/com/intellij/platform/eel/fs/ReadFileArgsBuilder.kt new file mode 100644 index 000000000000..73f1a5ce51e5 --- /dev/null +++ b/platform/eel/gen-builders/com/intellij/platform/eel/fs/ReadFileArgsBuilder.kt @@ -0,0 +1,88 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +/** + * This file is generated by [com.intellij.platform.eel.codegen.BuildersGeneratorTest]. + */ +package com.intellij.platform.eel.fs + +import com.intellij.platform.eel.GeneratedBuilder +import com.intellij.platform.eel.channels.EelDelicateApi +import com.intellij.platform.eel.fs.EelFileSystemApi.ReadFileArgs +import com.intellij.platform.eel.path.EelPath +import org.jetbrains.annotations.ApiStatus +import java.nio.ByteBuffer + + +@GeneratedBuilder.Result +@ApiStatus.Internal +class ReadFileArgsBuilder( + private var path: EelPath, +) { + private var buffer: ByteBuffer? = null + + private var failFastIfBeyondLimit: Boolean = false + + private var limit: Int? = null + + private var mayReturnSameBuffer: Boolean = true + + /** + * Use some specific buffer for reading files instead of creating a temporary buffer. + * + * The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call. + * + * The buffer is ready for reading, no need to call `flip`. + */ + @EelDelicateApi + fun buffer(arg: ByteBuffer?): ReadFileArgsBuilder = apply { + this.buffer = arg + } + + /** + * If this flag is set, the implementation checks the file size before trying to read data, + * and if the file is certainly bigger than [limit], no data is read. + */ + @EelDelicateApi + fun failFastIfBeyondLimit(arg: Boolean): ReadFileArgsBuilder = apply { + this.failFastIfBeyondLimit = arg + } + + /** + * Maximal number of bytes to read. + */ + fun limit(arg: Int?): ReadFileArgsBuilder = apply { + this.limit = arg + } + + /** + * If this flag is set and [buffer] is specified, the implementation reads the whole + * file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer. + * + * If [buffer] is not specified, the value of the flag is ignored. + */ + @EelDelicateApi + fun mayReturnSameBuffer(arg: Boolean): ReadFileArgsBuilder = apply { + this.mayReturnSameBuffer = arg + } + + fun path(arg: EelPath): ReadFileArgsBuilder = apply { + this.path = arg + } + + fun build(): ReadFileArgs = + ReadFileArgsImpl( + buffer = buffer, + failFastIfBeyondLimit = failFastIfBeyondLimit, + limit = limit, + mayReturnSameBuffer = mayReturnSameBuffer, + path = path, + ) +} + +@GeneratedBuilder.Result +internal class ReadFileArgsImpl( + override val buffer: ByteBuffer?, + override val failFastIfBeyondLimit: Boolean, + override val limit: Int?, + override val mayReturnSameBuffer: Boolean, + override val path: EelPath, +) : ReadFileArgs \ No newline at end of file diff --git a/platform/eel/src/com/intellij/platform/eel/channels/EelLowLevelApi.kt b/platform/eel/src/com/intellij/platform/eel/channels/EelLowLevelApi.kt index 680e2422e975..563490af9b7a 100644 --- a/platform/eel/src/com/intellij/platform/eel/channels/EelLowLevelApi.kt +++ b/platform/eel/src/com/intellij/platform/eel/channels/EelLowLevelApi.kt @@ -10,4 +10,13 @@ import org.jetbrains.annotations.ApiStatus message = "This function might send only part of the buffer. Consider using `sendWholeBuffer` instead" ) @ApiStatus.Internal -annotation class EelSendApi \ No newline at end of file +annotation class EelSendApi + +@MustBeDocumented +@Retention(value = AnnotationRetention.BINARY) +@RequiresOptIn( + level = RequiresOptIn.Level.WARNING, + message = "Ensure that you read the documentation thoroughly or that you're ready to shoot your foot" +) +@ApiStatus.Internal +annotation class EelDelicateApi \ No newline at end of file diff --git a/platform/eel/src/com/intellij/platform/eel/fs/EelFileSystemApi.kt b/platform/eel/src/com/intellij/platform/eel/fs/EelFileSystemApi.kt index 41d8ede65431..defcb7754342 100644 --- a/platform/eel/src/com/intellij/platform/eel/fs/EelFileSystemApi.kt +++ b/platform/eel/src/com/intellij/platform/eel/fs/EelFileSystemApi.kt @@ -2,6 +2,7 @@ package com.intellij.platform.eel.fs import com.intellij.platform.eel.* +import com.intellij.platform.eel.channels.EelDelicateApi import com.intellij.platform.eel.fs.EelFileSystemApi.StatError import com.intellij.platform.eel.path.EelPath import kotlinx.coroutines.flow.Flow @@ -178,46 +179,120 @@ interface EelFileSystemApi { } /** - * Opens the file only for reading + * Opens the file only for reading. + * + * In many cases [readFile] suits better than [openForReading]. */ @CheckReturnValue - suspend fun openForReading(path: EelPath): EelResult< + suspend fun openForReading(@GeneratedBuilder args: OpenForReadingArgs): EelResult< EelOpenedFile.Reader, FileReaderError> + interface OpenForReadingArgs { + val path: EelPath + + /** + * When specified, data from the file MAY be written into this buffer. + * [ByteBuffer.position] and [ByteBuffer.limit] are always changed. + * The buffer is prepared for reading after the call, + * so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading]. + * + * If some data is written into this buffer, + * the first call of [EelOpenedFile.Reader.read] reads the data following this buffer. + */ + @EelDelicateApi + val readFirstChunkInto: ByteBuffer? get() = null + + /** + * When specified, the implementation closes its internal file descriptor + * as soon as it internally reaches the end of the file. + * + * There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read] + * * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call. + * * By checking inexpensive but unreliable [EelOpenedFile.isClosed]. + */ + @EelDelicateApi + val autoCloseAfterLastChunk: Boolean get() = false + + /** + * An optimization suitable for reading into memory. + * It allows aborting the reading fast if the whole file content + * won't fit into some buffer. + * + * If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested]. + */ + @EelDelicateApi + val closeImmediatelyIfFileBiggerThan: Long? get() = null + } + sealed interface FileReaderError : EelFsError { interface AlreadyExists : FileReaderError, EelFsError.AlreadyExists interface DoesNotExist : FileReaderError, EelFsError.DoesNotExist interface PermissionDenied : FileReaderError, EelFsError.PermissionDenied interface NotDirectory : FileReaderError, EelFsError.NotDirectory interface NotFile : FileReaderError, EelFsError.NotFile + + /** See [com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs.closeImmediatelyIfFileBiggerThan] */ + interface FileBiggerThanRequested : FileReaderError + interface Other : FileReaderError, EelFsError.Other } - enum class OverflowPolicy { - DROP, - RETAIN, + /** + * Fully or partially reads the file. + * + * Although it's possible to implement file reading with [openForReading], + * this function is optimized and covered with tests. + * + * The returned [ReadFileResult.bytes] is prepared for reading. + */ + suspend fun readFile(@GeneratedBuilder args: ReadFileArgs): EelResult + + interface ReadFileArgs { + val path: EelPath + + /** + * Maximal number of bytes to read. + */ + val limit: Int? get() = null + + /** + * If this flag is set, the implementation checks the file size before trying to read data, + * and if the file is certainly bigger than [limit], no data is read. + */ + @EelDelicateApi + val failFastIfBeyondLimit: Boolean get() = false + + /** + * Use some specific buffer for reading files instead of creating a temporary buffer. + * + * The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call. + * + * The buffer is ready for reading, no need to call `flip`. + */ + @EelDelicateApi + val buffer: ByteBuffer? get() = null + + /** + * If this flag is set and [buffer] is specified, the implementation reads the whole + * file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer. + * + * If [buffer] is not specified, the value of the flag is ignored. + */ + @EelDelicateApi + val mayReturnSameBuffer: Boolean get() = true } - sealed interface FullReadResult { - interface Overflow : FullReadResult - interface BytesOverflown : FullReadResult { - val bytes: ByteArray - } + interface ReadFileResult { + /** + * It's ready for reading, the position and the limit are already set. + */ + val bytes: ByteBuffer - interface Bytes : FullReadResult { - val bytes: ByteArray - } - } - - @CheckReturnValue - suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: OverflowPolicy): EelResult - - sealed interface FullReadError : EelFsError { - interface DoesNotExist : FullReadError, EelFsError.DoesNotExist - interface PermissionDenied : FullReadError, EelFsError.PermissionDenied - interface NotFile : FullReadError, EelFsError.NotFile - interface Other : FullReadError, EelFsError.Other + /** + * `true` if [bytes] contains the whole file, `false` only if the part of the file. + */ + val fullyRead: Boolean } interface WalkDirectoryOptions { @@ -754,6 +829,18 @@ sealed interface EelOpenedFile { @CheckReturnValue suspend fun close(): EelResult + /** + * This method is to be used for avoiding potentially excessive calls. + * However, rely on this function with suspicion: the implementation may return `null` whatever happens. + * + * Returns: + * * `true` if the file is closed. + * * `false` if it's not closed. + * * `null` if it's not possible to determine if the file is closed without calling any suspending method. + */ + @EelDelicateApi + val isClosed: Boolean? + sealed interface CloseError : EelFsError { interface Other : CloseError, EelFsError.Other } diff --git a/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/IjentFailSafeFileSystemPosixApi.kt b/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/IjentFailSafeFileSystemPosixApi.kt index f7a08e463e79..f8574071ebf5 100644 --- a/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/IjentFailSafeFileSystemPosixApi.kt +++ b/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/IjentFailSafeFileSystemPosixApi.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.platform.ijent.community.impl import com.intellij.platform.eel.EelDescriptor @@ -9,6 +9,7 @@ import com.intellij.platform.eel.fs.EelFileSystemApi import com.intellij.platform.eel.fs.EelFileSystemPosixApi import com.intellij.platform.eel.fs.EelOpenedFile import com.intellij.platform.eel.fs.EelPosixFileInfo +import com.intellij.platform.eel.fs.* import com.intellij.platform.eel.path.EelPath import com.intellij.platform.ijent.IjentApi import com.intellij.platform.ijent.IjentPosixApi @@ -164,15 +165,18 @@ private class IjentFailSafeFileSystemPosixApiImpl( } override suspend fun openForReading( - path: EelPath, + args: EelFileSystemApi.OpenForReadingArgs, ): EelResult = holder.withDelegateRetrying { - openForReading(path) + openForReading(args) } - override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult = holder.withDelegateRetrying { - readFully(path, limit, overflowPolicy) - } + override suspend fun readFile( + args: EelFileSystemApi.ReadFileArgs, + ): EelResult = + holder.withDelegateRetrying { + readFile(args) + } override suspend fun openForWriting( options: EelFileSystemApi.WriteOptions, diff --git a/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/nio/IjentNioFileChannel.kt b/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/nio/IjentNioFileChannel.kt index 1c74732cbbea..c71702d49a8d 100644 --- a/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/nio/IjentNioFileChannel.kt +++ b/platform/ijent/impl/src/com/intellij/platform/ijent/community/impl/nio/IjentNioFileChannel.kt @@ -1,13 +1,10 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.platform.ijent.community.impl.nio import com.intellij.openapi.diagnostic.logger import com.intellij.openapi.diagnostic.thisLogger import com.intellij.platform.eel.ReadResult -import com.intellij.platform.eel.fs.EelFileInfo -import com.intellij.platform.eel.fs.EelFileSystemApi -import com.intellij.platform.eel.fs.EelOpenedFile -import com.intellij.platform.eel.fs.EelPosixFileInfo +import com.intellij.platform.eel.fs.* import com.intellij.platform.eel.path.EelPath import com.intellij.platform.eel.provider.utils.getOrThrowFileSystemException import com.intellij.platform.ijent.spi.RECOMMENDED_MAX_PACKET_SIZE diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/util.kt b/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/util.kt index 72869f78ea02..22ae4690916f 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/util.kt +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/util.kt @@ -12,7 +12,7 @@ import com.intellij.openapi.vfs.impl.local.LocalFileSystemBase.LOG import com.intellij.openapi.vfs.impl.local.LocalFileSystemBase.toIoPath import com.intellij.openapi.vfs.limits.FileSizeLimit import com.intellij.platform.eel.EelApi -import com.intellij.platform.eel.LocalEelApi +import com.intellij.platform.eel.channels.EelDelicateApi import com.intellij.platform.eel.fs.* import com.intellij.platform.eel.getOr import com.intellij.platform.eel.path.EelPath @@ -26,6 +26,7 @@ import com.intellij.platform.ijent.community.impl.nio.IjentNioPosixFileAttribute import com.intellij.platform.ijent.community.impl.nio.fsBlocking import com.intellij.util.containers.CollectionFactory import com.intellij.util.containers.ContainerUtil +import com.intellij.util.io.toByteArray import kotlinx.coroutines.runBlocking import java.io.IOException import java.nio.file.* @@ -38,34 +39,39 @@ private val map = ContainerUtil.createConcurrentWeakMap() * This is unacceptable in the remote setting when each request to IO results in RPC. * Here we try to invoke a specialized function that can read all bytes from [path] in one request. */ +@OptIn(EelDelicateApi::class) @Suppress("RAW_RUN_BLOCKING") internal fun readWholeFileIfNotTooLargeWithEel(path: Path): ByteArray? { if (!Registry.`is`("vfs.try.eel.for.content.loading", false)) { return null } val root = path.root ?: return null + + // TODO Check if this if-else can be removed. The only reason why it's kept is to avoid possible performance degradations in hot code. val eelDescriptor = root.getEelDescriptor() if (eelDescriptor == LocalEelDescriptor) { return null } - val api = map.computeIfAbsent(root) { + + val api = map.computeIfAbsent(root) { // TODO Does this cache make sense nowadays? runBlocking { root.getEelDescriptor().toEelApi() } } - if (api is LocalEelApi) { - return null - } + val eelPath = path.asEelPath() val limit = FileSizeLimit.getContentLoadLimit(FileUtilRt.getExtension(path.fileName.toString())) - return runBlocking { - when (val res = api.fs.readFully(eelPath, limit.toULong(), EelFileSystemApi.OverflowPolicy.DROP).getOrThrowFileSystemException()) { - is EelFileSystemApi.FullReadResult.Bytes -> res.bytes - is EelFileSystemApi.FullReadResult.BytesOverflown -> error("Never returned") - is EelFileSystemApi.FullReadResult.Overflow -> throw FileTooBigException("File $path is bigger than $limit bytes") + val result = runBlocking { + try { + api.fs.readFile(eelPath).limit(limit).failFastIfBeyondLimit(true).getOrThrowFileSystemException() + } + catch (err: FileSystemException) { + throw err.cause.takeIf { it is FileTooBigException } ?: err } } + + return result.bytes.toByteArray() } internal fun toEelPath(parent: VirtualFile, childName: String): EelPath? = diff --git a/platform/testFramework/junit5/eel/src/impl/EelTestPosixApi.kt b/platform/testFramework/junit5/eel/src/impl/EelTestPosixApi.kt index 735735af10dd..520b6906be96 100644 --- a/platform/testFramework/junit5/eel/src/impl/EelTestPosixApi.kt +++ b/platform/testFramework/junit5/eel/src/impl/EelTestPosixApi.kt @@ -36,10 +36,6 @@ internal class EelTestPosixApi(override val descriptor: EelTestDescriptor, fileS private class EelTestFileSystemPosixApi(override val descriptor: EelTestDescriptor, fileSystem: EelUnitTestFileSystem) : PosixNioBasedEelFileSystemApi(fileSystem, EelTestPosixUserInfo(descriptor)) { - override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult { - TODO("Not yet implemented") - } - override suspend fun createTemporaryDirectory(options: EelFileSystemApi.CreateTemporaryEntryOptions): EelResult { return wrapIntoEelResult { val nioTempDir = Files.createTempDirectory(fs.rootDirectories.single(), options.prefix) diff --git a/platform/testFramework/junit5/eel/src/impl/EelTestWindowsApi.kt b/platform/testFramework/junit5/eel/src/impl/EelTestWindowsApi.kt index 0a120aae13a1..f7bc1e5e68da 100644 --- a/platform/testFramework/junit5/eel/src/impl/EelTestWindowsApi.kt +++ b/platform/testFramework/junit5/eel/src/impl/EelTestWindowsApi.kt @@ -30,10 +30,6 @@ internal class EelTestWindowsApi(override val descriptor: EelTestDescriptor, fil private class EelTestFileSystemWindowsApi(override val descriptor: EelDescriptor, fileSystem: EelUnitTestFileSystem) : WindowsNioBasedEelFileSystemApi(fileSystem, EelTestWindowsUserInfo(descriptor)) { - override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult { - TODO("Not yet implemented") - } - override suspend fun createTemporaryDirectory(options: EelFileSystemApi.CreateTemporaryEntryOptions): EelResult { return wrapIntoEelResult { val nioTempDir = Files.createTempDirectory(options.prefix)