IJPL-201569 IJent: reimplement function for reading whole files to not require a special channel

GitOrigin-RevId: 31c89d8e1ef66de5b59652813c8466c994802c53
This commit is contained in:
Vladimir Lagunov
2025-09-30 22:50:34 +00:00
committed by intellij-monorepo-bot
parent dab9db55b9
commit 2f2a86bc38
14 changed files with 882 additions and 111 deletions
@@ -1,4 +1,4 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.eel.impl.fs
import com.intellij.platform.eel.EelResult
@@ -15,10 +15,6 @@ object EelFsResultImpl {
data class DiskInfoImpl(override val totalSpace: ULong, override val availableSpace: ULong) : EelFileSystemApi.DiskInfo
data class FullBytesReadImpl(override val bytes: ByteArray) : EelFileSystemApi.FullReadResult.Bytes
data class FullBytesReadOverflown(override val bytes: ByteArray) : EelFileSystemApi.FullReadResult.BytesOverflown
data object Overflow : EelFileSystemApi.FullReadResult.Overflow
data class Other(override val where: EelPath, override val message: String) :
EelFileSystemApi.FileReaderError.Other,
EelFileSystemApi.FileWriterError.Other,
@@ -40,7 +36,6 @@ object EelFsResultImpl {
EelFileSystemPosixApi.CreateSymbolicLinkError.Other,
EelOpenedFile.CloseError.Other,
EelOpenedFile.Writer.TruncateError.Other,
EelFileSystemApi.FullReadError.Other,
EelFileSystemApi.WalkDirectoryError.Other
data class DoesNotExist(override val where: EelPath, override val message: String) :
@@ -51,7 +46,6 @@ object EelFsResultImpl {
EelFileSystemApi.SameFileError.DoesNotExist,
EelFileSystemApi.StatError.DoesNotExist,
EelFileSystemApi.CanonicalizeError.DoesNotExist,
EelFileSystemApi.FullReadError.DoesNotExist,
EelFileSystemApi.DiskInfoError.PathDoesNotExists,
EelFileSystemPosixApi.CreateDirectoryError.ParentNotFound,
EelFileSystemApi.CopyError.SourceDoesNotExist,
@@ -81,7 +75,6 @@ object EelFsResultImpl {
EelFileSystemApi.CopyError.PermissionDenied,
EelFileSystemApi.MoveError.PermissionDenied,
EelFileSystemPosixApi.CreateSymbolicLinkError.PermissionDenied,
EelFileSystemApi.FullReadError.PermissionDenied,
EelFileSystemApi.WalkDirectoryError.Other
data class NotDirectory(override val where: EelPath, override val message: String) :
@@ -106,8 +99,7 @@ object EelFsResultImpl {
EelFileSystemApi.FileWriterError.NotFile,
EelFileSystemApi.SameFileError.NotFile,
EelFileSystemApi.StatError.NotFile,
EelFileSystemApi.MoveError.TargetIsDirectory,
EelFileSystemApi.FullReadError.NotFile
EelFileSystemApi.MoveError.TargetIsDirectory
data class InvalidValue(override val where: EelPath, override val message: String) :
EelOpenedFile.Reader.ReadError.InvalidValue,
@@ -147,4 +139,7 @@ object EelFsResultImpl {
data class OffsetTooBig(override val where: EelPath, override val message: String) :
EelOpenedFile.Writer.TruncateError.OffsetTooBig
data class FileBiggerThanRequested(override val where: EelPath, override val message: String) :
EelFileSystemApi.FileReaderError.FileBiggerThanRequested
}
@@ -0,0 +1,253 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.eel.impl.fs
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.serviceAsync
import com.intellij.openapi.diagnostic.logger
import com.intellij.platform.eel.EelResult
import com.intellij.platform.eel.ReadResult
import com.intellij.platform.eel.channels.EelDelicateApi
import com.intellij.platform.eel.fs.EelFileSystemApi
import com.intellij.platform.eel.fs.EelOpenedFile
import com.intellij.platform.eel.fs.openForReading
import com.intellij.platform.eel.getOr
import com.intellij.platform.eel.getOrThrow
import com.intellij.platform.eel.path.EelPath
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
import java.nio.ByteBuffer
import kotlin.math.max
@OptIn(EelDelicateApi::class)
suspend fun EelFileSystemApi.readFileImpl(args: EelFileSystemApi.ReadFileArgs): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
readFileImpl(
path = args.path,
initialBuffer = args.buffer,
mayReturnSameBuffer = args.mayReturnSameBuffer,
totalLimit = args.limit?.toUInt() ?: UInt.MAX_VALUE,
failFastIfBeyondLimit = args.failFastIfBeyondLimit,
)
@OptIn(EelDelicateApi::class)
private suspend fun EelFileSystemApi.readFileImpl(
path: EelPath,
initialBuffer: ByteBuffer?,
mayReturnSameBuffer: Boolean,
totalLimit: UInt,
failFastIfBeyondLimit: Boolean,
): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> {
val isTotalLimitSet = totalLimit != UInt.MAX_VALUE
val totalLimit = totalLimit.coerceAtMost(Int.MAX_VALUE.toUInt()).toInt()
var buffer: ByteBuffer = initialBuffer ?: ByteBuffer.allocate(totalLimit.coerceAtMost(RECOMMENDED_BUFFER_SIZE))
buffer.position(0).limit(totalLimit.coerceAtMost(buffer.capacity()))
val reader = openForReading(path)
.autoCloseAfterLastChunk(true)
.readFirstChunkInto(buffer)
.apply {
if (failFastIfBeyondLimit) {
closeImmediatelyIfFileBiggerThan(totalLimit.toLong())
}
}
.eelIt()
.getOr {
return it
}
if (reader.isClosed == true) {
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = true)
}
try {
if (buffer.limit() >= totalLimit) {
val fullyRead = reader.isClosed ?: readOneByteToCheckEof(reader, buffer).getOr {
return it
}
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = fullyRead)
}
if (reader.isClosed == true) {
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = true)
}
buffer.position(buffer.limit())
var fullyRead = false
mainLoop@ while (true) {
while (buffer.position() < totalLimit) {
buffer.limit(
totalLimit
.coerceAtMost(buffer.limit() + RECOMMENDED_BUFFER_SIZE)
.coerceAtMost(buffer.capacity())
)
if (buffer.remaining() < MINIMAL_EFFECTIVE_BUFFER_SIZE && buffer.capacity() < totalLimit) {
break
}
val readResult = reader.read(buffer).getOr {
return EelFsResultImpl.Error(it.error.mapError())
}
when (readResult) {
ReadResult.EOF -> {
fullyRead = true
buffer.flip()
break@mainLoop
}
ReadResult.NOT_EOF -> Unit
}
}
buffer.flip()
if (buffer.limit() >= totalLimit) {
break
}
val newBuffer = ByteBuffer.allocate(run {
val candidate = newLength(buffer.capacity())
if (totalLimit < candidate + MINIMAL_EFFECTIVE_BUFFER_SIZE) totalLimit
else candidate
})
newBuffer.put(buffer)
buffer = newBuffer
}
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = fullyRead)
}
finally {
serviceAsync<AsyncCloser>().closeSometimeLater(reader)
}
}
private suspend fun readOneByteToCheckEof(
reader: EelOpenedFile.Reader,
existingBuffer: ByteBuffer,
): EelResult<Boolean, EelFileSystemApi.FileReaderError> {
val (buffer, restoreValues) =
if (existingBuffer.limit() == existingBuffer.capacity()) {
ByteBuffer.allocate(1) to false
}
else {
existingBuffer to true
}
if (restoreValues) {
buffer.position(buffer.limit())
buffer.limit(buffer.limit() + 1)
}
val oldPosition = buffer.position()
reader.read(buffer).getOr {
return EelFsResultImpl.Error(it.error.mapError())
}
val fullyRead = buffer.position() == oldPosition
if (restoreValues) {
buffer.position(0)
buffer.limit(buffer.limit() - 1)
}
return EelFsResultImpl.Ok(fullyRead)
}
private fun prepareResult(
initialBuffer: ByteBuffer?,
buffer: ByteBuffer,
mayReturnSameBuffer: Boolean,
fullyRead: Boolean,
): EelFsResultImpl.Ok<EelFileSystemApi.ReadFileResult> =
EelFsResultImpl.Ok(object : EelFileSystemApi.ReadFileResult {
override val bytes: ByteBuffer =
if (buffer === initialBuffer && !mayReturnSameBuffer)
ByteBuffer.allocate(buffer.limit()).put(buffer).flip()
else
buffer
override val fullyRead: Boolean = fullyRead
})
/**
* The algorithm tries to not allocate buffers smaller than this threshold.
*
* It's useful to have such buffers, but the size doesn't matter.
* Here is a statistic about the sizes of all files in a personal checkout of the IntelliJ monorepo,
* including various build caches for Java and Rust projects.
* This constant is used only when a file can't be read with a single read operation.
* Hence, files smaller than 131,072 bytes are not considered.
* Hypothetically, fine-tuning of this constant can spare one read operation.
* Considering remainders of division by 131,072: if the distribution had peaks or falls,
* a special value of this constant could cover them.
* But there's only one peak: for value 0, which requires no specific optimization.
* Without 0, the distribution of this particular data is almost flat.
*
* Therefore, since any value passes, 16 * 1024 is chosen just because the author finds it attractive.
*/
private const val MINIMAL_EFFECTIVE_BUFFER_SIZE = 16 * 1024
/**
* 1. It's equal to `com.intellij.platform.ijent.spi.IjentSpiConstKt.RECOMMENDED_MAX_PACKET_SIZE`.
* Better to keep them in sync, at least for not allocating excessive buffers,
* but nothing should break down if they diverge.
* 2. It's big enough.
* Here is a statistic about the sizes of all files in a personal checkout of the IntelliJ monorepo,
* including various build caches for Java and Rust projects.
* * Total size = 182,106,998,688
* * File count = 2,922,717
* * Median size = 1369
* * P90 = 11,579
* * P95 = 30,480
* * P97 = 80,104
* * P98 = 138,401
* * P999 = 8,388,608
* * Max size = 27,239,038,774 -- if you're curious, it's one of `.git/objects/pack/pack-*.pack`
*
* This buffer is enough to read more than 97% of all files with a single read operation.
*/
private const val RECOMMENDED_BUFFER_SIZE = 131_072
/** A copy-paste of `jdk.internal.util.ArraysSupport.SOFT_MAX_ARRAY_LENGTH` */
private const val SOFT_MAX_ARRAY_LENGTH = Int.MAX_VALUE - 8
/** A stripped copy-paste of `jdk.internal.util.ArraysSupport.newLength` */
private fun newLength(oldLength: Int): Int {
val prefLength: Long = oldLength.toLong() + max(RECOMMENDED_BUFFER_SIZE, oldLength)
if (prefLength <= SOFT_MAX_ARRAY_LENGTH) {
return prefLength.toInt()
}
else {
// put code cold in a separate method
return hugeLength(oldLength)
}
}
/** A copy-paste of `jdk.internal.util.ArraysSupport.hugeLength` */
private fun hugeLength(oldLength: Int): Int {
val minLength: Long = oldLength.toLong() + RECOMMENDED_BUFFER_SIZE
if (minLength >= Int.MAX_VALUE) { // overflow
throw OutOfMemoryError(
"Required array length $oldLength + $RECOMMENDED_BUFFER_SIZE is too large")
}
return minLength.toInt().coerceAtLeast(SOFT_MAX_ARRAY_LENGTH)
}
private fun EelOpenedFile.Reader.ReadError.mapError(): EelFileSystemApi.FileReaderError =
when (this) {
is EelOpenedFile.Reader.ReadError.UnknownFile, is EelOpenedFile.Reader.ReadError.InvalidValue -> EelFsResultImpl.Other(where, toString())
is EelOpenedFile.Reader.ReadError.Other -> EelFsResultImpl.Other(where, message)
}
@Service
private class AsyncCloser(private val coroutineScope: CoroutineScope) {
fun closeSometimeLater(reader: EelOpenedFile.Reader) {
coroutineScope.launch {
try {
reader.close().getOrThrow()
}
catch (err: Exception) {
logger<AsyncCloser>().info("Failed to close $reader", err)
}
}
}
}
@@ -7,6 +7,7 @@ import com.intellij.platform.eel.EelResult
import com.intellij.platform.eel.EelUserPosixInfo
import com.intellij.platform.eel.EelUserWindowsInfo
import com.intellij.platform.eel.ReadResult
import com.intellij.platform.eel.channels.EelDelicateApi
import com.intellij.platform.eel.fs.*
import com.intellij.platform.eel.fs.EelFileSystemApi.FileWriterCreationMode.*
import com.intellij.platform.eel.path.EelPath
@@ -30,12 +31,8 @@ import java.nio.file.attribute.*
import java.time.Instant
import java.time.ZoneId
import java.time.ZonedDateTime
import kotlin.io.path.exists
import kotlin.io.path.fileAttributesView
import kotlin.io.path.fileStore
import kotlin.io.path.isDirectory
import kotlin.io.path.listDirectoryEntries
import kotlin.io.path.pathString
import java.util.concurrent.atomic.AtomicReference
import kotlin.io.path.*
import kotlin.streams.asSequence
abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : EelFileSystemApi {
@@ -51,10 +48,16 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
}
inline fun <T, reified E : EelFsError> wrapIntoEelResult(body: () -> T): EelResult<T, E> =
wrapIntoEelResult(null, body)
inline fun <T, reified E : EelFsError> wrapIntoEelResult(isClosed: AtomicReference<Boolean?>?, body: () -> T): EelResult<T, E> =
try {
EelFsResultImpl.Ok(body())
val result = EelFsResultImpl.Ok(body())
isClosed?.updateAndGet { it ?: false }
result
}
catch (err: FileSystemException) {
isClosed?.set(true)
val path =
try {
EelPath.parse(err.file.toString(), LocalEelDescriptor)
@@ -122,15 +125,58 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
Files.isSameFile(source.toNioPath(), target.toNioPath())
}
override suspend fun openForReading(path: EelPath): EelResult<
@OptIn(EelDelicateApi::class)
override suspend fun openForReading(args: EelFileSystemApi.OpenForReadingArgs): EelResult<
EelOpenedFile.Reader,
EelFileSystemApi.FileReaderError
> =
wrapIntoEelResult {
val nioPath = path.toNioPath()
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, setOf(StandardOpenOption.READ))
LocalEelOpenedFileReader(this, byteChannel, path)
> {
val result = openForReadingImpl(
path = args.path,
autoCloseAfterLastChunk = args.autoCloseAfterLastChunk,
closeImmediatelyIfFileBiggerThan = args.closeImmediatelyIfFileBiggerThan,
)
// Not filling the buffer. This optimization makes no sense for the local case.
args.readFirstChunkInto?.run {
position(0)
limit(0)
}
return result
}
@OptIn(EelDelicateApi::class)
private fun openForReadingImpl(
path: EelPath,
autoCloseAfterLastChunk: Boolean,
closeImmediatelyIfFileBiggerThan: Long?,
): EelResult<LocalEelOpenedFileReader, EelFileSystemApi.FileReaderError> = wrapIntoEelResult {
val nioPath = path.toNioPath()
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, setOf(StandardOpenOption.READ))
val isClosed = AtomicReference<Boolean?>(null)
if (closeImmediatelyIfFileBiggerThan != null) {
if (byteChannel.size() > closeImmediatelyIfFileBiggerThan) {
isClosed.set(true)
try {
doClose(this@NioBasedEelFileSystemApi, byteChannel, path)
}
catch (_: IOException) {
// Ignored.
}
return EelFsResultImpl.Error(EelFsResultImpl.FileBiggerThanRequested(path, "The file is bigger than the requested size"))
}
}
LocalEelOpenedFileReader(
eelFs = this,
byteChannel = byteChannel,
path_ = path,
isClosed_ = isClosed,
autoCloseAfterLastChunk = autoCloseAfterLastChunk,
)
}
override suspend fun readFile(args: EelFileSystemApi.ReadFileArgs): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
readFileImpl(args)
override suspend fun openForWriting(options: EelFileSystemApi.WriteOptions): EelResult<
EelOpenedFile.Writer,
@@ -140,7 +186,7 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
val nioPath = path.toNioPath()
val nioOptions = writeOptionsToNioOptions(options)
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, nioOptions)
LocalEelOpenedFileWriter(this, byteChannel, path)
LocalEelOpenedFileWriter(this, byteChannel, path, AtomicReference(null))
}
private fun writeOptionsToNioOptions(options: EelFileSystemApi.WriteOptions): MutableSet<StandardOpenOption> {
@@ -168,15 +214,16 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
val nioOptions = writeOptionsToNioOptions(options)
nioOptions += StandardOpenOption.READ
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, nioOptions)
object : EelOpenedFile.ReaderWriter, EelOpenedFile.Writer by LocalEelOpenedFileWriter(this, byteChannel, path) {
val isClosed = AtomicReference<Boolean?>(null)
object : EelOpenedFile.ReaderWriter, EelOpenedFile.Writer by LocalEelOpenedFileWriter(this, byteChannel, path, isClosed) {
override suspend fun read(buf: ByteBuffer): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
doRead(this@NioBasedEelFileSystemApi, byteChannel, buf)
doRead(this@NioBasedEelFileSystemApi, byteChannel, buf, isClosed, autoCloseAfterLastChunk = false)
override suspend fun read(
buf: ByteBuffer,
offset: Long,
): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
doRead(this@NioBasedEelFileSystemApi, byteChannel, offset, buf)
doRead(this@NioBasedEelFileSystemApi, byteChannel, offset, buf, isClosed)
}
}
@@ -231,32 +278,40 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
EelPathUtils.walkingTransfer(sourceNioPath, targetNioPath, removeSource = true, copyAttributes = true)
}
}
}
}
@EelDelicateApi
internal class LocalEelOpenedFileReader(
private val eelFs: NioBasedEelFileSystemApi,
private val byteChannel: SeekableByteChannel,
private val path_: EelPath,
private val isClosed_: AtomicReference<Boolean?>,
private val autoCloseAfterLastChunk: Boolean,
) : EelOpenedFile.Reader {
override val isClosed: Boolean?
get() = isClosed_.get()
override suspend fun read(buf: ByteBuffer): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
doRead(eelFs, byteChannel, buf)
doRead(eelFs, byteChannel, buf, isClosed_, autoCloseAfterLastChunk)
override suspend fun read(buf: ByteBuffer, offset: Long): EelResult<
ReadResult,
EelOpenedFile.Reader.ReadError
> =
doRead(eelFs, byteChannel, offset, buf)
doRead(eelFs, byteChannel, offset, buf, isClosed_)
override val path: EelPath = path_
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> =
doClose(eelFs, byteChannel, path_)
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> {
isClosed_.set(true)
return doClose(eelFs, byteChannel, path_)
}
override suspend fun tell(): EelResult<Long, EelOpenedFile.TellError> =
doTell(eelFs, byteChannel, path_)
doTell(eelFs, byteChannel, isClosed_)
override suspend fun seek(offset: Long, whence: EelOpenedFile.SeekWhence): EelResult<Long, EelOpenedFile.SeekError> =
doSeek(eelFs, byteChannel, path_, whence, offset)
doSeek(eelFs, byteChannel, path_, whence, offset, isClosed_)
override suspend fun stat(): EelResult<EelFileInfo, EelFileSystemApi.StatError> =
eelFs.stat(path_, EelFileSystemApi.SymlinkPolicy.RESOLVE_AND_FOLLOW)
@@ -266,14 +321,19 @@ private class LocalEelOpenedFileWriter(
private val eelFs: NioBasedEelFileSystemApi,
private val byteChannel: SeekableByteChannel,
private val path_: EelPath,
private val isClosed_: AtomicReference<Boolean?>,
) : EelOpenedFile.Writer {
@EelDelicateApi
override val isClosed: Boolean?
get() = isClosed_.get()
override suspend fun write(buf: ByteBuffer): EelResult<Int, EelOpenedFile.Writer.WriteError> =
eelFs.wrapIntoEelResult {
eelFs.wrapIntoEelResult(isClosed_) {
byteChannel.write(buf)
}
override suspend fun write(buf: ByteBuffer, pos: Long): EelResult<Int, EelOpenedFile.Writer.WriteError> =
eelFs.wrapIntoEelResult {
eelFs.wrapIntoEelResult(isClosed_) {
val oldPosition = byteChannel.position()
byteChannel.position(pos)
val written = byteChannel.write(buf)
@@ -286,20 +346,22 @@ private class LocalEelOpenedFileWriter(
}
override suspend fun truncate(size: Long): EelResult<Unit, EelOpenedFile.Writer.TruncateError> =
eelFs.wrapIntoEelResult {
eelFs.wrapIntoEelResult(isClosed_) {
byteChannel.truncate(size)
}
override val path: EelPath = path_
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> =
doClose(eelFs, byteChannel, path_)
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> {
isClosed_.set(true)
return doClose(eelFs, byteChannel, path_)
}
override suspend fun tell(): EelResult<Long, EelOpenedFile.TellError> =
doTell(eelFs, byteChannel, path_)
doTell(eelFs, byteChannel, isClosed_)
override suspend fun seek(offset: Long, whence: EelOpenedFile.SeekWhence): EelResult<Long, EelOpenedFile.SeekError> =
doSeek(eelFs, byteChannel, path_, whence, offset)
doSeek(eelFs, byteChannel, path_, whence, offset, isClosed_)
override suspend fun stat(): EelResult<EelFileInfo, EelFileSystemApi.StatError> =
eelFs.stat(path_, EelFileSystemApi.SymlinkPolicy.RESOLVE_AND_FOLLOW)
@@ -309,11 +371,19 @@ private fun doRead(
eelFs: NioBasedEelFileSystemApi,
byteChannel: SeekableByteChannel,
buf: ByteBuffer,
isClosed: AtomicReference<Boolean?>,
autoCloseAfterLastChunk: Boolean,
): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
eelFs.wrapIntoEelResult {
eelFs.wrapIntoEelResult(isClosed) {
val read = byteChannel.read(buf)
ReadResult.fromNumberOfReadBytes(read)
val result = ReadResult.fromNumberOfReadBytes(read)
if (autoCloseAfterLastChunk) {
when (result) {
ReadResult.EOF -> isClosed.set(true)
ReadResult.NOT_EOF -> isClosed.set(false)
}
}
result
}
private fun doRead(
@@ -321,8 +391,9 @@ private fun doRead(
byteChannel: SeekableByteChannel,
offset: Long,
buf: ByteBuffer,
isClosed: AtomicReference<Boolean?>,
): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
eelFs.wrapIntoEelResult {
eelFs.wrapIntoEelResult(isClosed) {
val oldPosition = byteChannel.position()
byteChannel.position(offset)
val read = byteChannel.read(buf)
@@ -337,7 +408,8 @@ private fun doSeek(
path: EelPath,
whence: EelOpenedFile.SeekWhence,
offset: Long,
): EelResult<Long, EelOpenedFile.SeekError> = eelFs.wrapIntoEelResult {
isClosed: AtomicReference<Boolean?>,
): EelResult<Long, EelOpenedFile.SeekError> = eelFs.wrapIntoEelResult(isClosed) {
val newPosition = when (whence) {
EelOpenedFile.SeekWhence.START -> offset
EelOpenedFile.SeekWhence.CURRENT -> byteChannel.position() + offset
@@ -347,8 +419,12 @@ private fun doSeek(
newPosition
}
private fun doTell(eelFs: NioBasedEelFileSystemApi, byteChannel: SeekableByteChannel, path: EelPath): EelResult<Long, EelOpenedFile.TellError> =
eelFs.wrapIntoEelResult {
private fun doTell(
eelFs: NioBasedEelFileSystemApi,
byteChannel: SeekableByteChannel,
isClosed: AtomicReference<Boolean?>,
): EelResult<Long, EelOpenedFile.TellError> =
eelFs.wrapIntoEelResult(isClosed) {
byteChannel.position().toLong()
}
@@ -424,10 +500,6 @@ abstract class PosixNioBasedEelFileSystemApi(
Files.createSymbolicLink(linkPath.toNioPath(), targetPath)
}
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
TODO("Not yet implemented")
}
override suspend fun walkDirectory(options: EelFileSystemApi.WalkDirectoryOptions): Flow<WalkDirectoryEntryResult> = flow {
val rootDir = options.path.asNioPath()
@@ -514,7 +586,7 @@ abstract class PosixNioBasedEelFileSystemApi(
currentItem: Path,
sourceAttrs: PosixFileAttributes,
emptyFileHash: Long,
options: EelFileSystemApi.WalkDirectoryOptions
options: EelFileSystemApi.WalkDirectoryOptions,
): WalkDirectoryEntryResult? {
var creationTime: ZonedDateTime? = null
var lastModifiedTime: ZonedDateTime? = null
@@ -656,10 +728,6 @@ abstract class WindowsNioBasedEelFileSystemApi(
// TODO File permissions for windows.
}
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
TODO("Not yet implemented")
}
override suspend fun walkDirectory(options: EelFileSystemApi.WalkDirectoryOptions): Flow<WalkDirectoryEntryResult> = flow {
val rootDir = options.path.asNioPath()
@@ -746,7 +814,7 @@ abstract class WindowsNioBasedEelFileSystemApi(
currentItem: Path,
sourceAttrs: DosFileAttributes,
emptyFileHash: Long,
options: EelFileSystemApi.WalkDirectoryOptions
options: EelFileSystemApi.WalkDirectoryOptions,
): WalkDirectoryEntryResult? {
var creationTime: ZonedDateTime? = null
var lastModifiedTime: ZonedDateTime? = null
@@ -2,6 +2,7 @@
package com.intellij.platform.eel.provider.utils
import com.intellij.openapi.progress.runBlockingMaybeCancellable
import com.intellij.openapi.util.io.FileTooBigException
import com.intellij.platform.eel.EelExecApi
import com.intellij.platform.eel.EelPlatform
import com.intellij.platform.eel.EelResult
@@ -87,6 +88,8 @@ fun EelFsError.throwFileSystemException(): Nothing {
-> throw IllegalArgumentException(message)
is EelOpenedFile.Writer.WriteError.InvalidValue -> throw IllegalArgumentException(message)
is EelFileSystemApi.DeleteError.UnresolvedLink -> throw FileSystemException(where.toString(), null, message)
is EelFileSystemApi.FileReaderError.FileBiggerThanRequested ->
throw FileSystemException(where.toString(), null, "File is too big").apply { initCause(FileTooBigException("File is too big")) }
is EelFsError.Other -> FileSystemException(where.toString(), null, message.nullize())
}
}
@@ -7,6 +7,7 @@ package com.intellij.platform.eel.fs
import com.intellij.platform.eel.EelResult
import com.intellij.platform.eel.GeneratedBuilder
import com.intellij.platform.eel.OwnedBuilder
import com.intellij.platform.eel.channels.EelDelicateApi
import com.intellij.platform.eel.fs.EelFileInfo.Permissions
import com.intellij.platform.eel.fs.EelFileSystemApi.FileChangeType
import com.intellij.platform.eel.fs.EelFileSystemApi.FileWriterCreationMode
@@ -23,6 +24,7 @@ import com.intellij.platform.eel.path.EelPath
import kotlinx.coroutines.flow.Flow
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.CheckReturnValue
import java.nio.ByteBuffer
@GeneratedBuilder.Result
@@ -91,6 +93,39 @@ fun EelFileSystemApi.move(
target = target,
)
/**
* Opens the file only for reading.
*
* In many cases [readFile] suits better than [openForReading].
*/
@GeneratedBuilder.Result
@ApiStatus.Internal
fun EelFileSystemApi.openForReading(
path: EelPath,
): EelFileSystemApiHelpers.OpenForReading =
EelFileSystemApiHelpers.OpenForReading(
owner = this,
path = path,
)
/**
* Fully or partially reads the file.
*
* Although it's possible to implement file reading with [openForReading],
* this function is optimized and covered with tests.
*
* The returned [ReadFileResult.bytes] is prepared for reading.
*/
@GeneratedBuilder.Result
@ApiStatus.Internal
fun EelFileSystemApi.readFile(
path: EelPath,
): EelFileSystemApiHelpers.ReadFile =
EelFileSystemApiHelpers.ReadFile(
owner = this,
path = path,
)
/**
* Similar to stat(2) and lstat(2). [symlinkPolicy] has an impact only on [EelFileInfo.type] if [path] points on a symlink.
*/
@@ -494,6 +529,156 @@ object EelFileSystemApiHelpers {
)
}
/**
* Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.openForReading].
*/
@GeneratedBuilder.Result
@ApiStatus.Internal
class OpenForReading(
private val owner: EelFileSystemApi,
private var path: EelPath,
) : OwnedBuilder<EelResult<EelOpenedFile.Reader, EelFileSystemApi.FileReaderError>> {
private var autoCloseAfterLastChunk: Boolean = false
private var closeImmediatelyIfFileBiggerThan: Long? = null
private var readFirstChunkInto: ByteBuffer? = null
/**
* When specified, the implementation closes its internal file descriptor
* as soon as it internally reaches the end of the file.
*
* There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read]
* * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call.
* * By checking inexpensive but unreliable [EelOpenedFile.isClosed].
*/
@EelDelicateApi
fun autoCloseAfterLastChunk(arg: Boolean): OpenForReading = apply {
this.autoCloseAfterLastChunk = arg
}
/**
* An optimization suitable for reading into memory.
* It allows aborting the reading fast if the whole file content
* won't fit into some buffer.
*
* If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested].
*/
@EelDelicateApi
fun closeImmediatelyIfFileBiggerThan(arg: Long?): OpenForReading = apply {
this.closeImmediatelyIfFileBiggerThan = arg
}
fun path(arg: EelPath): OpenForReading = apply {
this.path = arg
}
/**
* When specified, data from the file MAY be written into this buffer.
* [ByteBuffer.position] and [ByteBuffer.limit] are always changed.
* The buffer is prepared for reading after the call,
* so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading].
*
* If some data is written into this buffer,
* the first call of [EelOpenedFile.Reader.read] reads the data following this buffer.
*/
@EelDelicateApi
fun readFirstChunkInto(arg: ByteBuffer?): OpenForReading = apply {
this.readFirstChunkInto = arg
}
/**
* Complete the builder and call [com.intellij.platform.eel.fs.EelFileSystemApi.openForReading]
* with an instance of [com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs].
*/
@CheckReturnValue
override suspend fun eelIt(): EelResult<EelOpenedFile.Reader, EelFileSystemApi.FileReaderError> =
owner.openForReading(
OpenForReadingArgsImpl(
autoCloseAfterLastChunk = autoCloseAfterLastChunk,
closeImmediatelyIfFileBiggerThan = closeImmediatelyIfFileBiggerThan,
path = path,
readFirstChunkInto = readFirstChunkInto,
)
)
}
/**
* Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.readFile].
*/
@GeneratedBuilder.Result
@ApiStatus.Internal
class ReadFile(
private val owner: EelFileSystemApi,
private var path: EelPath,
) : OwnedBuilder<EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError>> {
private var buffer: ByteBuffer? = null
private var failFastIfBeyondLimit: Boolean = false
private var limit: Int? = null
private var mayReturnSameBuffer: Boolean = true
/**
* Use some specific buffer for reading files instead of creating a temporary buffer.
*
* The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call.
*
* The buffer is ready for reading, no need to call `flip`.
*/
@EelDelicateApi
fun buffer(arg: ByteBuffer?): ReadFile = apply {
this.buffer = arg
}
/**
* If this flag is set, the implementation checks the file size before trying to read data,
* and if the file is certainly bigger than [limit], no data is read.
*/
@EelDelicateApi
fun failFastIfBeyondLimit(arg: Boolean): ReadFile = apply {
this.failFastIfBeyondLimit = arg
}
/**
* Maximal number of bytes to read.
*/
fun limit(arg: Int?): ReadFile = apply {
this.limit = arg
}
/**
* If this flag is set and [buffer] is specified, the implementation reads the whole
* file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer.
*
* If [buffer] is not specified, the value of the flag is ignored.
*/
@EelDelicateApi
fun mayReturnSameBuffer(arg: Boolean): ReadFile = apply {
this.mayReturnSameBuffer = arg
}
fun path(arg: EelPath): ReadFile = apply {
this.path = arg
}
/**
* Complete the builder and call [com.intellij.platform.eel.fs.EelFileSystemApi.readFile]
* with an instance of [com.intellij.platform.eel.fs.EelFileSystemApi.ReadFileArgs].
*/
override suspend fun eelIt(): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
owner.readFile(
ReadFileArgsImpl(
buffer = buffer,
failFastIfBeyondLimit = failFastIfBeyondLimit,
limit = limit,
mayReturnSameBuffer = mayReturnSameBuffer,
path = path,
)
)
}
/**
* Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.stat].
*/
@@ -0,0 +1,84 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
/**
* This file is generated by [com.intellij.platform.eel.codegen.BuildersGeneratorTest].
*/
package com.intellij.platform.eel.fs
import com.intellij.platform.eel.GeneratedBuilder
import com.intellij.platform.eel.channels.EelDelicateApi
import com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs
import com.intellij.platform.eel.path.EelPath
import org.jetbrains.annotations.ApiStatus
import java.nio.ByteBuffer
@GeneratedBuilder.Result
@ApiStatus.Internal
class OpenForReadingArgsBuilder(
private var path: EelPath,
) {
private var autoCloseAfterLastChunk: Boolean = false
private var closeImmediatelyIfFileBiggerThan: Long? = null
private var readFirstChunkInto: ByteBuffer? = null
/**
* When specified, the implementation closes its internal file descriptor
* as soon as it internally reaches the end of the file.
*
* There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read]
* * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call.
* * By checking inexpensive but unreliable [EelOpenedFile.isClosed].
*/
@EelDelicateApi
fun autoCloseAfterLastChunk(arg: Boolean): OpenForReadingArgsBuilder = apply {
this.autoCloseAfterLastChunk = arg
}
/**
* An optimization suitable for reading into memory.
* It allows aborting the reading fast if the whole file content
* won't fit into some buffer.
*
* If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested].
*/
@EelDelicateApi
fun closeImmediatelyIfFileBiggerThan(arg: Long?): OpenForReadingArgsBuilder = apply {
this.closeImmediatelyIfFileBiggerThan = arg
}
fun path(arg: EelPath): OpenForReadingArgsBuilder = apply {
this.path = arg
}
/**
* When specified, data from the file MAY be written into this buffer.
* [ByteBuffer.position] and [ByteBuffer.limit] are always changed.
* The buffer is prepared for reading after the call,
* so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading].
*
* If some data is written into this buffer,
* the first call of [EelOpenedFile.Reader.read] reads the data following this buffer.
*/
@EelDelicateApi
fun readFirstChunkInto(arg: ByteBuffer?): OpenForReadingArgsBuilder = apply {
this.readFirstChunkInto = arg
}
fun build(): OpenForReadingArgs =
OpenForReadingArgsImpl(
autoCloseAfterLastChunk = autoCloseAfterLastChunk,
closeImmediatelyIfFileBiggerThan = closeImmediatelyIfFileBiggerThan,
path = path,
readFirstChunkInto = readFirstChunkInto,
)
}
@GeneratedBuilder.Result
internal class OpenForReadingArgsImpl(
override val autoCloseAfterLastChunk: Boolean,
override val closeImmediatelyIfFileBiggerThan: Long?,
override val path: EelPath,
override val readFirstChunkInto: ByteBuffer?,
) : OpenForReadingArgs
@@ -0,0 +1,88 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
/**
* This file is generated by [com.intellij.platform.eel.codegen.BuildersGeneratorTest].
*/
package com.intellij.platform.eel.fs
import com.intellij.platform.eel.GeneratedBuilder
import com.intellij.platform.eel.channels.EelDelicateApi
import com.intellij.platform.eel.fs.EelFileSystemApi.ReadFileArgs
import com.intellij.platform.eel.path.EelPath
import org.jetbrains.annotations.ApiStatus
import java.nio.ByteBuffer
@GeneratedBuilder.Result
@ApiStatus.Internal
class ReadFileArgsBuilder(
private var path: EelPath,
) {
private var buffer: ByteBuffer? = null
private var failFastIfBeyondLimit: Boolean = false
private var limit: Int? = null
private var mayReturnSameBuffer: Boolean = true
/**
* Use some specific buffer for reading files instead of creating a temporary buffer.
*
* The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call.
*
* The buffer is ready for reading, no need to call `flip`.
*/
@EelDelicateApi
fun buffer(arg: ByteBuffer?): ReadFileArgsBuilder = apply {
this.buffer = arg
}
/**
* If this flag is set, the implementation checks the file size before trying to read data,
* and if the file is certainly bigger than [limit], no data is read.
*/
@EelDelicateApi
fun failFastIfBeyondLimit(arg: Boolean): ReadFileArgsBuilder = apply {
this.failFastIfBeyondLimit = arg
}
/**
* Maximal number of bytes to read.
*/
fun limit(arg: Int?): ReadFileArgsBuilder = apply {
this.limit = arg
}
/**
* If this flag is set and [buffer] is specified, the implementation reads the whole
* file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer.
*
* If [buffer] is not specified, the value of the flag is ignored.
*/
@EelDelicateApi
fun mayReturnSameBuffer(arg: Boolean): ReadFileArgsBuilder = apply {
this.mayReturnSameBuffer = arg
}
fun path(arg: EelPath): ReadFileArgsBuilder = apply {
this.path = arg
}
fun build(): ReadFileArgs =
ReadFileArgsImpl(
buffer = buffer,
failFastIfBeyondLimit = failFastIfBeyondLimit,
limit = limit,
mayReturnSameBuffer = mayReturnSameBuffer,
path = path,
)
}
@GeneratedBuilder.Result
internal class ReadFileArgsImpl(
override val buffer: ByteBuffer?,
override val failFastIfBeyondLimit: Boolean,
override val limit: Int?,
override val mayReturnSameBuffer: Boolean,
override val path: EelPath,
) : ReadFileArgs
@@ -10,4 +10,13 @@ import org.jetbrains.annotations.ApiStatus
message = "This function might send only part of the buffer. Consider using `sendWholeBuffer` instead"
)
@ApiStatus.Internal
annotation class EelSendApi
annotation class EelSendApi
@MustBeDocumented
@Retention(value = AnnotationRetention.BINARY)
@RequiresOptIn(
level = RequiresOptIn.Level.WARNING,
message = "Ensure that you read the documentation thoroughly or that you're ready to shoot your foot"
)
@ApiStatus.Internal
annotation class EelDelicateApi
@@ -2,6 +2,7 @@
package com.intellij.platform.eel.fs
import com.intellij.platform.eel.*
import com.intellij.platform.eel.channels.EelDelicateApi
import com.intellij.platform.eel.fs.EelFileSystemApi.StatError
import com.intellij.platform.eel.path.EelPath
import kotlinx.coroutines.flow.Flow
@@ -178,46 +179,120 @@ interface EelFileSystemApi {
}
/**
* Opens the file only for reading
* Opens the file only for reading.
*
* In many cases [readFile] suits better than [openForReading].
*/
@CheckReturnValue
suspend fun openForReading(path: EelPath): EelResult<
suspend fun openForReading(@GeneratedBuilder args: OpenForReadingArgs): EelResult<
EelOpenedFile.Reader,
FileReaderError>
interface OpenForReadingArgs {
val path: EelPath
/**
* When specified, data from the file MAY be written into this buffer.
* [ByteBuffer.position] and [ByteBuffer.limit] are always changed.
* The buffer is prepared for reading after the call,
* so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading].
*
* If some data is written into this buffer,
* the first call of [EelOpenedFile.Reader.read] reads the data following this buffer.
*/
@EelDelicateApi
val readFirstChunkInto: ByteBuffer? get() = null
/**
* When specified, the implementation closes its internal file descriptor
* as soon as it internally reaches the end of the file.
*
* There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read]
* * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call.
* * By checking inexpensive but unreliable [EelOpenedFile.isClosed].
*/
@EelDelicateApi
val autoCloseAfterLastChunk: Boolean get() = false
/**
* An optimization suitable for reading into memory.
* It allows aborting the reading fast if the whole file content
* won't fit into some buffer.
*
* If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested].
*/
@EelDelicateApi
val closeImmediatelyIfFileBiggerThan: Long? get() = null
}
sealed interface FileReaderError : EelFsError {
interface AlreadyExists : FileReaderError, EelFsError.AlreadyExists
interface DoesNotExist : FileReaderError, EelFsError.DoesNotExist
interface PermissionDenied : FileReaderError, EelFsError.PermissionDenied
interface NotDirectory : FileReaderError, EelFsError.NotDirectory
interface NotFile : FileReaderError, EelFsError.NotFile
/** See [com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs.closeImmediatelyIfFileBiggerThan] */
interface FileBiggerThanRequested : FileReaderError
interface Other : FileReaderError, EelFsError.Other
}
enum class OverflowPolicy {
DROP,
RETAIN,
/**
* Fully or partially reads the file.
*
* Although it's possible to implement file reading with [openForReading],
* this function is optimized and covered with tests.
*
* The returned [ReadFileResult.bytes] is prepared for reading.
*/
suspend fun readFile(@GeneratedBuilder args: ReadFileArgs): EelResult<ReadFileResult, FileReaderError>
interface ReadFileArgs {
val path: EelPath
/**
* Maximal number of bytes to read.
*/
val limit: Int? get() = null
/**
* If this flag is set, the implementation checks the file size before trying to read data,
* and if the file is certainly bigger than [limit], no data is read.
*/
@EelDelicateApi
val failFastIfBeyondLimit: Boolean get() = false
/**
* Use some specific buffer for reading files instead of creating a temporary buffer.
*
* The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call.
*
* The buffer is ready for reading, no need to call `flip`.
*/
@EelDelicateApi
val buffer: ByteBuffer? get() = null
/**
* If this flag is set and [buffer] is specified, the implementation reads the whole
* file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer.
*
* If [buffer] is not specified, the value of the flag is ignored.
*/
@EelDelicateApi
val mayReturnSameBuffer: Boolean get() = true
}
sealed interface FullReadResult {
interface Overflow : FullReadResult
interface BytesOverflown : FullReadResult {
val bytes: ByteArray
}
interface ReadFileResult {
/**
* It's ready for reading, the position and the limit are already set.
*/
val bytes: ByteBuffer
interface Bytes : FullReadResult {
val bytes: ByteArray
}
}
@CheckReturnValue
suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: OverflowPolicy): EelResult<FullReadResult, FullReadError>
sealed interface FullReadError : EelFsError {
interface DoesNotExist : FullReadError, EelFsError.DoesNotExist
interface PermissionDenied : FullReadError, EelFsError.PermissionDenied
interface NotFile : FullReadError, EelFsError.NotFile
interface Other : FullReadError, EelFsError.Other
/**
* `true` if [bytes] contains the whole file, `false` only if the part of the file.
*/
val fullyRead: Boolean
}
interface WalkDirectoryOptions {
@@ -754,6 +829,18 @@ sealed interface EelOpenedFile {
@CheckReturnValue
suspend fun close(): EelResult<Unit, CloseError>
/**
* This method is to be used for avoiding potentially excessive calls.
* However, rely on this function with suspicion: the implementation may return `null` whatever happens.
*
* Returns:
* * `true` if the file is closed.
* * `false` if it's not closed.
* * `null` if it's not possible to determine if the file is closed without calling any suspending method.
*/
@EelDelicateApi
val isClosed: Boolean?
sealed interface CloseError : EelFsError {
interface Other : CloseError, EelFsError.Other
}
@@ -1,4 +1,4 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.ijent.community.impl
import com.intellij.platform.eel.EelDescriptor
@@ -9,6 +9,7 @@ import com.intellij.platform.eel.fs.EelFileSystemApi
import com.intellij.platform.eel.fs.EelFileSystemPosixApi
import com.intellij.platform.eel.fs.EelOpenedFile
import com.intellij.platform.eel.fs.EelPosixFileInfo
import com.intellij.platform.eel.fs.*
import com.intellij.platform.eel.path.EelPath
import com.intellij.platform.ijent.IjentApi
import com.intellij.platform.ijent.IjentPosixApi
@@ -164,15 +165,18 @@ private class IjentFailSafeFileSystemPosixApiImpl(
}
override suspend fun openForReading(
path: EelPath,
args: EelFileSystemApi.OpenForReadingArgs,
): EelResult<EelOpenedFile.Reader, EelFileSystemApi.FileReaderError> =
holder.withDelegateRetrying {
openForReading(path)
openForReading(args)
}
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> = holder.withDelegateRetrying {
readFully(path, limit, overflowPolicy)
}
override suspend fun readFile(
args: EelFileSystemApi.ReadFileArgs,
): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
holder.withDelegateRetrying {
readFile(args)
}
override suspend fun openForWriting(
options: EelFileSystemApi.WriteOptions,
@@ -1,13 +1,10 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.ijent.community.impl.nio
import com.intellij.openapi.diagnostic.logger
import com.intellij.openapi.diagnostic.thisLogger
import com.intellij.platform.eel.ReadResult
import com.intellij.platform.eel.fs.EelFileInfo
import com.intellij.platform.eel.fs.EelFileSystemApi
import com.intellij.platform.eel.fs.EelOpenedFile
import com.intellij.platform.eel.fs.EelPosixFileInfo
import com.intellij.platform.eel.fs.*
import com.intellij.platform.eel.path.EelPath
import com.intellij.platform.eel.provider.utils.getOrThrowFileSystemException
import com.intellij.platform.ijent.spi.RECOMMENDED_MAX_PACKET_SIZE
@@ -12,7 +12,7 @@ import com.intellij.openapi.vfs.impl.local.LocalFileSystemBase.LOG
import com.intellij.openapi.vfs.impl.local.LocalFileSystemBase.toIoPath
import com.intellij.openapi.vfs.limits.FileSizeLimit
import com.intellij.platform.eel.EelApi
import com.intellij.platform.eel.LocalEelApi
import com.intellij.platform.eel.channels.EelDelicateApi
import com.intellij.platform.eel.fs.*
import com.intellij.platform.eel.getOr
import com.intellij.platform.eel.path.EelPath
@@ -26,6 +26,7 @@ import com.intellij.platform.ijent.community.impl.nio.IjentNioPosixFileAttribute
import com.intellij.platform.ijent.community.impl.nio.fsBlocking
import com.intellij.util.containers.CollectionFactory
import com.intellij.util.containers.ContainerUtil
import com.intellij.util.io.toByteArray
import kotlinx.coroutines.runBlocking
import java.io.IOException
import java.nio.file.*
@@ -38,34 +39,39 @@ private val map = ContainerUtil.createConcurrentWeakMap<Path, EelApi>()
* This is unacceptable in the remote setting when each request to IO results in RPC.
* Here we try to invoke a specialized function that can read all bytes from [path] in one request.
*/
@OptIn(EelDelicateApi::class)
@Suppress("RAW_RUN_BLOCKING")
internal fun readWholeFileIfNotTooLargeWithEel(path: Path): ByteArray? {
if (!Registry.`is`("vfs.try.eel.for.content.loading", false)) {
return null
}
val root = path.root ?: return null
// TODO Check if this if-else can be removed. The only reason why it's kept is to avoid possible performance degradations in hot code.
val eelDescriptor = root.getEelDescriptor()
if (eelDescriptor == LocalEelDescriptor) {
return null
}
val api = map.computeIfAbsent(root) {
val api = map.computeIfAbsent(root) { // TODO Does this cache make sense nowadays?
runBlocking {
root.getEelDescriptor().toEelApi()
}
}
if (api is LocalEelApi) {
return null
}
val eelPath = path.asEelPath()
val limit = FileSizeLimit.getContentLoadLimit(FileUtilRt.getExtension(path.fileName.toString()))
return runBlocking {
when (val res = api.fs.readFully(eelPath, limit.toULong(), EelFileSystemApi.OverflowPolicy.DROP).getOrThrowFileSystemException()) {
is EelFileSystemApi.FullReadResult.Bytes -> res.bytes
is EelFileSystemApi.FullReadResult.BytesOverflown -> error("Never returned")
is EelFileSystemApi.FullReadResult.Overflow -> throw FileTooBigException("File $path is bigger than $limit bytes")
val result = runBlocking {
try {
api.fs.readFile(eelPath).limit(limit).failFastIfBeyondLimit(true).getOrThrowFileSystemException()
}
catch (err: FileSystemException) {
throw err.cause.takeIf { it is FileTooBigException } ?: err
}
}
return result.bytes.toByteArray()
}
internal fun toEelPath(parent: VirtualFile, childName: String): EelPath? =
@@ -36,10 +36,6 @@ internal class EelTestPosixApi(override val descriptor: EelTestDescriptor, fileS
private class EelTestFileSystemPosixApi(override val descriptor: EelTestDescriptor, fileSystem: EelUnitTestFileSystem) : PosixNioBasedEelFileSystemApi(fileSystem, EelTestPosixUserInfo(descriptor)) {
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
TODO("Not yet implemented")
}
override suspend fun createTemporaryDirectory(options: EelFileSystemApi.CreateTemporaryEntryOptions): EelResult<EelPath, EelFileSystemApi.CreateTemporaryEntryError> {
return wrapIntoEelResult {
val nioTempDir = Files.createTempDirectory(fs.rootDirectories.single(), options.prefix)
@@ -30,10 +30,6 @@ internal class EelTestWindowsApi(override val descriptor: EelTestDescriptor, fil
private class EelTestFileSystemWindowsApi(override val descriptor: EelDescriptor, fileSystem: EelUnitTestFileSystem) : WindowsNioBasedEelFileSystemApi(fileSystem, EelTestWindowsUserInfo(descriptor)) {
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
TODO("Not yet implemented")
}
override suspend fun createTemporaryDirectory(options: EelFileSystemApi.CreateTemporaryEntryOptions): EelResult<EelPath, EelFileSystemApi.CreateTemporaryEntryError> {
return wrapIntoEelResult {
val nioTempDir = Files.createTempDirectory(options.prefix)