mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-09-27 10:03:11 +07:00
IJPL-201569 IJent: reimplement function for reading whole files to not require a special channel
GitOrigin-RevId: 31c89d8e1ef66de5b59652813c8466c994802c53
This commit is contained in:
committed by
intellij-monorepo-bot
parent
dab9db55b9
commit
2f2a86bc38
@@ -1,4 +1,4 @@
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package com.intellij.platform.eel.impl.fs
|
||||
|
||||
import com.intellij.platform.eel.EelResult
|
||||
@@ -15,10 +15,6 @@ object EelFsResultImpl {
|
||||
|
||||
data class DiskInfoImpl(override val totalSpace: ULong, override val availableSpace: ULong) : EelFileSystemApi.DiskInfo
|
||||
|
||||
data class FullBytesReadImpl(override val bytes: ByteArray) : EelFileSystemApi.FullReadResult.Bytes
|
||||
data class FullBytesReadOverflown(override val bytes: ByteArray) : EelFileSystemApi.FullReadResult.BytesOverflown
|
||||
data object Overflow : EelFileSystemApi.FullReadResult.Overflow
|
||||
|
||||
data class Other(override val where: EelPath, override val message: String) :
|
||||
EelFileSystemApi.FileReaderError.Other,
|
||||
EelFileSystemApi.FileWriterError.Other,
|
||||
@@ -40,7 +36,6 @@ object EelFsResultImpl {
|
||||
EelFileSystemPosixApi.CreateSymbolicLinkError.Other,
|
||||
EelOpenedFile.CloseError.Other,
|
||||
EelOpenedFile.Writer.TruncateError.Other,
|
||||
EelFileSystemApi.FullReadError.Other,
|
||||
EelFileSystemApi.WalkDirectoryError.Other
|
||||
|
||||
data class DoesNotExist(override val where: EelPath, override val message: String) :
|
||||
@@ -51,7 +46,6 @@ object EelFsResultImpl {
|
||||
EelFileSystemApi.SameFileError.DoesNotExist,
|
||||
EelFileSystemApi.StatError.DoesNotExist,
|
||||
EelFileSystemApi.CanonicalizeError.DoesNotExist,
|
||||
EelFileSystemApi.FullReadError.DoesNotExist,
|
||||
EelFileSystemApi.DiskInfoError.PathDoesNotExists,
|
||||
EelFileSystemPosixApi.CreateDirectoryError.ParentNotFound,
|
||||
EelFileSystemApi.CopyError.SourceDoesNotExist,
|
||||
@@ -81,7 +75,6 @@ object EelFsResultImpl {
|
||||
EelFileSystemApi.CopyError.PermissionDenied,
|
||||
EelFileSystemApi.MoveError.PermissionDenied,
|
||||
EelFileSystemPosixApi.CreateSymbolicLinkError.PermissionDenied,
|
||||
EelFileSystemApi.FullReadError.PermissionDenied,
|
||||
EelFileSystemApi.WalkDirectoryError.Other
|
||||
|
||||
data class NotDirectory(override val where: EelPath, override val message: String) :
|
||||
@@ -106,8 +99,7 @@ object EelFsResultImpl {
|
||||
EelFileSystemApi.FileWriterError.NotFile,
|
||||
EelFileSystemApi.SameFileError.NotFile,
|
||||
EelFileSystemApi.StatError.NotFile,
|
||||
EelFileSystemApi.MoveError.TargetIsDirectory,
|
||||
EelFileSystemApi.FullReadError.NotFile
|
||||
EelFileSystemApi.MoveError.TargetIsDirectory
|
||||
|
||||
data class InvalidValue(override val where: EelPath, override val message: String) :
|
||||
EelOpenedFile.Reader.ReadError.InvalidValue,
|
||||
@@ -147,4 +139,7 @@ object EelFsResultImpl {
|
||||
|
||||
data class OffsetTooBig(override val where: EelPath, override val message: String) :
|
||||
EelOpenedFile.Writer.TruncateError.OffsetTooBig
|
||||
|
||||
data class FileBiggerThanRequested(override val where: EelPath, override val message: String) :
|
||||
EelFileSystemApi.FileReaderError.FileBiggerThanRequested
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package com.intellij.platform.eel.impl.fs
|
||||
|
||||
import com.intellij.openapi.components.Service
|
||||
import com.intellij.openapi.components.serviceAsync
|
||||
import com.intellij.openapi.diagnostic.logger
|
||||
import com.intellij.platform.eel.EelResult
|
||||
import com.intellij.platform.eel.ReadResult
|
||||
import com.intellij.platform.eel.channels.EelDelicateApi
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi
|
||||
import com.intellij.platform.eel.fs.EelOpenedFile
|
||||
import com.intellij.platform.eel.fs.openForReading
|
||||
import com.intellij.platform.eel.getOr
|
||||
import com.intellij.platform.eel.getOrThrow
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import java.nio.ByteBuffer
|
||||
import kotlin.math.max
|
||||
|
||||
@OptIn(EelDelicateApi::class)
|
||||
suspend fun EelFileSystemApi.readFileImpl(args: EelFileSystemApi.ReadFileArgs): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
|
||||
readFileImpl(
|
||||
path = args.path,
|
||||
initialBuffer = args.buffer,
|
||||
mayReturnSameBuffer = args.mayReturnSameBuffer,
|
||||
totalLimit = args.limit?.toUInt() ?: UInt.MAX_VALUE,
|
||||
failFastIfBeyondLimit = args.failFastIfBeyondLimit,
|
||||
)
|
||||
|
||||
@OptIn(EelDelicateApi::class)
|
||||
private suspend fun EelFileSystemApi.readFileImpl(
|
||||
path: EelPath,
|
||||
initialBuffer: ByteBuffer?,
|
||||
mayReturnSameBuffer: Boolean,
|
||||
totalLimit: UInt,
|
||||
failFastIfBeyondLimit: Boolean,
|
||||
): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> {
|
||||
val isTotalLimitSet = totalLimit != UInt.MAX_VALUE
|
||||
val totalLimit = totalLimit.coerceAtMost(Int.MAX_VALUE.toUInt()).toInt()
|
||||
var buffer: ByteBuffer = initialBuffer ?: ByteBuffer.allocate(totalLimit.coerceAtMost(RECOMMENDED_BUFFER_SIZE))
|
||||
buffer.position(0).limit(totalLimit.coerceAtMost(buffer.capacity()))
|
||||
|
||||
val reader = openForReading(path)
|
||||
.autoCloseAfterLastChunk(true)
|
||||
.readFirstChunkInto(buffer)
|
||||
.apply {
|
||||
if (failFastIfBeyondLimit) {
|
||||
closeImmediatelyIfFileBiggerThan(totalLimit.toLong())
|
||||
}
|
||||
}
|
||||
.eelIt()
|
||||
.getOr {
|
||||
return it
|
||||
}
|
||||
|
||||
if (reader.isClosed == true) {
|
||||
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = true)
|
||||
}
|
||||
|
||||
try {
|
||||
if (buffer.limit() >= totalLimit) {
|
||||
val fullyRead = reader.isClosed ?: readOneByteToCheckEof(reader, buffer).getOr {
|
||||
return it
|
||||
}
|
||||
|
||||
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = fullyRead)
|
||||
}
|
||||
|
||||
if (reader.isClosed == true) {
|
||||
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = true)
|
||||
}
|
||||
|
||||
buffer.position(buffer.limit())
|
||||
|
||||
var fullyRead = false
|
||||
|
||||
mainLoop@ while (true) {
|
||||
while (buffer.position() < totalLimit) {
|
||||
buffer.limit(
|
||||
totalLimit
|
||||
.coerceAtMost(buffer.limit() + RECOMMENDED_BUFFER_SIZE)
|
||||
.coerceAtMost(buffer.capacity())
|
||||
)
|
||||
if (buffer.remaining() < MINIMAL_EFFECTIVE_BUFFER_SIZE && buffer.capacity() < totalLimit) {
|
||||
break
|
||||
}
|
||||
|
||||
val readResult = reader.read(buffer).getOr {
|
||||
return EelFsResultImpl.Error(it.error.mapError())
|
||||
}
|
||||
|
||||
when (readResult) {
|
||||
ReadResult.EOF -> {
|
||||
fullyRead = true
|
||||
buffer.flip()
|
||||
break@mainLoop
|
||||
}
|
||||
ReadResult.NOT_EOF -> Unit
|
||||
}
|
||||
}
|
||||
|
||||
buffer.flip()
|
||||
if (buffer.limit() >= totalLimit) {
|
||||
break
|
||||
}
|
||||
|
||||
val newBuffer = ByteBuffer.allocate(run {
|
||||
val candidate = newLength(buffer.capacity())
|
||||
if (totalLimit < candidate + MINIMAL_EFFECTIVE_BUFFER_SIZE) totalLimit
|
||||
else candidate
|
||||
})
|
||||
newBuffer.put(buffer)
|
||||
buffer = newBuffer
|
||||
}
|
||||
|
||||
return prepareResult(initialBuffer = initialBuffer, buffer = buffer, mayReturnSameBuffer = mayReturnSameBuffer, fullyRead = fullyRead)
|
||||
}
|
||||
finally {
|
||||
serviceAsync<AsyncCloser>().closeSometimeLater(reader)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun readOneByteToCheckEof(
|
||||
reader: EelOpenedFile.Reader,
|
||||
existingBuffer: ByteBuffer,
|
||||
): EelResult<Boolean, EelFileSystemApi.FileReaderError> {
|
||||
val (buffer, restoreValues) =
|
||||
if (existingBuffer.limit() == existingBuffer.capacity()) {
|
||||
ByteBuffer.allocate(1) to false
|
||||
}
|
||||
else {
|
||||
existingBuffer to true
|
||||
}
|
||||
|
||||
if (restoreValues) {
|
||||
buffer.position(buffer.limit())
|
||||
buffer.limit(buffer.limit() + 1)
|
||||
}
|
||||
|
||||
val oldPosition = buffer.position()
|
||||
reader.read(buffer).getOr {
|
||||
return EelFsResultImpl.Error(it.error.mapError())
|
||||
}
|
||||
val fullyRead = buffer.position() == oldPosition
|
||||
|
||||
if (restoreValues) {
|
||||
buffer.position(0)
|
||||
buffer.limit(buffer.limit() - 1)
|
||||
}
|
||||
|
||||
return EelFsResultImpl.Ok(fullyRead)
|
||||
}
|
||||
|
||||
private fun prepareResult(
|
||||
initialBuffer: ByteBuffer?,
|
||||
buffer: ByteBuffer,
|
||||
mayReturnSameBuffer: Boolean,
|
||||
fullyRead: Boolean,
|
||||
): EelFsResultImpl.Ok<EelFileSystemApi.ReadFileResult> =
|
||||
EelFsResultImpl.Ok(object : EelFileSystemApi.ReadFileResult {
|
||||
override val bytes: ByteBuffer =
|
||||
if (buffer === initialBuffer && !mayReturnSameBuffer)
|
||||
ByteBuffer.allocate(buffer.limit()).put(buffer).flip()
|
||||
else
|
||||
buffer
|
||||
|
||||
override val fullyRead: Boolean = fullyRead
|
||||
})
|
||||
|
||||
/**
|
||||
* The algorithm tries to not allocate buffers smaller than this threshold.
|
||||
*
|
||||
* It's useful to have such buffers, but the size doesn't matter.
|
||||
* Here is a statistic about the sizes of all files in a personal checkout of the IntelliJ monorepo,
|
||||
* including various build caches for Java and Rust projects.
|
||||
* This constant is used only when a file can't be read with a single read operation.
|
||||
* Hence, files smaller than 131,072 bytes are not considered.
|
||||
* Hypothetically, fine-tuning of this constant can spare one read operation.
|
||||
* Considering remainders of division by 131,072: if the distribution had peaks or falls,
|
||||
* a special value of this constant could cover them.
|
||||
* But there's only one peak: for value 0, which requires no specific optimization.
|
||||
* Without 0, the distribution of this particular data is almost flat.
|
||||
*
|
||||
* Therefore, since any value passes, 16 * 1024 is chosen just because the author finds it attractive.
|
||||
*/
|
||||
private const val MINIMAL_EFFECTIVE_BUFFER_SIZE = 16 * 1024
|
||||
|
||||
/**
|
||||
* 1. It's equal to `com.intellij.platform.ijent.spi.IjentSpiConstKt.RECOMMENDED_MAX_PACKET_SIZE`.
|
||||
* Better to keep them in sync, at least for not allocating excessive buffers,
|
||||
* but nothing should break down if they diverge.
|
||||
* 2. It's big enough.
|
||||
* Here is a statistic about the sizes of all files in a personal checkout of the IntelliJ monorepo,
|
||||
* including various build caches for Java and Rust projects.
|
||||
* * Total size = 182,106,998,688
|
||||
* * File count = 2,922,717
|
||||
* * Median size = 1369
|
||||
* * P90 = 11,579
|
||||
* * P95 = 30,480
|
||||
* * P97 = 80,104
|
||||
* * P98 = 138,401
|
||||
* * P999 = 8,388,608
|
||||
* * Max size = 27,239,038,774 -- if you're curious, it's one of `.git/objects/pack/pack-*.pack`
|
||||
*
|
||||
* This buffer is enough to read more than 97% of all files with a single read operation.
|
||||
*/
|
||||
private const val RECOMMENDED_BUFFER_SIZE = 131_072
|
||||
|
||||
/** A copy-paste of `jdk.internal.util.ArraysSupport.SOFT_MAX_ARRAY_LENGTH` */
|
||||
private const val SOFT_MAX_ARRAY_LENGTH = Int.MAX_VALUE - 8
|
||||
|
||||
/** A stripped copy-paste of `jdk.internal.util.ArraysSupport.newLength` */
|
||||
private fun newLength(oldLength: Int): Int {
|
||||
val prefLength: Long = oldLength.toLong() + max(RECOMMENDED_BUFFER_SIZE, oldLength)
|
||||
if (prefLength <= SOFT_MAX_ARRAY_LENGTH) {
|
||||
return prefLength.toInt()
|
||||
}
|
||||
else {
|
||||
// put code cold in a separate method
|
||||
return hugeLength(oldLength)
|
||||
}
|
||||
}
|
||||
|
||||
/** A copy-paste of `jdk.internal.util.ArraysSupport.hugeLength` */
|
||||
private fun hugeLength(oldLength: Int): Int {
|
||||
val minLength: Long = oldLength.toLong() + RECOMMENDED_BUFFER_SIZE
|
||||
if (minLength >= Int.MAX_VALUE) { // overflow
|
||||
throw OutOfMemoryError(
|
||||
"Required array length $oldLength + $RECOMMENDED_BUFFER_SIZE is too large")
|
||||
}
|
||||
return minLength.toInt().coerceAtLeast(SOFT_MAX_ARRAY_LENGTH)
|
||||
}
|
||||
|
||||
private fun EelOpenedFile.Reader.ReadError.mapError(): EelFileSystemApi.FileReaderError =
|
||||
when (this) {
|
||||
is EelOpenedFile.Reader.ReadError.UnknownFile, is EelOpenedFile.Reader.ReadError.InvalidValue -> EelFsResultImpl.Other(where, toString())
|
||||
is EelOpenedFile.Reader.ReadError.Other -> EelFsResultImpl.Other(where, message)
|
||||
}
|
||||
|
||||
@Service
|
||||
private class AsyncCloser(private val coroutineScope: CoroutineScope) {
|
||||
fun closeSometimeLater(reader: EelOpenedFile.Reader) {
|
||||
coroutineScope.launch {
|
||||
try {
|
||||
reader.close().getOrThrow()
|
||||
}
|
||||
catch (err: Exception) {
|
||||
logger<AsyncCloser>().info("Failed to close $reader", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import com.intellij.platform.eel.EelResult
|
||||
import com.intellij.platform.eel.EelUserPosixInfo
|
||||
import com.intellij.platform.eel.EelUserWindowsInfo
|
||||
import com.intellij.platform.eel.ReadResult
|
||||
import com.intellij.platform.eel.channels.EelDelicateApi
|
||||
import com.intellij.platform.eel.fs.*
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi.FileWriterCreationMode.*
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
@@ -30,12 +31,8 @@ import java.nio.file.attribute.*
|
||||
import java.time.Instant
|
||||
import java.time.ZoneId
|
||||
import java.time.ZonedDateTime
|
||||
import kotlin.io.path.exists
|
||||
import kotlin.io.path.fileAttributesView
|
||||
import kotlin.io.path.fileStore
|
||||
import kotlin.io.path.isDirectory
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
import kotlin.io.path.pathString
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
import kotlin.io.path.*
|
||||
import kotlin.streams.asSequence
|
||||
|
||||
abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) : EelFileSystemApi {
|
||||
@@ -51,10 +48,16 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
|
||||
}
|
||||
|
||||
inline fun <T, reified E : EelFsError> wrapIntoEelResult(body: () -> T): EelResult<T, E> =
|
||||
wrapIntoEelResult(null, body)
|
||||
|
||||
inline fun <T, reified E : EelFsError> wrapIntoEelResult(isClosed: AtomicReference<Boolean?>?, body: () -> T): EelResult<T, E> =
|
||||
try {
|
||||
EelFsResultImpl.Ok(body())
|
||||
val result = EelFsResultImpl.Ok(body())
|
||||
isClosed?.updateAndGet { it ?: false }
|
||||
result
|
||||
}
|
||||
catch (err: FileSystemException) {
|
||||
isClosed?.set(true)
|
||||
val path =
|
||||
try {
|
||||
EelPath.parse(err.file.toString(), LocalEelDescriptor)
|
||||
@@ -122,15 +125,58 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
|
||||
Files.isSameFile(source.toNioPath(), target.toNioPath())
|
||||
}
|
||||
|
||||
override suspend fun openForReading(path: EelPath): EelResult<
|
||||
@OptIn(EelDelicateApi::class)
|
||||
override suspend fun openForReading(args: EelFileSystemApi.OpenForReadingArgs): EelResult<
|
||||
EelOpenedFile.Reader,
|
||||
EelFileSystemApi.FileReaderError
|
||||
> =
|
||||
wrapIntoEelResult {
|
||||
val nioPath = path.toNioPath()
|
||||
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, setOf(StandardOpenOption.READ))
|
||||
LocalEelOpenedFileReader(this, byteChannel, path)
|
||||
> {
|
||||
val result = openForReadingImpl(
|
||||
path = args.path,
|
||||
autoCloseAfterLastChunk = args.autoCloseAfterLastChunk,
|
||||
closeImmediatelyIfFileBiggerThan = args.closeImmediatelyIfFileBiggerThan,
|
||||
)
|
||||
// Not filling the buffer. This optimization makes no sense for the local case.
|
||||
args.readFirstChunkInto?.run {
|
||||
position(0)
|
||||
limit(0)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@OptIn(EelDelicateApi::class)
|
||||
private fun openForReadingImpl(
|
||||
path: EelPath,
|
||||
autoCloseAfterLastChunk: Boolean,
|
||||
closeImmediatelyIfFileBiggerThan: Long?,
|
||||
): EelResult<LocalEelOpenedFileReader, EelFileSystemApi.FileReaderError> = wrapIntoEelResult {
|
||||
val nioPath = path.toNioPath()
|
||||
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, setOf(StandardOpenOption.READ))
|
||||
val isClosed = AtomicReference<Boolean?>(null)
|
||||
|
||||
if (closeImmediatelyIfFileBiggerThan != null) {
|
||||
if (byteChannel.size() > closeImmediatelyIfFileBiggerThan) {
|
||||
isClosed.set(true)
|
||||
try {
|
||||
doClose(this@NioBasedEelFileSystemApi, byteChannel, path)
|
||||
}
|
||||
catch (_: IOException) {
|
||||
// Ignored.
|
||||
}
|
||||
return EelFsResultImpl.Error(EelFsResultImpl.FileBiggerThanRequested(path, "The file is bigger than the requested size"))
|
||||
}
|
||||
}
|
||||
|
||||
LocalEelOpenedFileReader(
|
||||
eelFs = this,
|
||||
byteChannel = byteChannel,
|
||||
path_ = path,
|
||||
isClosed_ = isClosed,
|
||||
autoCloseAfterLastChunk = autoCloseAfterLastChunk,
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun readFile(args: EelFileSystemApi.ReadFileArgs): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
|
||||
readFileImpl(args)
|
||||
|
||||
override suspend fun openForWriting(options: EelFileSystemApi.WriteOptions): EelResult<
|
||||
EelOpenedFile.Writer,
|
||||
@@ -140,7 +186,7 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
|
||||
val nioPath = path.toNioPath()
|
||||
val nioOptions = writeOptionsToNioOptions(options)
|
||||
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, nioOptions)
|
||||
LocalEelOpenedFileWriter(this, byteChannel, path)
|
||||
LocalEelOpenedFileWriter(this, byteChannel, path, AtomicReference(null))
|
||||
}
|
||||
|
||||
private fun writeOptionsToNioOptions(options: EelFileSystemApi.WriteOptions): MutableSet<StandardOpenOption> {
|
||||
@@ -168,15 +214,16 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
|
||||
val nioOptions = writeOptionsToNioOptions(options)
|
||||
nioOptions += StandardOpenOption.READ
|
||||
val byteChannel: SeekableByteChannel = nioPath.fileSystem.provider().newByteChannel(nioPath, nioOptions)
|
||||
object : EelOpenedFile.ReaderWriter, EelOpenedFile.Writer by LocalEelOpenedFileWriter(this, byteChannel, path) {
|
||||
val isClosed = AtomicReference<Boolean?>(null)
|
||||
object : EelOpenedFile.ReaderWriter, EelOpenedFile.Writer by LocalEelOpenedFileWriter(this, byteChannel, path, isClosed) {
|
||||
override suspend fun read(buf: ByteBuffer): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
|
||||
doRead(this@NioBasedEelFileSystemApi, byteChannel, buf)
|
||||
doRead(this@NioBasedEelFileSystemApi, byteChannel, buf, isClosed, autoCloseAfterLastChunk = false)
|
||||
|
||||
override suspend fun read(
|
||||
buf: ByteBuffer,
|
||||
offset: Long,
|
||||
): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
|
||||
doRead(this@NioBasedEelFileSystemApi, byteChannel, offset, buf)
|
||||
doRead(this@NioBasedEelFileSystemApi, byteChannel, offset, buf, isClosed)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -231,32 +278,40 @@ abstract class NioBasedEelFileSystemApi(@VisibleForTesting val fs: FileSystem) :
|
||||
EelPathUtils.walkingTransfer(sourceNioPath, targetNioPath, removeSource = true, copyAttributes = true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@EelDelicateApi
|
||||
internal class LocalEelOpenedFileReader(
|
||||
private val eelFs: NioBasedEelFileSystemApi,
|
||||
private val byteChannel: SeekableByteChannel,
|
||||
private val path_: EelPath,
|
||||
private val isClosed_: AtomicReference<Boolean?>,
|
||||
private val autoCloseAfterLastChunk: Boolean,
|
||||
) : EelOpenedFile.Reader {
|
||||
override val isClosed: Boolean?
|
||||
get() = isClosed_.get()
|
||||
|
||||
override suspend fun read(buf: ByteBuffer): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
|
||||
doRead(eelFs, byteChannel, buf)
|
||||
doRead(eelFs, byteChannel, buf, isClosed_, autoCloseAfterLastChunk)
|
||||
|
||||
override suspend fun read(buf: ByteBuffer, offset: Long): EelResult<
|
||||
ReadResult,
|
||||
EelOpenedFile.Reader.ReadError
|
||||
> =
|
||||
doRead(eelFs, byteChannel, offset, buf)
|
||||
doRead(eelFs, byteChannel, offset, buf, isClosed_)
|
||||
|
||||
override val path: EelPath = path_
|
||||
|
||||
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> =
|
||||
doClose(eelFs, byteChannel, path_)
|
||||
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> {
|
||||
isClosed_.set(true)
|
||||
return doClose(eelFs, byteChannel, path_)
|
||||
}
|
||||
|
||||
override suspend fun tell(): EelResult<Long, EelOpenedFile.TellError> =
|
||||
doTell(eelFs, byteChannel, path_)
|
||||
doTell(eelFs, byteChannel, isClosed_)
|
||||
|
||||
override suspend fun seek(offset: Long, whence: EelOpenedFile.SeekWhence): EelResult<Long, EelOpenedFile.SeekError> =
|
||||
doSeek(eelFs, byteChannel, path_, whence, offset)
|
||||
doSeek(eelFs, byteChannel, path_, whence, offset, isClosed_)
|
||||
|
||||
override suspend fun stat(): EelResult<EelFileInfo, EelFileSystemApi.StatError> =
|
||||
eelFs.stat(path_, EelFileSystemApi.SymlinkPolicy.RESOLVE_AND_FOLLOW)
|
||||
@@ -266,14 +321,19 @@ private class LocalEelOpenedFileWriter(
|
||||
private val eelFs: NioBasedEelFileSystemApi,
|
||||
private val byteChannel: SeekableByteChannel,
|
||||
private val path_: EelPath,
|
||||
private val isClosed_: AtomicReference<Boolean?>,
|
||||
) : EelOpenedFile.Writer {
|
||||
@EelDelicateApi
|
||||
override val isClosed: Boolean?
|
||||
get() = isClosed_.get()
|
||||
|
||||
override suspend fun write(buf: ByteBuffer): EelResult<Int, EelOpenedFile.Writer.WriteError> =
|
||||
eelFs.wrapIntoEelResult {
|
||||
eelFs.wrapIntoEelResult(isClosed_) {
|
||||
byteChannel.write(buf)
|
||||
}
|
||||
|
||||
override suspend fun write(buf: ByteBuffer, pos: Long): EelResult<Int, EelOpenedFile.Writer.WriteError> =
|
||||
eelFs.wrapIntoEelResult {
|
||||
eelFs.wrapIntoEelResult(isClosed_) {
|
||||
val oldPosition = byteChannel.position()
|
||||
byteChannel.position(pos)
|
||||
val written = byteChannel.write(buf)
|
||||
@@ -286,20 +346,22 @@ private class LocalEelOpenedFileWriter(
|
||||
}
|
||||
|
||||
override suspend fun truncate(size: Long): EelResult<Unit, EelOpenedFile.Writer.TruncateError> =
|
||||
eelFs.wrapIntoEelResult {
|
||||
eelFs.wrapIntoEelResult(isClosed_) {
|
||||
byteChannel.truncate(size)
|
||||
}
|
||||
|
||||
override val path: EelPath = path_
|
||||
|
||||
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> =
|
||||
doClose(eelFs, byteChannel, path_)
|
||||
override suspend fun close(): EelResult<Unit, EelOpenedFile.CloseError> {
|
||||
isClosed_.set(true)
|
||||
return doClose(eelFs, byteChannel, path_)
|
||||
}
|
||||
|
||||
override suspend fun tell(): EelResult<Long, EelOpenedFile.TellError> =
|
||||
doTell(eelFs, byteChannel, path_)
|
||||
doTell(eelFs, byteChannel, isClosed_)
|
||||
|
||||
override suspend fun seek(offset: Long, whence: EelOpenedFile.SeekWhence): EelResult<Long, EelOpenedFile.SeekError> =
|
||||
doSeek(eelFs, byteChannel, path_, whence, offset)
|
||||
doSeek(eelFs, byteChannel, path_, whence, offset, isClosed_)
|
||||
|
||||
override suspend fun stat(): EelResult<EelFileInfo, EelFileSystemApi.StatError> =
|
||||
eelFs.stat(path_, EelFileSystemApi.SymlinkPolicy.RESOLVE_AND_FOLLOW)
|
||||
@@ -309,11 +371,19 @@ private fun doRead(
|
||||
eelFs: NioBasedEelFileSystemApi,
|
||||
byteChannel: SeekableByteChannel,
|
||||
buf: ByteBuffer,
|
||||
isClosed: AtomicReference<Boolean?>,
|
||||
autoCloseAfterLastChunk: Boolean,
|
||||
): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
|
||||
eelFs.wrapIntoEelResult {
|
||||
eelFs.wrapIntoEelResult(isClosed) {
|
||||
val read = byteChannel.read(buf)
|
||||
|
||||
ReadResult.fromNumberOfReadBytes(read)
|
||||
val result = ReadResult.fromNumberOfReadBytes(read)
|
||||
if (autoCloseAfterLastChunk) {
|
||||
when (result) {
|
||||
ReadResult.EOF -> isClosed.set(true)
|
||||
ReadResult.NOT_EOF -> isClosed.set(false)
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
private fun doRead(
|
||||
@@ -321,8 +391,9 @@ private fun doRead(
|
||||
byteChannel: SeekableByteChannel,
|
||||
offset: Long,
|
||||
buf: ByteBuffer,
|
||||
isClosed: AtomicReference<Boolean?>,
|
||||
): EelResult<ReadResult, EelOpenedFile.Reader.ReadError> =
|
||||
eelFs.wrapIntoEelResult {
|
||||
eelFs.wrapIntoEelResult(isClosed) {
|
||||
val oldPosition = byteChannel.position()
|
||||
byteChannel.position(offset)
|
||||
val read = byteChannel.read(buf)
|
||||
@@ -337,7 +408,8 @@ private fun doSeek(
|
||||
path: EelPath,
|
||||
whence: EelOpenedFile.SeekWhence,
|
||||
offset: Long,
|
||||
): EelResult<Long, EelOpenedFile.SeekError> = eelFs.wrapIntoEelResult {
|
||||
isClosed: AtomicReference<Boolean?>,
|
||||
): EelResult<Long, EelOpenedFile.SeekError> = eelFs.wrapIntoEelResult(isClosed) {
|
||||
val newPosition = when (whence) {
|
||||
EelOpenedFile.SeekWhence.START -> offset
|
||||
EelOpenedFile.SeekWhence.CURRENT -> byteChannel.position() + offset
|
||||
@@ -347,8 +419,12 @@ private fun doSeek(
|
||||
newPosition
|
||||
}
|
||||
|
||||
private fun doTell(eelFs: NioBasedEelFileSystemApi, byteChannel: SeekableByteChannel, path: EelPath): EelResult<Long, EelOpenedFile.TellError> =
|
||||
eelFs.wrapIntoEelResult {
|
||||
private fun doTell(
|
||||
eelFs: NioBasedEelFileSystemApi,
|
||||
byteChannel: SeekableByteChannel,
|
||||
isClosed: AtomicReference<Boolean?>,
|
||||
): EelResult<Long, EelOpenedFile.TellError> =
|
||||
eelFs.wrapIntoEelResult(isClosed) {
|
||||
byteChannel.position().toLong()
|
||||
}
|
||||
|
||||
@@ -424,10 +500,6 @@ abstract class PosixNioBasedEelFileSystemApi(
|
||||
Files.createSymbolicLink(linkPath.toNioPath(), targetPath)
|
||||
}
|
||||
|
||||
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
override suspend fun walkDirectory(options: EelFileSystemApi.WalkDirectoryOptions): Flow<WalkDirectoryEntryResult> = flow {
|
||||
val rootDir = options.path.asNioPath()
|
||||
|
||||
@@ -514,7 +586,7 @@ abstract class PosixNioBasedEelFileSystemApi(
|
||||
currentItem: Path,
|
||||
sourceAttrs: PosixFileAttributes,
|
||||
emptyFileHash: Long,
|
||||
options: EelFileSystemApi.WalkDirectoryOptions
|
||||
options: EelFileSystemApi.WalkDirectoryOptions,
|
||||
): WalkDirectoryEntryResult? {
|
||||
var creationTime: ZonedDateTime? = null
|
||||
var lastModifiedTime: ZonedDateTime? = null
|
||||
@@ -656,10 +728,6 @@ abstract class WindowsNioBasedEelFileSystemApi(
|
||||
// TODO File permissions for windows.
|
||||
}
|
||||
|
||||
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
override suspend fun walkDirectory(options: EelFileSystemApi.WalkDirectoryOptions): Flow<WalkDirectoryEntryResult> = flow {
|
||||
val rootDir = options.path.asNioPath()
|
||||
|
||||
@@ -746,7 +814,7 @@ abstract class WindowsNioBasedEelFileSystemApi(
|
||||
currentItem: Path,
|
||||
sourceAttrs: DosFileAttributes,
|
||||
emptyFileHash: Long,
|
||||
options: EelFileSystemApi.WalkDirectoryOptions
|
||||
options: EelFileSystemApi.WalkDirectoryOptions,
|
||||
): WalkDirectoryEntryResult? {
|
||||
var creationTime: ZonedDateTime? = null
|
||||
var lastModifiedTime: ZonedDateTime? = null
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
package com.intellij.platform.eel.provider.utils
|
||||
|
||||
import com.intellij.openapi.progress.runBlockingMaybeCancellable
|
||||
import com.intellij.openapi.util.io.FileTooBigException
|
||||
import com.intellij.platform.eel.EelExecApi
|
||||
import com.intellij.platform.eel.EelPlatform
|
||||
import com.intellij.platform.eel.EelResult
|
||||
@@ -87,6 +88,8 @@ fun EelFsError.throwFileSystemException(): Nothing {
|
||||
-> throw IllegalArgumentException(message)
|
||||
is EelOpenedFile.Writer.WriteError.InvalidValue -> throw IllegalArgumentException(message)
|
||||
is EelFileSystemApi.DeleteError.UnresolvedLink -> throw FileSystemException(where.toString(), null, message)
|
||||
is EelFileSystemApi.FileReaderError.FileBiggerThanRequested ->
|
||||
throw FileSystemException(where.toString(), null, "File is too big").apply { initCause(FileTooBigException("File is too big")) }
|
||||
is EelFsError.Other -> FileSystemException(where.toString(), null, message.nullize())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ package com.intellij.platform.eel.fs
|
||||
import com.intellij.platform.eel.EelResult
|
||||
import com.intellij.platform.eel.GeneratedBuilder
|
||||
import com.intellij.platform.eel.OwnedBuilder
|
||||
import com.intellij.platform.eel.channels.EelDelicateApi
|
||||
import com.intellij.platform.eel.fs.EelFileInfo.Permissions
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi.FileChangeType
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi.FileWriterCreationMode
|
||||
@@ -23,6 +24,7 @@ import com.intellij.platform.eel.path.EelPath
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import org.jetbrains.annotations.ApiStatus
|
||||
import org.jetbrains.annotations.CheckReturnValue
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
|
||||
@GeneratedBuilder.Result
|
||||
@@ -91,6 +93,39 @@ fun EelFileSystemApi.move(
|
||||
target = target,
|
||||
)
|
||||
|
||||
/**
|
||||
* Opens the file only for reading.
|
||||
*
|
||||
* In many cases [readFile] suits better than [openForReading].
|
||||
*/
|
||||
@GeneratedBuilder.Result
|
||||
@ApiStatus.Internal
|
||||
fun EelFileSystemApi.openForReading(
|
||||
path: EelPath,
|
||||
): EelFileSystemApiHelpers.OpenForReading =
|
||||
EelFileSystemApiHelpers.OpenForReading(
|
||||
owner = this,
|
||||
path = path,
|
||||
)
|
||||
|
||||
/**
|
||||
* Fully or partially reads the file.
|
||||
*
|
||||
* Although it's possible to implement file reading with [openForReading],
|
||||
* this function is optimized and covered with tests.
|
||||
*
|
||||
* The returned [ReadFileResult.bytes] is prepared for reading.
|
||||
*/
|
||||
@GeneratedBuilder.Result
|
||||
@ApiStatus.Internal
|
||||
fun EelFileSystemApi.readFile(
|
||||
path: EelPath,
|
||||
): EelFileSystemApiHelpers.ReadFile =
|
||||
EelFileSystemApiHelpers.ReadFile(
|
||||
owner = this,
|
||||
path = path,
|
||||
)
|
||||
|
||||
/**
|
||||
* Similar to stat(2) and lstat(2). [symlinkPolicy] has an impact only on [EelFileInfo.type] if [path] points on a symlink.
|
||||
*/
|
||||
@@ -494,6 +529,156 @@ object EelFileSystemApiHelpers {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.openForReading].
|
||||
*/
|
||||
@GeneratedBuilder.Result
|
||||
@ApiStatus.Internal
|
||||
class OpenForReading(
|
||||
private val owner: EelFileSystemApi,
|
||||
private var path: EelPath,
|
||||
) : OwnedBuilder<EelResult<EelOpenedFile.Reader, EelFileSystemApi.FileReaderError>> {
|
||||
private var autoCloseAfterLastChunk: Boolean = false
|
||||
|
||||
private var closeImmediatelyIfFileBiggerThan: Long? = null
|
||||
|
||||
private var readFirstChunkInto: ByteBuffer? = null
|
||||
|
||||
/**
|
||||
* When specified, the implementation closes its internal file descriptor
|
||||
* as soon as it internally reaches the end of the file.
|
||||
*
|
||||
* There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read]
|
||||
* * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call.
|
||||
* * By checking inexpensive but unreliable [EelOpenedFile.isClosed].
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun autoCloseAfterLastChunk(arg: Boolean): OpenForReading = apply {
|
||||
this.autoCloseAfterLastChunk = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* An optimization suitable for reading into memory.
|
||||
* It allows aborting the reading fast if the whole file content
|
||||
* won't fit into some buffer.
|
||||
*
|
||||
* If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested].
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun closeImmediatelyIfFileBiggerThan(arg: Long?): OpenForReading = apply {
|
||||
this.closeImmediatelyIfFileBiggerThan = arg
|
||||
}
|
||||
|
||||
fun path(arg: EelPath): OpenForReading = apply {
|
||||
this.path = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* When specified, data from the file MAY be written into this buffer.
|
||||
* [ByteBuffer.position] and [ByteBuffer.limit] are always changed.
|
||||
* The buffer is prepared for reading after the call,
|
||||
* so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading].
|
||||
*
|
||||
* If some data is written into this buffer,
|
||||
* the first call of [EelOpenedFile.Reader.read] reads the data following this buffer.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun readFirstChunkInto(arg: ByteBuffer?): OpenForReading = apply {
|
||||
this.readFirstChunkInto = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete the builder and call [com.intellij.platform.eel.fs.EelFileSystemApi.openForReading]
|
||||
* with an instance of [com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs].
|
||||
*/
|
||||
@CheckReturnValue
|
||||
override suspend fun eelIt(): EelResult<EelOpenedFile.Reader, EelFileSystemApi.FileReaderError> =
|
||||
owner.openForReading(
|
||||
OpenForReadingArgsImpl(
|
||||
autoCloseAfterLastChunk = autoCloseAfterLastChunk,
|
||||
closeImmediatelyIfFileBiggerThan = closeImmediatelyIfFileBiggerThan,
|
||||
path = path,
|
||||
readFirstChunkInto = readFirstChunkInto,
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.readFile].
|
||||
*/
|
||||
@GeneratedBuilder.Result
|
||||
@ApiStatus.Internal
|
||||
class ReadFile(
|
||||
private val owner: EelFileSystemApi,
|
||||
private var path: EelPath,
|
||||
) : OwnedBuilder<EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError>> {
|
||||
private var buffer: ByteBuffer? = null
|
||||
|
||||
private var failFastIfBeyondLimit: Boolean = false
|
||||
|
||||
private var limit: Int? = null
|
||||
|
||||
private var mayReturnSameBuffer: Boolean = true
|
||||
|
||||
/**
|
||||
* Use some specific buffer for reading files instead of creating a temporary buffer.
|
||||
*
|
||||
* The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call.
|
||||
*
|
||||
* The buffer is ready for reading, no need to call `flip`.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun buffer(arg: ByteBuffer?): ReadFile = apply {
|
||||
this.buffer = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* If this flag is set, the implementation checks the file size before trying to read data,
|
||||
* and if the file is certainly bigger than [limit], no data is read.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun failFastIfBeyondLimit(arg: Boolean): ReadFile = apply {
|
||||
this.failFastIfBeyondLimit = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* Maximal number of bytes to read.
|
||||
*/
|
||||
fun limit(arg: Int?): ReadFile = apply {
|
||||
this.limit = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* If this flag is set and [buffer] is specified, the implementation reads the whole
|
||||
* file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer.
|
||||
*
|
||||
* If [buffer] is not specified, the value of the flag is ignored.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun mayReturnSameBuffer(arg: Boolean): ReadFile = apply {
|
||||
this.mayReturnSameBuffer = arg
|
||||
}
|
||||
|
||||
fun path(arg: EelPath): ReadFile = apply {
|
||||
this.path = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete the builder and call [com.intellij.platform.eel.fs.EelFileSystemApi.readFile]
|
||||
* with an instance of [com.intellij.platform.eel.fs.EelFileSystemApi.ReadFileArgs].
|
||||
*/
|
||||
override suspend fun eelIt(): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
|
||||
owner.readFile(
|
||||
ReadFileArgsImpl(
|
||||
buffer = buffer,
|
||||
failFastIfBeyondLimit = failFastIfBeyondLimit,
|
||||
limit = limit,
|
||||
mayReturnSameBuffer = mayReturnSameBuffer,
|
||||
path = path,
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create it via [com.intellij.platform.eel.fs.EelFileSystemApi.stat].
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
/**
|
||||
* This file is generated by [com.intellij.platform.eel.codegen.BuildersGeneratorTest].
|
||||
*/
|
||||
package com.intellij.platform.eel.fs
|
||||
|
||||
import com.intellij.platform.eel.GeneratedBuilder
|
||||
import com.intellij.platform.eel.channels.EelDelicateApi
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
import org.jetbrains.annotations.ApiStatus
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
|
||||
@GeneratedBuilder.Result
|
||||
@ApiStatus.Internal
|
||||
class OpenForReadingArgsBuilder(
|
||||
private var path: EelPath,
|
||||
) {
|
||||
private var autoCloseAfterLastChunk: Boolean = false
|
||||
|
||||
private var closeImmediatelyIfFileBiggerThan: Long? = null
|
||||
|
||||
private var readFirstChunkInto: ByteBuffer? = null
|
||||
|
||||
/**
|
||||
* When specified, the implementation closes its internal file descriptor
|
||||
* as soon as it internally reaches the end of the file.
|
||||
*
|
||||
* There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read]
|
||||
* * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call.
|
||||
* * By checking inexpensive but unreliable [EelOpenedFile.isClosed].
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun autoCloseAfterLastChunk(arg: Boolean): OpenForReadingArgsBuilder = apply {
|
||||
this.autoCloseAfterLastChunk = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* An optimization suitable for reading into memory.
|
||||
* It allows aborting the reading fast if the whole file content
|
||||
* won't fit into some buffer.
|
||||
*
|
||||
* If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested].
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun closeImmediatelyIfFileBiggerThan(arg: Long?): OpenForReadingArgsBuilder = apply {
|
||||
this.closeImmediatelyIfFileBiggerThan = arg
|
||||
}
|
||||
|
||||
fun path(arg: EelPath): OpenForReadingArgsBuilder = apply {
|
||||
this.path = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* When specified, data from the file MAY be written into this buffer.
|
||||
* [ByteBuffer.position] and [ByteBuffer.limit] are always changed.
|
||||
* The buffer is prepared for reading after the call,
|
||||
* so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading].
|
||||
*
|
||||
* If some data is written into this buffer,
|
||||
* the first call of [EelOpenedFile.Reader.read] reads the data following this buffer.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun readFirstChunkInto(arg: ByteBuffer?): OpenForReadingArgsBuilder = apply {
|
||||
this.readFirstChunkInto = arg
|
||||
}
|
||||
|
||||
fun build(): OpenForReadingArgs =
|
||||
OpenForReadingArgsImpl(
|
||||
autoCloseAfterLastChunk = autoCloseAfterLastChunk,
|
||||
closeImmediatelyIfFileBiggerThan = closeImmediatelyIfFileBiggerThan,
|
||||
path = path,
|
||||
readFirstChunkInto = readFirstChunkInto,
|
||||
)
|
||||
}
|
||||
|
||||
@GeneratedBuilder.Result
|
||||
internal class OpenForReadingArgsImpl(
|
||||
override val autoCloseAfterLastChunk: Boolean,
|
||||
override val closeImmediatelyIfFileBiggerThan: Long?,
|
||||
override val path: EelPath,
|
||||
override val readFirstChunkInto: ByteBuffer?,
|
||||
) : OpenForReadingArgs
|
||||
@@ -0,0 +1,88 @@
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
/**
|
||||
* This file is generated by [com.intellij.platform.eel.codegen.BuildersGeneratorTest].
|
||||
*/
|
||||
package com.intellij.platform.eel.fs
|
||||
|
||||
import com.intellij.platform.eel.GeneratedBuilder
|
||||
import com.intellij.platform.eel.channels.EelDelicateApi
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi.ReadFileArgs
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
import org.jetbrains.annotations.ApiStatus
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
|
||||
@GeneratedBuilder.Result
|
||||
@ApiStatus.Internal
|
||||
class ReadFileArgsBuilder(
|
||||
private var path: EelPath,
|
||||
) {
|
||||
private var buffer: ByteBuffer? = null
|
||||
|
||||
private var failFastIfBeyondLimit: Boolean = false
|
||||
|
||||
private var limit: Int? = null
|
||||
|
||||
private var mayReturnSameBuffer: Boolean = true
|
||||
|
||||
/**
|
||||
* Use some specific buffer for reading files instead of creating a temporary buffer.
|
||||
*
|
||||
* The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call.
|
||||
*
|
||||
* The buffer is ready for reading, no need to call `flip`.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun buffer(arg: ByteBuffer?): ReadFileArgsBuilder = apply {
|
||||
this.buffer = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* If this flag is set, the implementation checks the file size before trying to read data,
|
||||
* and if the file is certainly bigger than [limit], no data is read.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun failFastIfBeyondLimit(arg: Boolean): ReadFileArgsBuilder = apply {
|
||||
this.failFastIfBeyondLimit = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* Maximal number of bytes to read.
|
||||
*/
|
||||
fun limit(arg: Int?): ReadFileArgsBuilder = apply {
|
||||
this.limit = arg
|
||||
}
|
||||
|
||||
/**
|
||||
* If this flag is set and [buffer] is specified, the implementation reads the whole
|
||||
* file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer.
|
||||
*
|
||||
* If [buffer] is not specified, the value of the flag is ignored.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
fun mayReturnSameBuffer(arg: Boolean): ReadFileArgsBuilder = apply {
|
||||
this.mayReturnSameBuffer = arg
|
||||
}
|
||||
|
||||
fun path(arg: EelPath): ReadFileArgsBuilder = apply {
|
||||
this.path = arg
|
||||
}
|
||||
|
||||
fun build(): ReadFileArgs =
|
||||
ReadFileArgsImpl(
|
||||
buffer = buffer,
|
||||
failFastIfBeyondLimit = failFastIfBeyondLimit,
|
||||
limit = limit,
|
||||
mayReturnSameBuffer = mayReturnSameBuffer,
|
||||
path = path,
|
||||
)
|
||||
}
|
||||
|
||||
@GeneratedBuilder.Result
|
||||
internal class ReadFileArgsImpl(
|
||||
override val buffer: ByteBuffer?,
|
||||
override val failFastIfBeyondLimit: Boolean,
|
||||
override val limit: Int?,
|
||||
override val mayReturnSameBuffer: Boolean,
|
||||
override val path: EelPath,
|
||||
) : ReadFileArgs
|
||||
@@ -10,4 +10,13 @@ import org.jetbrains.annotations.ApiStatus
|
||||
message = "This function might send only part of the buffer. Consider using `sendWholeBuffer` instead"
|
||||
)
|
||||
@ApiStatus.Internal
|
||||
annotation class EelSendApi
|
||||
annotation class EelSendApi
|
||||
|
||||
@MustBeDocumented
|
||||
@Retention(value = AnnotationRetention.BINARY)
|
||||
@RequiresOptIn(
|
||||
level = RequiresOptIn.Level.WARNING,
|
||||
message = "Ensure that you read the documentation thoroughly or that you're ready to shoot your foot"
|
||||
)
|
||||
@ApiStatus.Internal
|
||||
annotation class EelDelicateApi
|
||||
@@ -2,6 +2,7 @@
|
||||
package com.intellij.platform.eel.fs
|
||||
|
||||
import com.intellij.platform.eel.*
|
||||
import com.intellij.platform.eel.channels.EelDelicateApi
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi.StatError
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
@@ -178,46 +179,120 @@ interface EelFileSystemApi {
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the file only for reading
|
||||
* Opens the file only for reading.
|
||||
*
|
||||
* In many cases [readFile] suits better than [openForReading].
|
||||
*/
|
||||
@CheckReturnValue
|
||||
suspend fun openForReading(path: EelPath): EelResult<
|
||||
suspend fun openForReading(@GeneratedBuilder args: OpenForReadingArgs): EelResult<
|
||||
EelOpenedFile.Reader,
|
||||
FileReaderError>
|
||||
|
||||
interface OpenForReadingArgs {
|
||||
val path: EelPath
|
||||
|
||||
/**
|
||||
* When specified, data from the file MAY be written into this buffer.
|
||||
* [ByteBuffer.position] and [ByteBuffer.limit] are always changed.
|
||||
* The buffer is prepared for reading after the call,
|
||||
* so the caller SHOULD NOT call [ByteBuffer.flip] after calling [openForReading].
|
||||
*
|
||||
* If some data is written into this buffer,
|
||||
* the first call of [EelOpenedFile.Reader.read] reads the data following this buffer.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
val readFirstChunkInto: ByteBuffer? get() = null
|
||||
|
||||
/**
|
||||
* When specified, the implementation closes its internal file descriptor
|
||||
* as soon as it internally reaches the end of the file.
|
||||
*
|
||||
* There are two ways to figure out if the file is closed after calling [openForReading] or [EelOpenedFile.Reader.read]
|
||||
* * By calling [EelOpenedFile.Reader.read], which implies an additional system call or an RPC call.
|
||||
* * By checking inexpensive but unreliable [EelOpenedFile.isClosed].
|
||||
*/
|
||||
@EelDelicateApi
|
||||
val autoCloseAfterLastChunk: Boolean get() = false
|
||||
|
||||
/**
|
||||
* An optimization suitable for reading into memory.
|
||||
* It allows aborting the reading fast if the whole file content
|
||||
* won't fit into some buffer.
|
||||
*
|
||||
* If it happens, [readFile] returns [FileReaderError.FileBiggerThanRequested].
|
||||
*/
|
||||
@EelDelicateApi
|
||||
val closeImmediatelyIfFileBiggerThan: Long? get() = null
|
||||
}
|
||||
|
||||
sealed interface FileReaderError : EelFsError {
|
||||
interface AlreadyExists : FileReaderError, EelFsError.AlreadyExists
|
||||
interface DoesNotExist : FileReaderError, EelFsError.DoesNotExist
|
||||
interface PermissionDenied : FileReaderError, EelFsError.PermissionDenied
|
||||
interface NotDirectory : FileReaderError, EelFsError.NotDirectory
|
||||
interface NotFile : FileReaderError, EelFsError.NotFile
|
||||
|
||||
/** See [com.intellij.platform.eel.fs.EelFileSystemApi.OpenForReadingArgs.closeImmediatelyIfFileBiggerThan] */
|
||||
interface FileBiggerThanRequested : FileReaderError
|
||||
|
||||
interface Other : FileReaderError, EelFsError.Other
|
||||
}
|
||||
|
||||
enum class OverflowPolicy {
|
||||
DROP,
|
||||
RETAIN,
|
||||
/**
|
||||
* Fully or partially reads the file.
|
||||
*
|
||||
* Although it's possible to implement file reading with [openForReading],
|
||||
* this function is optimized and covered with tests.
|
||||
*
|
||||
* The returned [ReadFileResult.bytes] is prepared for reading.
|
||||
*/
|
||||
suspend fun readFile(@GeneratedBuilder args: ReadFileArgs): EelResult<ReadFileResult, FileReaderError>
|
||||
|
||||
interface ReadFileArgs {
|
||||
val path: EelPath
|
||||
|
||||
/**
|
||||
* Maximal number of bytes to read.
|
||||
*/
|
||||
val limit: Int? get() = null
|
||||
|
||||
/**
|
||||
* If this flag is set, the implementation checks the file size before trying to read data,
|
||||
* and if the file is certainly bigger than [limit], no data is read.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
val failFastIfBeyondLimit: Boolean get() = false
|
||||
|
||||
/**
|
||||
* Use some specific buffer for reading files instead of creating a temporary buffer.
|
||||
*
|
||||
* The implementation MAY use only a fraction of this buffer for invoking a single system or RPC call.
|
||||
*
|
||||
* The buffer is ready for reading, no need to call `flip`.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
val buffer: ByteBuffer? get() = null
|
||||
|
||||
/**
|
||||
* If this flag is set and [buffer] is specified, the implementation reads the whole
|
||||
* file into [buffer] and [ReadFileResult.bytes] contains a reference to [buffer]. However, if the file size is greater than the capacity of the buffer, the implementation returns a different buffer.
|
||||
*
|
||||
* If [buffer] is not specified, the value of the flag is ignored.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
val mayReturnSameBuffer: Boolean get() = true
|
||||
}
|
||||
|
||||
sealed interface FullReadResult {
|
||||
interface Overflow : FullReadResult
|
||||
interface BytesOverflown : FullReadResult {
|
||||
val bytes: ByteArray
|
||||
}
|
||||
interface ReadFileResult {
|
||||
/**
|
||||
* It's ready for reading, the position and the limit are already set.
|
||||
*/
|
||||
val bytes: ByteBuffer
|
||||
|
||||
interface Bytes : FullReadResult {
|
||||
val bytes: ByteArray
|
||||
}
|
||||
}
|
||||
|
||||
@CheckReturnValue
|
||||
suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: OverflowPolicy): EelResult<FullReadResult, FullReadError>
|
||||
|
||||
sealed interface FullReadError : EelFsError {
|
||||
interface DoesNotExist : FullReadError, EelFsError.DoesNotExist
|
||||
interface PermissionDenied : FullReadError, EelFsError.PermissionDenied
|
||||
interface NotFile : FullReadError, EelFsError.NotFile
|
||||
interface Other : FullReadError, EelFsError.Other
|
||||
/**
|
||||
* `true` if [bytes] contains the whole file, `false` only if the part of the file.
|
||||
*/
|
||||
val fullyRead: Boolean
|
||||
}
|
||||
|
||||
interface WalkDirectoryOptions {
|
||||
@@ -754,6 +829,18 @@ sealed interface EelOpenedFile {
|
||||
@CheckReturnValue
|
||||
suspend fun close(): EelResult<Unit, CloseError>
|
||||
|
||||
/**
|
||||
* This method is to be used for avoiding potentially excessive calls.
|
||||
* However, rely on this function with suspicion: the implementation may return `null` whatever happens.
|
||||
*
|
||||
* Returns:
|
||||
* * `true` if the file is closed.
|
||||
* * `false` if it's not closed.
|
||||
* * `null` if it's not possible to determine if the file is closed without calling any suspending method.
|
||||
*/
|
||||
@EelDelicateApi
|
||||
val isClosed: Boolean?
|
||||
|
||||
sealed interface CloseError : EelFsError {
|
||||
interface Other : CloseError, EelFsError.Other
|
||||
}
|
||||
|
||||
+10
-6
@@ -1,4 +1,4 @@
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package com.intellij.platform.ijent.community.impl
|
||||
|
||||
import com.intellij.platform.eel.EelDescriptor
|
||||
@@ -9,6 +9,7 @@ import com.intellij.platform.eel.fs.EelFileSystemApi
|
||||
import com.intellij.platform.eel.fs.EelFileSystemPosixApi
|
||||
import com.intellij.platform.eel.fs.EelOpenedFile
|
||||
import com.intellij.platform.eel.fs.EelPosixFileInfo
|
||||
import com.intellij.platform.eel.fs.*
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
import com.intellij.platform.ijent.IjentApi
|
||||
import com.intellij.platform.ijent.IjentPosixApi
|
||||
@@ -164,15 +165,18 @@ private class IjentFailSafeFileSystemPosixApiImpl(
|
||||
}
|
||||
|
||||
override suspend fun openForReading(
|
||||
path: EelPath,
|
||||
args: EelFileSystemApi.OpenForReadingArgs,
|
||||
): EelResult<EelOpenedFile.Reader, EelFileSystemApi.FileReaderError> =
|
||||
holder.withDelegateRetrying {
|
||||
openForReading(path)
|
||||
openForReading(args)
|
||||
}
|
||||
|
||||
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> = holder.withDelegateRetrying {
|
||||
readFully(path, limit, overflowPolicy)
|
||||
}
|
||||
override suspend fun readFile(
|
||||
args: EelFileSystemApi.ReadFileArgs,
|
||||
): EelResult<EelFileSystemApi.ReadFileResult, EelFileSystemApi.FileReaderError> =
|
||||
holder.withDelegateRetrying {
|
||||
readFile(args)
|
||||
}
|
||||
|
||||
override suspend fun openForWriting(
|
||||
options: EelFileSystemApi.WriteOptions,
|
||||
|
||||
+2
-5
@@ -1,13 +1,10 @@
|
||||
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
|
||||
package com.intellij.platform.ijent.community.impl.nio
|
||||
|
||||
import com.intellij.openapi.diagnostic.logger
|
||||
import com.intellij.openapi.diagnostic.thisLogger
|
||||
import com.intellij.platform.eel.ReadResult
|
||||
import com.intellij.platform.eel.fs.EelFileInfo
|
||||
import com.intellij.platform.eel.fs.EelFileSystemApi
|
||||
import com.intellij.platform.eel.fs.EelOpenedFile
|
||||
import com.intellij.platform.eel.fs.EelPosixFileInfo
|
||||
import com.intellij.platform.eel.fs.*
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
import com.intellij.platform.eel.provider.utils.getOrThrowFileSystemException
|
||||
import com.intellij.platform.ijent.spi.RECOMMENDED_MAX_PACKET_SIZE
|
||||
|
||||
@@ -12,7 +12,7 @@ import com.intellij.openapi.vfs.impl.local.LocalFileSystemBase.LOG
|
||||
import com.intellij.openapi.vfs.impl.local.LocalFileSystemBase.toIoPath
|
||||
import com.intellij.openapi.vfs.limits.FileSizeLimit
|
||||
import com.intellij.platform.eel.EelApi
|
||||
import com.intellij.platform.eel.LocalEelApi
|
||||
import com.intellij.platform.eel.channels.EelDelicateApi
|
||||
import com.intellij.platform.eel.fs.*
|
||||
import com.intellij.platform.eel.getOr
|
||||
import com.intellij.platform.eel.path.EelPath
|
||||
@@ -26,6 +26,7 @@ import com.intellij.platform.ijent.community.impl.nio.IjentNioPosixFileAttribute
|
||||
import com.intellij.platform.ijent.community.impl.nio.fsBlocking
|
||||
import com.intellij.util.containers.CollectionFactory
|
||||
import com.intellij.util.containers.ContainerUtil
|
||||
import com.intellij.util.io.toByteArray
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.io.IOException
|
||||
import java.nio.file.*
|
||||
@@ -38,34 +39,39 @@ private val map = ContainerUtil.createConcurrentWeakMap<Path, EelApi>()
|
||||
* This is unacceptable in the remote setting when each request to IO results in RPC.
|
||||
* Here we try to invoke a specialized function that can read all bytes from [path] in one request.
|
||||
*/
|
||||
@OptIn(EelDelicateApi::class)
|
||||
@Suppress("RAW_RUN_BLOCKING")
|
||||
internal fun readWholeFileIfNotTooLargeWithEel(path: Path): ByteArray? {
|
||||
if (!Registry.`is`("vfs.try.eel.for.content.loading", false)) {
|
||||
return null
|
||||
}
|
||||
val root = path.root ?: return null
|
||||
|
||||
// TODO Check if this if-else can be removed. The only reason why it's kept is to avoid possible performance degradations in hot code.
|
||||
val eelDescriptor = root.getEelDescriptor()
|
||||
if (eelDescriptor == LocalEelDescriptor) {
|
||||
return null
|
||||
}
|
||||
val api = map.computeIfAbsent(root) {
|
||||
|
||||
val api = map.computeIfAbsent(root) { // TODO Does this cache make sense nowadays?
|
||||
runBlocking {
|
||||
root.getEelDescriptor().toEelApi()
|
||||
}
|
||||
}
|
||||
if (api is LocalEelApi) {
|
||||
return null
|
||||
}
|
||||
|
||||
val eelPath = path.asEelPath()
|
||||
val limit = FileSizeLimit.getContentLoadLimit(FileUtilRt.getExtension(path.fileName.toString()))
|
||||
|
||||
return runBlocking {
|
||||
when (val res = api.fs.readFully(eelPath, limit.toULong(), EelFileSystemApi.OverflowPolicy.DROP).getOrThrowFileSystemException()) {
|
||||
is EelFileSystemApi.FullReadResult.Bytes -> res.bytes
|
||||
is EelFileSystemApi.FullReadResult.BytesOverflown -> error("Never returned")
|
||||
is EelFileSystemApi.FullReadResult.Overflow -> throw FileTooBigException("File $path is bigger than $limit bytes")
|
||||
val result = runBlocking {
|
||||
try {
|
||||
api.fs.readFile(eelPath).limit(limit).failFastIfBeyondLimit(true).getOrThrowFileSystemException()
|
||||
}
|
||||
catch (err: FileSystemException) {
|
||||
throw err.cause.takeIf { it is FileTooBigException } ?: err
|
||||
}
|
||||
}
|
||||
|
||||
return result.bytes.toByteArray()
|
||||
}
|
||||
|
||||
internal fun toEelPath(parent: VirtualFile, childName: String): EelPath? =
|
||||
|
||||
@@ -36,10 +36,6 @@ internal class EelTestPosixApi(override val descriptor: EelTestDescriptor, fileS
|
||||
|
||||
private class EelTestFileSystemPosixApi(override val descriptor: EelTestDescriptor, fileSystem: EelUnitTestFileSystem) : PosixNioBasedEelFileSystemApi(fileSystem, EelTestPosixUserInfo(descriptor)) {
|
||||
|
||||
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
override suspend fun createTemporaryDirectory(options: EelFileSystemApi.CreateTemporaryEntryOptions): EelResult<EelPath, EelFileSystemApi.CreateTemporaryEntryError> {
|
||||
return wrapIntoEelResult {
|
||||
val nioTempDir = Files.createTempDirectory(fs.rootDirectories.single(), options.prefix)
|
||||
|
||||
@@ -30,10 +30,6 @@ internal class EelTestWindowsApi(override val descriptor: EelTestDescriptor, fil
|
||||
|
||||
private class EelTestFileSystemWindowsApi(override val descriptor: EelDescriptor, fileSystem: EelUnitTestFileSystem) : WindowsNioBasedEelFileSystemApi(fileSystem, EelTestWindowsUserInfo(descriptor)) {
|
||||
|
||||
override suspend fun readFully(path: EelPath, limit: ULong, overflowPolicy: EelFileSystemApi.OverflowPolicy): EelResult<EelFileSystemApi.FullReadResult, EelFileSystemApi.FullReadError> {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
override suspend fun createTemporaryDirectory(options: EelFileSystemApi.CreateTemporaryEntryOptions): EelResult<EelPath, EelFileSystemApi.CreateTemporaryEntryError> {
|
||||
return wrapIntoEelResult {
|
||||
val nioTempDir = Files.createTempDirectory(options.prefix)
|
||||
|
||||
Reference in New Issue
Block a user