[eel] eel path utils: merge most of the transferring functions to the one

GitOrigin-RevId: ab93e3eb2973d2b22d864ba98007cbe2f5e89941
This commit is contained in:
Andrii Zinchenko
2025-03-30 13:38:41 +02:00
committed by intellij-monorepo-bot
parent 38c9aa1985
commit 39aea0ea62
8 changed files with 112 additions and 181 deletions

View File

@@ -10,8 +10,17 @@ import com.intellij.openapi.components.service
import com.intellij.openapi.diagnostic.logger
import com.intellij.openapi.project.Project
import com.intellij.openapi.project.getProjectCacheFileName
import com.intellij.platform.eel.*
import com.intellij.platform.eel.provider.*
import com.intellij.platform.eel.EelApi
import com.intellij.platform.eel.EelPlatform
import com.intellij.platform.eel.EelTunnelsApi
import com.intellij.platform.eel.LocalEelApi
import com.intellij.platform.eel.pathSeparator
import com.intellij.platform.eel.provider.LocalEelDescriptor
import com.intellij.platform.eel.provider.asEelPath
import com.intellij.platform.eel.provider.asNioPath
import com.intellij.platform.eel.provider.getEelDescriptor
import com.intellij.platform.eel.provider.routingPrefixes
import com.intellij.platform.eel.provider.upgradeBlocking
import com.intellij.platform.eel.provider.utils.EelPathUtils
import com.intellij.platform.eel.provider.utils.forwardLocalServer
import kotlinx.coroutines.CoroutineScope
@@ -19,7 +28,6 @@ import kotlinx.coroutines.future.asCompletableFuture
import java.nio.charset.Charset
import java.nio.file.FileSystems
import java.nio.file.Path
import kotlin.io.path.isDirectory
import kotlin.io.path.name
class EelBuildCommandLineBuilder(val project: Project, exePath: Path) : BuildCommandLineBuilder {
@@ -71,7 +79,7 @@ class EelBuildCommandLineBuilder(val project: Project, exePath: Path) : BuildCom
return path
}
val remotePath = workingDirectory.resolve(path.name)
return transferPathToRemoteIfRequired(path, remotePath)
return EelPathUtils.transferContentsIfNonLocal(eel, path, remotePath)
}
override fun copyProjectSpecificPathToTargetIfRequired(project: Project, path: Path): Path {
@@ -80,7 +88,7 @@ class EelBuildCommandLineBuilder(val project: Project, exePath: Path) : BuildCom
}
val cacheFileName = project.getProjectCacheFileName()
val target = cacheDirectory.resolve(cacheFileName).resolve(path.name)
return transferPathToRemoteIfRequired(path, target)
return EelPathUtils.transferContentsIfNonLocal(eel, path, target)
}
override fun getYjpAgentPath(yourKitProfilerService: YourKitProfilerService?): String? {
@@ -105,16 +113,6 @@ class EelBuildCommandLineBuilder(val project: Project, exePath: Path) : BuildCom
return eel.descriptor.routingPrefixes().map { it.toString().removeSuffix(FileSystems.getDefault().separator) }.toSet()
}
private fun transferPathToRemoteIfRequired(source: Path, target: Path): Path {
if (source.isDirectory()) {
EelPathUtils.transferContentsIfNonLocal(eel, source, target)
}
else {
EelPathUtils.transferFileIfNonLocal(source, target)
}
return target
}
/**
* Ensures that connections from the environment of the build process can reach `localhost:[localPort]`
*/

View File

@@ -3,32 +3,24 @@ package com.intellij.platform.eel.provider.utils
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.diagnostic.thisLogger
import com.intellij.openapi.progress.runBlockingMaybeCancellable
import com.intellij.openapi.project.Project
import com.intellij.platform.eel.EelApi
import com.intellij.platform.eel.EelDescriptor
import com.intellij.platform.eel.EelResult
import com.intellij.platform.eel.LocalEelApi
import com.intellij.platform.eel.fs.createTemporaryDirectory
import com.intellij.platform.eel.fs.createTemporaryFile
import com.intellij.platform.eel.getOrThrow
import com.intellij.platform.eel.path.EelPath
import com.intellij.platform.eel.provider.LocalEelDescriptor
import com.intellij.platform.eel.provider.asEelPath
import com.intellij.platform.eel.provider.asNioPath
import com.intellij.platform.eel.provider.getEelDescriptor
import com.intellij.platform.eel.provider.upgradeBlocking
import com.intellij.platform.eel.provider.utils.EelPathUtils.calculateFileHashUsingMetadata
import com.intellij.util.awaitCancellationAndInvoke
import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.async
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.jetbrains.annotations.ApiStatus
import java.io.IOException
import java.net.URI
@@ -140,132 +132,86 @@ object EelPathUtils {
}
/**
* Transfers contents of a directory or file at [source] to [sink].
* This function works efficiently when [source] and [sink] located on different environments.
* Transfers local content from the specified source path to a remote (non-local) Eel environment using the given EelApi.
*
* @param sink the required location for the file. If it is `null`, then a temporary directory will be created.
* @return the path which contains the transferred data. Sometimes this value can coincide with [source] if the [sink]
* This method is designed to copy content from a local source (one whose descriptor equals LocalEelDescriptor)
* to a remote environment. Its behavior is as follows:
*
* 1. If the provided EelApi instance represents a local environment (i.e. it is an instance of LocalEelApi),
* no transfer is performed and the original source path is returned.
*
* 2. If the source path is not local (i.e. its descriptor is not LocalEelDescriptor):
* - If a sink is provided and its descriptor differs from the source's descriptor, an UnsupportedOperationException is thrown,
* as transferring between different Eel environments is not supported.
* - Otherwise, no transfer is performed and the original source path is returned because the content is already non-local.
*
* 3. If no sink path is provided (sink is null), the method uses a cached transfer mechanism via the
* [TransferredContentHolder] service to copy the content to a temporary location in the remote environment.
* The path to the transferred content is then returned.
*
* 4. If a sink path is provided:
* - For directories: If the sink does not exist, the directory content is recursively copied from the source
* to the sink using an internal transfer procedure (walkingTransfer). The sink path is then returned.
* - For files: The method calculates SHA-256 hashes (using partial content reading) for both the source and
* the existing sink file (if present). If the hashes differ, indicating that the file content has changed,
* any existing file at the sink is deleted and the source file is copied to the sink using walkingTransfer.
* In either case, the sink path is returned.
*
* The fileAttributesStrategy parameter defines how file attributes are handled during the transfer
* (for example, whether they are copied or skipped). The default strategy is FileTransferAttributesStrategy.Copy.
*
* @param eel the target EelApi instance representing the remote environment to which the content should be transferred.
* @param source the local source path whose content is to be transferred.
* @param sink (optional) the target path in the remote environment where the content should be placed.
* If null, a temporary location is created in the remote environment.
* @param fileAttributesStrategy the strategy for handling file attributes during transfer; defaults to FileTransferAttributesStrategy.Copy.
* @return a Path pointing to the location of the transferred content in the remote Eel environment.
* @throws UnsupportedOperationException if the source path is not local or if an attempt is made to transfer
* between different Eel environments.
*/
@JvmStatic
fun transferContentsIfNonLocal(eel: EelApi, source: Path, sink: Path? = null): Path {
if (eel is LocalEelApi) return source
@JvmOverloads
fun transferContentsIfNonLocal(
eel: EelApi,
source: Path,
sink: Path? = null,
fileAttributesStrategy: FileTransferAttributesStrategy = FileTransferAttributesStrategy.Copy,
): Path {
if (eel is LocalEelApi) {
return source
}
if (source.getEelDescriptor() !is LocalEelDescriptor) {
if (sink != null && source.getEelDescriptor() != sink.getEelDescriptor()) {
throw UnsupportedOperationException("Transferring between different Eels is not supported yet")
}
return source
}
// todo: intergrate checksums here so that files could be refreshed in case of changes
if (sink != null) {
if (sink == null) {
return runBlockingMaybeCancellable {
service<TransferredContentHolder>().transferIfNeeded(eel, source, fileAttributesStrategy)
}
}
if (source.isDirectory()) { // todo: use checksums for directories?
if (!Files.exists(sink)) {
walkingTransfer(source, sink, false, true)
walkingTransfer(source, sink, false, fileAttributesStrategy)
}
return sink
}
else {
val temp = runBlockingMaybeCancellable { eel.createTempFor(source, false) }
val remoteHash = if (Files.exists(sink)) calculateFileHashUsingPartialContent(sink) else ""
val sourceHash = calculateFileHashUsingPartialContent(source)
walkingTransfer(source, temp, false, true)
return temp
}
}
/**
* Transfers a local file to a remote temporary environment if required.
*
* This function is useful for transferring files that are located on the local machine
* to a remote environment. It can be particularly helpful for files stored in plugin
* resources, such as:
*
* ```kotlin
* Path.of(PathManager.getPluginsPath()).resolve(pluginId)
* ```
*
* ### Behavior:
* - If the file is **not local**, an exception will be thrown.
* - If the `eel` is a local environment (`LocalEelApi`), the function directly returns the source as an [EelPath].
* - If the file needs to be transferred to a remote environment:
* - A temporary directory is created on the remote environment.
* - The file is transferred into the temporary directory.
* - The temporary directory will be automatically deleted upon exit.
*
* ### Hash Calculation:
* - **Purpose:** A SHA-256 hash is computed for the source file to determine whether its content has changed,
* thereby avoiding unnecessary transfers.
* - **Mechanism:** The hash is computed using [calculateFileHashUsingMetadata], which calculates a SHA-256 digest
* based solely on the file's metadata (including file size, last modified time, creation time, and file key).
* This fast, metadata-based approach minimizes I/O overhead while detecting meaningful changes.
* - A hash cache is maintained to map source files to their computed hashes, reducing redundant transfers.
*
* ### Parameters:
* @param eel the [EelApi] instance representing the target environment (local or remote).
* @param source the [Path] of the file to be transferred.
*
* ### Returns:
* An [EelPath] representing the source file's location in the target environment.
*
* ### Exceptions:
* - Throws [IllegalStateException] if the source file is not local.
*
* ### Example:
* ```kotlin
* val eel: EelApi = ...
* val sourcePath = Path.of("/path/to/local/file.txt")
*
* val eelPath = transferLocalContentToRemoteTempIfNeeded(eel, sourcePath)
* println("File transferred to: $eelPath")
* ```
*
* ### Internal Details:
* The function uses the [TransferredContentHolder] to manage caching and file transfers.
* The file hash is computed using [calculateFileHashUsingMetadata] to detect changes based on metadata,
* thereby minimizing unnecessary transfers.
*
* ### See Also:
* - [TransferredContentHolder]: For detailed caching and transfer mechanisms.
* - [MessageDigest]: For hash calculation.
*/
@JvmStatic
@JvmOverloads
fun transferLocalContentToRemoteTempIfNeeded(
eel: EelApi,
source: Path,
fileAttributesStrategy: FileTransferAttributesStrategy = FileTransferAttributesStrategy.Copy,
): EelPath {
val sourceDescriptor = source.getEelDescriptor()
check(sourceDescriptor is LocalEelDescriptor)
if (eel is LocalEelApi) {
return source.asEelPath()
}
return runBlockingMaybeCancellable {
service<TransferredContentHolder>().transferIfNeeded(eel, source, fileAttributesStrategy).asEelPath()
}
}
@JvmStatic
fun transferFileIfNonLocal(source: Path, remotePath: Path) {
val sourceDescriptor = source.getEelDescriptor()
check(sourceDescriptor is LocalEelDescriptor)
val sourceHash = calculateFileHashUsingPartialContent(source)
val remoteHash = if (Files.exists(remotePath)) {
calculateFileHashUsingPartialContent(remotePath)
}
else {
""
}
if (sourceHash != remoteHash) {
if (remoteHash.isNotEmpty()) {
Files.delete(remotePath)
if (sourceHash != remoteHash) {
if (remoteHash.isNotEmpty()) {
Files.delete(sink)
}
walkingTransfer(source, sink, false, fileAttributesStrategy)
}
transferContentsIfNonLocal(remotePath.getEelDescriptor().upgradeBlocking(), source, remotePath)
return sink
}
}
@@ -432,6 +378,7 @@ object EelPathUtils {
return someEelPath.asNioPath()
}
// TODO: internal?
@RequiresBackgroundThread
fun walkingTransfer(sourceRoot: Path, targetRoot: Path, removeSource: Boolean, copyAttributes: Boolean) {
val fileAttributesStrategy = if (copyAttributes) FileTransferAttributesStrategy.Copy else FileTransferAttributesStrategy.Skip
@@ -466,7 +413,7 @@ object EelPathUtils {
}
@RequiresBackgroundThread
fun walkingTransfer(sourceRoot: Path, targetRoot: Path, removeSource: Boolean, fileAttributesStrategy: FileTransferAttributesStrategy) {
private fun walkingTransfer(sourceRoot: Path, targetRoot: Path, removeSource: Boolean, fileAttributesStrategy: FileTransferAttributesStrategy) {
val shouldObtainExtendedAttributes = when (fileAttributesStrategy) {
FileTransferAttributesStrategy.Skip -> false
is FileTransferAttributesStrategy.SourceAware -> true
@@ -689,36 +636,6 @@ object EelPathUtils {
from.creationTime(),
)
}
suspend fun maybeUploadPath(scope: CoroutineScope, path: Path, target: EelDescriptor): EelPath {
val originalPath = path.asEelPath()
if (originalPath.descriptor == target) {
return originalPath
}
val eelApi = target.upgrade()
val tmpDir = eelApi.fs.createTemporaryDirectory()
.prefix(path.fileName.toString())
.suffix("eel")
.deleteOnExit(true)
.getOrThrow()
val referencedPath = tmpDir.resolve(path.name)
withContext(Dispatchers.IO) {
walkingTransfer(path, referencedPath.asNioPath(), removeSource = false, copyAttributes = true)
}
scope.awaitCancellationAndInvoke {
when (val result = eelApi.fs.delete(tmpDir, true)) {
is EelResult.Ok -> Unit
is EelResult.Error -> thisLogger().warn("Failed to delete temporary directory $tmpDir: ${result.error}")
}
}
return referencedPath
}
}
private inline fun <T> Result<T>.handleIOExceptionOrThrow(action: (exception: IOException) -> Unit): Result<T> =

View File

@@ -268,7 +268,7 @@ class MavenShCommandLineState(val environment: ExecutionEnvironment, private val
}
if (distribution is DaemonedMavenDistribution) {
return distribution.daemonHome.resolve("bin").resolve(if (isWindows()) "mvnd.cmd" else "mvnd.sh").asEelPath().toString()
return distribution.daemonHome.resolve("bin").resolve(if (isWindows()) "mvnd.cmd" else "mvnd.sh").asEelPath().toString()
}
else {
return mavenHome.resolve("bin").resolve(if (isWindows()) "mvn.cmd" else "mvn").asEelPath().toString()

View File

@@ -2,14 +2,17 @@
package org.jetbrains.idea.maven.server.eel
import com.intellij.execution.Executor
import com.intellij.execution.configurations.*
import com.intellij.execution.configurations.CompositeParameterTargetedValue
import com.intellij.execution.configurations.ParameterTargetValuePart
import com.intellij.execution.configurations.ParametersList
import com.intellij.execution.configurations.RunProfileState
import com.intellij.execution.configurations.SimpleJavaParameters
import com.intellij.execution.process.KillableColoredProcessHandler
import com.intellij.execution.process.ProcessHandler
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.diagnostic.logger
import com.intellij.openapi.externalSystem.util.wsl.connectRetrying
import com.intellij.openapi.progress.runBlockingCancellable
import com.intellij.openapi.progress.runBlockingMaybeCancellable
import com.intellij.openapi.project.Project
import com.intellij.openapi.projectRoots.Sdk
@@ -21,7 +24,7 @@ import com.intellij.platform.eel.fs.pathSeparator
import com.intellij.platform.eel.getOrThrow
import com.intellij.platform.eel.path.EelPath
import com.intellij.platform.eel.provider.asEelPath
import com.intellij.platform.eel.provider.utils.EelPathUtils
import com.intellij.platform.eel.provider.utils.EelPathUtils.transferContentsIfNonLocal
import com.intellij.platform.eel.provider.utils.fetchLoginShellEnvVariablesBlocking
import com.intellij.platform.eel.provider.utils.forwardLocalPort
import com.intellij.platform.util.coroutines.childScope
@@ -160,7 +163,7 @@ private class EelMavenCmdState(
eelParams.charset = parameters.charset
eelParams.vmParametersList.add("-classpath")
eelParams.vmParametersList.add(parameters.classPath.pathList.mapNotNull {
runBlockingCancellable { EelPathUtils.maybeUploadPath(scope, Path(it), eel.descriptor).toString() }
transferContentsIfNonLocal(eel, Path(it)).asEelPath().toString()
}.joinToString(eel.fs.pathSeparator))
return eelParams
@@ -172,9 +175,7 @@ private class EelMavenCmdState(
for (part in item.parts) {
when (part) {
is ParameterTargetValuePart.Const -> append(part.localValue)
is ParameterTargetValuePart.Path -> runBlockingCancellable {
append(EelPathUtils.maybeUploadPath(scope, Path.of(part.localValue), eel.descriptor).toString())
}
is ParameterTargetValuePart.Path -> append(transferContentsIfNonLocal(eel, Path.of(part.localValue)).asEelPath().toString())
ParameterTargetValuePart.PathSeparator -> append(eel.fs.pathSeparator)
is ParameterTargetValuePart.PromiseValue -> append(part.localValue) // todo?
}

View File

@@ -32,7 +32,7 @@ import java.util.*;
import static com.intellij.platform.eel.provider.EelNioBridgeServiceKt.asEelPath;
import static com.intellij.platform.eel.provider.EelProviderUtil.getEelDescriptor;
import static com.intellij.platform.eel.provider.EelProviderUtil.upgradeBlocking;
import static com.intellij.platform.eel.provider.utils.EelPathUtils.transferLocalContentToRemoteTempIfNeeded;
import static com.intellij.platform.eel.provider.utils.EelPathUtils.transferContentsIfNonLocal;
import static com.intellij.sh.ShBundle.message;
import static com.intellij.sh.ShNotification.NOTIFICATION_GROUP_ID;
@@ -111,7 +111,7 @@ public final class ShExternalFormatter extends AsyncDocumentFormattingService {
FileTransferAttributesStrategy.copyWithRequiredPosixPermissions(PosixFilePermission.OWNER_EXECUTE);
GeneralCommandLine commandLine = new GeneralCommandLine()
.withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE)
.withExePath(transferLocalContentToRemoteTempIfNeeded(eel, Path.of(shFmtExecutable), forceExecutePermission).toString())
.withExePath(asEelPath(transferContentsIfNonLocal(eel, Path.of(shFmtExecutable), null, forceExecutePermission)).toString())
.withWorkingDirectory(path.getParent())
.withParameters(params);

View File

@@ -52,9 +52,10 @@ import java.util.Collections;
import java.util.List;
import java.util.Objects;
import static com.intellij.platform.eel.provider.EelNioBridgeServiceKt.asEelPath;
import static com.intellij.platform.eel.provider.EelProviderUtil.getEelDescriptor;
import static com.intellij.platform.eel.provider.EelProviderUtil.upgradeBlocking;
import static com.intellij.platform.eel.provider.utils.EelPathUtils.transferLocalContentToRemoteTempIfNeeded;
import static com.intellij.platform.eel.provider.utils.EelPathUtils.transferContentsIfNonLocal;
import static java.util.Arrays.asList;
public class ShShellcheckExternalAnnotator
@@ -98,7 +99,7 @@ public class ShShellcheckExternalAnnotator
FileTransferAttributesStrategy.copyWithRequiredPosixPermissions(PosixFilePermission.OWNER_EXECUTE);
GeneralCommandLine commandLine = new GeneralCommandLine()
.withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE)
.withExePath(transferLocalContentToRemoteTempIfNeeded(eel, Path.of(shellcheckExecutable), forceExecutePermission).toString())
.withExePath(asEelPath(transferContentsIfNonLocal(eel, Path.of(shellcheckExecutable), null, forceExecutePermission)).toString())
.withParameters(fileInfo.executionParams);
if (!ApplicationManager.getApplication().isUnitTestMode()) commandLine.withWorkDirectory(fileInfo.workDirectory);
long timestamp = fileInfo.modificationStamp;

View File

@@ -12,7 +12,6 @@ import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.platform.eel.EelDescriptor;
import com.intellij.platform.eel.provider.EelProviderUtil;
import com.intellij.platform.eel.provider.utils.EelPathUtils;
import com.intellij.terminal.ui.TerminalWidget;
import com.intellij.util.PathUtil;
import com.intellij.util.containers.ContainerUtil;
@@ -35,6 +34,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import static com.intellij.platform.eel.provider.EelNioBridgeServiceKt.asEelPath;
import static com.intellij.platform.eel.provider.utils.EelPathUtils.transferContentsIfNonLocal;
import static org.jetbrains.plugins.terminal.LocalTerminalDirectRunner.LOGIN_CLI_OPTIONS;
import static org.jetbrains.plugins.terminal.LocalTerminalDirectRunner.isBlockTerminalSupported;
@@ -204,7 +205,7 @@ public final class LocalShellIntegrationInjector {
result.add(rcfileOption);
if (eelDescriptor != null) {
final var eelApi = EelProviderUtil.upgradeBlocking(eelDescriptor);
result.add(EelPathUtils.transferLocalContentToRemoteTempIfNeeded(eelApi, Path.of(rcFilePath)).toString());
result.add(asEelPath(transferContentsIfNonLocal(eelApi, Path.of(rcFilePath))).toString());
}
else {
result.add(rcFilePath);

View File

@@ -3,11 +3,24 @@ package com.intellij.python.community.execService.impl
import com.intellij.execution.process.ProcessOutput
import com.intellij.openapi.diagnostic.fileLogger
import com.intellij.platform.eel.*
import com.intellij.platform.eel.EelApi
import com.intellij.platform.eel.EelExecApi
import com.intellij.platform.eel.EelProcess
import com.intellij.platform.eel.execute
import com.intellij.platform.eel.getOr
import com.intellij.platform.eel.path.EelPath
import com.intellij.platform.eel.provider.asEelPath
import com.intellij.platform.eel.provider.getEelDescriptor
import com.intellij.platform.eel.provider.utils.*
import com.intellij.python.community.execService.*
import com.intellij.platform.eel.provider.utils.EelPathUtils
import com.intellij.platform.eel.provider.utils.EelProcessExecutionResult
import com.intellij.platform.eel.provider.utils.awaitProcessResult
import com.intellij.platform.eel.provider.utils.stderrString
import com.intellij.platform.eel.provider.utils.stdoutString
import com.intellij.python.community.execService.EelProcessInteractiveHandler
import com.intellij.python.community.execService.ExecOptions
import com.intellij.python.community.execService.ExecService
import com.intellij.python.community.execService.ProcessOutputTransformer
import com.intellij.python.community.execService.WhatToExec
import com.jetbrains.python.PythonHelpersLocator
import com.jetbrains.python.Result
import com.jetbrains.python.errorProcessing.PyError.ExecException
@@ -92,7 +105,7 @@ private suspend fun WhatToExec.buildExecutableProcess(args: List<String>, option
val eel = python.getEelDescriptor().upgrade()
val localHelper = PythonHelpersLocator.findPathInHelpers(helper)
?: error("No ${helper} found: installation broken?")
val remoteHelper = EelPathUtils.transferLocalContentToRemoteTempIfNeeded(eel, localHelper).toString()
val remoteHelper = EelPathUtils.transferContentsIfNonLocal(eel, localHelper).asEelPath().toString()
Triple(eel, python.pathString, listOf(remoteHelper) + args)
}
is WhatToExec.Command -> Triple(eel, command, args)