From 4e5531e4c5187ce2e1fe55e8c101bc24c65ad31d Mon Sep 17 00:00:00 2001 From: Alexey Katsman Date: Fri, 17 Jul 2026 11:49:26 +0200 Subject: [PATCH] PY-52688 support selective uploads for target execution Let target-backed process launches create their working directory and upload only requested project metadata. Reuse persistent deployment mappings without scheduling their remote roots for cleanup, while keeping temporary upload roots disposable. (cherry picked from commit 746c082e3ad2d8ac4f30d1f95d433b8305f2b73d) IJ-MR-215198 GitOrigin-RevId: 29e84212fee02f037800535e264abe1d17b720cc --- .../messages/PyExecBundle.properties | 4 + .../python/community/execService/api.kt | 31 +++++- .../execService/impl/ExecServiceImpl.kt | 27 ++++-- .../impl/TargetEnvironmentRequestHandler.kt | 18 ++-- .../impl/processLaunchers/ProcessLauncher.kt | 2 + .../impl/processLaunchers/targets.kt | 96 +++++++++++++++++-- .../community/execService/implInterfaces.kt | 2 + 7 files changed, 154 insertions(+), 26 deletions(-) diff --git a/python/python-exec-service/resources/messages/PyExecBundle.properties b/python/python-exec-service/resources/messages/PyExecBundle.properties index d860a4351573..8d3302daca90 100644 --- a/python/python-exec-service/resources/messages/PyExecBundle.properties +++ b/python/python-exec-service/resources/messages/PyExecBundle.properties @@ -8,5 +8,9 @@ py.exec.error.not.zero=Exit code is not zero py.exec.error.unexpected.output=Unexpected output {0} py.exec.target.name.default="Local" py.exec.error.not.trusted=Could not start {0}: project is not trusted +py.exec.target.working.directory.create.error=Could not create target working directory {0}: {1} +py.exec.target.working.directory.create.interrupted=Interrupted while creating target working directory {0} +py.exec.target.working.directory.create.exitCode=Could not create target working directory {0}: mkdir exited with {1} +py.exec.target.working.directory.create.unsupported=Could not create target working directory {0}: Windows targets are not supported py.exec.target.binaries.are.not.supported=Target binaries are not supported diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/api.kt b/python/python-exec-service/src/com/intellij/python/community/execService/api.kt index 41201109cfe7..0a750336f4df 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/api.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/api.kt @@ -61,8 +61,16 @@ data class BinOnEel(val path: Path, internal val workDir: Path? = null) : Binary * Legacy Targets-based approach. Do not use it, unless you know what you are doing * if [target] "local" target is used */ -data class BinOnTarget(internal val configureTargetCmdLine: (TargetedCommandLineBuilder) -> Unit, val target: TargetEnvironmentConfiguration, val workingDir: Path? = null) : BinaryToExec { - constructor(exePath: FullPathOnTarget, target: TargetEnvironmentConfiguration, workingDir: Path? = null) : this({ it.setExePath(exePath) }, target, workingDir) +data class BinOnTarget( + internal val configureTargetCmdLine: (TargetedCommandLineBuilder) -> Unit, + val target: TargetEnvironmentConfiguration, + val workingDir: Path? = null, +) : BinaryToExec { + constructor( + exePath: FullPathOnTarget, + target: TargetEnvironmentConfiguration, + workingDir: Path? = null, + ) : this({ it.setExePath(exePath) }, target, workingDir) @RequiresBackgroundThread fun getLocalExePath(): Lazy = lazy { @@ -249,12 +257,26 @@ data class DownloadConfig( val relativePaths: List = emptyList(), ) +/** + * Configuration for uploading files before command execution. + * Uses existing upload volume mappings (from web deployment). + * + * @param relativePaths Relative paths to upload from the working directory. + * Empty list means upload no files. + * @param ensureWorkingDirectoryExists Creates the target working directory before execution, even when no files are uploaded. + */ +data class UploadConfig( + val relativePaths: List = emptyList(), + val ensureWorkingDirectoryExists: Boolean = true, +) + /** * @property[env] Environment variables to be applied with the process run * @property[timeout] Process gets killed after this timeout * @property[processDescription] optional description to be displayed to user * @property[tty] Much like [com.intellij.platform.eel.EelExecApi.Pty] * @property[weight] use it to limit the number of concurrent processes not to exhaust user resources, see [ConcurrentProcessWeight] + * @property[uploadBeforeExecution] configuration for uploading files before command execution (Target-based execution only) * @property[downloadAfterExecution] configuration for downloading files after command execution (Target-based execution only) */ data class ExecOptions( @@ -263,7 +285,8 @@ data class ExecOptions( val timeout: Duration = 5.minutes, override val tty: TtySize? = null, val weight: ConcurrentProcessWeight = ConcurrentProcessWeight.LIGHT, - val downloadAfterExecution: DownloadConfig? = null, + override val uploadBeforeExecution: UploadConfig? = null, + override val downloadAfterExecution: DownloadConfig? = null, ) : ExecOptionsBase @@ -275,6 +298,8 @@ data class ExecGetProcessOptions( override val env: Map = emptyMap(), override val processDescription: @Nls String? = null, override val tty: TtySize? = null, + override val uploadBeforeExecution: UploadConfig? = null, + override val downloadAfterExecution: DownloadConfig? = null, ) : ExecOptionsBase data class TtySize(val rows: UShort, val cols: UShort) diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt index 75dab2228cb2..c0132d9ca5fa 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt @@ -45,7 +45,12 @@ internal class ExecServiceImpl private constructor() : ExecService { } } - override suspend fun executeGetProcess(binary: BinaryToExec, args: Args, scopeToBind: CoroutineScope?, options: ExecGetProcessOptions): Result> { + override suspend fun executeGetProcess( + binary: BinaryToExec, + args: Args, + scopeToBind: CoroutineScope?, + options: ExecGetProcessOptions, + ): Result> { val launcher = create(binary, args, options, scopeToBind).getOr { return it } val process = launcher.start().getOr { val createExecError = launcher.createExecError(options.processDescription ?: "", it.error).error @@ -54,10 +59,14 @@ internal class ExecServiceImpl private constructor() : ExecService { return Result.success(process) } - private suspend fun create(binary: BinaryToExec, args: Args, options: ExecOptionsBase, scopeToBind: CoroutineScope? = null): Result { + private suspend fun create( + binary: BinaryToExec, + args: Args, + options: ExecOptionsBase, + scopeToBind: CoroutineScope? = null, + ): Result { val scope = scopeToBind ?: ApplicationManager.getApplication().service().scope - val downloadConfig = (options as? ExecOptions)?.downloadAfterExecution - val request = LaunchRequest(scope, args, options.env, options.tty, downloadConfig) + val request = LaunchRequest(scope, args, options.env, options.tty, options.uploadBeforeExecution, options.downloadAfterExecution) return Result.success( when (binary) { is BinOnEel -> createProcessLauncherOnEel(binary, request) @@ -70,7 +79,12 @@ internal class ExecServiceImpl private constructor() : ExecService { }) } - override suspend fun executeAdvanced(binary: BinaryToExec, args: Args, options: ExecOptions, processInteractiveHandler: ProcessInteractiveHandler): PyResult { + override suspend fun executeAdvanced( + binary: BinaryToExec, + args: Args, + options: ExecOptions, + processInteractiveHandler: ProcessInteractiveHandler, + ): PyResult { return coroutineScope { val processLauncher = create(binary, args, options, this).getOr { return@coroutineScope it.asPyError() @@ -89,7 +103,8 @@ internal class ExecServiceImpl private constructor() : ExecService { processInteractiveHandler: ProcessInteractiveHandler, ): Result { val description = options.processDescription - ?: PyExecBundle.message("py.exec.defaultName.process", (listOf(processLauncher.exeForError.toString()) + processLauncher.args).joinToString(" ")) + ?: PyExecBundle.message("py.exec.defaultName.process", + (listOf(processLauncher.exeForError.toString()) + processLauncher.args).joinToString(" ")) val process = processLauncher.start(options.weight).getOr { val message = PyExecBundle.message("py.exec.start.error", description, it.error.cantExecProcessError, it.error.errNo ?: "unknown") diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt index b778d76115ce..1824a8514c5b 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt @@ -1,11 +1,11 @@ // Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.execService.impl +import com.intellij.execution.target.FullPathOnTarget import com.intellij.execution.target.TargetEnvironment import com.intellij.execution.target.TargetEnvironmentRequest import com.intellij.openapi.extensions.ExtensionPointName import com.intellij.openapi.progress.coroutineToIndicator -import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler.Companion.mapUploadRoots import com.jetbrains.python.run.target.HelpersAwareTargetEnvironmentRequest import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -22,11 +22,10 @@ abstract class TargetEnvironmentRequestHandler(pri private suspend fun mapUploadRootsIfValid( request: TargetEnvironmentRequest, localDirs: Set, - workingDirToDownload: Path?, ): UploadInfo? = if (reqClass.isInstance(request)) { @Suppress("UNCHECKED_CAST") // Checked one line above - mapUploadRootsImpl(request as T, localDirs, workingDirToDownload) + mapUploadRootsImpl(request as T, localDirs) } else { null @@ -38,7 +37,6 @@ abstract class TargetEnvironmentRequestHandler(pri protected abstract suspend fun mapUploadRootsImpl( request: T, localDirs: Set, - workingDirToDownload: Path?, ): UploadInfo @@ -67,7 +65,7 @@ abstract class TargetEnvironmentRequestHandler(pri localDirs: Set, workingDirToDownload: Path?, ): Map = withContext(Dispatchers.IO) { - val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs, workingDirToDownload) } + val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs) } ?: error("No implementation of [${TargetEnvironmentRequestHandler::class.java}] is found for $request, broken bundle? " + "If you are in tests, set `@TestApplicationWithEel(useLegacyTargets=true)`") val localToRemoteHelpersRoots = coroutineToIndicator { @@ -86,10 +84,13 @@ abstract class TargetEnvironmentRequestHandler(pri ) } else { - // Just a random temp path, but we try to preserve location of workDir (most probably projDir) is set. + // Either we have a persistent path mapping, or it's just a random temp path, but we try to preserve location of + // workDir (most probably projDir) is set. + val (targetPath, removeAtShutdown) = uploadInfo.persistentPaths[localPath]?.let { TargetEnvironment.TargetPath.Persistent(it) to false } + ?: (TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString) to true) LocalPathToTargetResult( - targetPath = TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString), - removeAtShutdown = true, + targetPath = targetPath, + removeAtShutdown = removeAtShutdown, uploadVolumeExplicitly = true ) } @@ -113,6 +114,7 @@ abstract class TargetEnvironmentRequestHandler(pri * Each inheritor provides it so we can access the helpers. */ val helpersAware: HelpersAwareTargetEnvironmentRequest, + val persistentPaths: Map = emptyMap(), ) diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt index 83b6b0ce5e30..75621b97553a 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt @@ -7,6 +7,7 @@ import com.intellij.python.community.execService.Args import com.intellij.python.community.execService.ConcurrentProcessWeight import com.intellij.python.community.execService.DownloadConfig import com.intellij.python.community.execService.TtySize +import com.intellij.python.community.execService.UploadConfig import com.intellij.python.community.execService.impl.LoggingProcess import com.jetbrains.python.Result import com.jetbrains.python.TraceContext @@ -61,5 +62,6 @@ internal data class LaunchRequest( val args: Args, val env: Map, val usePty: TtySize?, + val uploadConfig: UploadConfig? = null, val downloadConfig: DownloadConfig? = null, ) diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt index 9a1821e6860c..8f95488179c6 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt @@ -4,9 +4,11 @@ package com.intellij.python.community.execService.impl.processLaunchers import com.intellij.execution.ExecutionException +import com.intellij.execution.Platform import com.intellij.execution.process.LocalPtyOptions import com.intellij.execution.target.FullPathOnTarget import com.intellij.execution.target.TargetEnvironment +import com.intellij.execution.target.TargetEnvironmentRequest import com.intellij.execution.target.TargetProgressIndicator import com.intellij.execution.target.TargetedCommandLine import com.intellij.execution.target.TargetedCommandLineBuilder @@ -22,6 +24,7 @@ import com.intellij.platform.eel.impl.base.bindProcessToScopeImpl import com.intellij.python.community.execService.BinOnTarget import com.intellij.python.community.execService.DownloadConfig import com.intellij.python.community.execService.ExecuteGetProcessError +import com.intellij.python.community.execService.UploadConfig import com.intellij.python.community.execService.impl.PyExecBundle import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler import com.intellij.remoteServer.util.ServerRuntimeException @@ -59,21 +62,21 @@ internal suspend fun createProcessLauncherOnTarget( } } - // Broken Targets API can only upload the whole directory + // Targets API maps local roots as directories; callers may still restrict which files are uploaded below. + val workingDir = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() } val dirsToMap = buildSet { addAll(launchRequest.args.localFiles.map { it.parent }) - binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.also { + workingDir?.also { add(it) } } - val uploadRoots = - TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }) + val uploadRoots = TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, workingDir) request.uploadVolumes.addAll(uploadRoots.map { it.value.root }) // Setup download roots if download is requested val downloadConfig = launchRequest.downloadConfig if (downloadConfig != null) { - val localDirsToDownload = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { setOf(it) } ?: emptySet() + val localDirsToDownload = workingDir?.let { setOf(it) } ?: emptySet() val downloadRoots = mapDownloadRoots(request.uploadVolumes, localDirsToDownload) request.downloadVolumes.addAll(downloadRoots) } @@ -89,10 +92,18 @@ internal suspend fun createProcessLauncherOnTarget( fileLogger().warn("Failed to start $target", e) // TODO: i18n return@withContext Result.failure(ExecuteGetProcessError.EnvironmentError(MessageError("Failed to start environment due to ${e.localizedMessage}"))) } + val workingDirOnTarget = workingDir?.let { targetEnv.getTargetPaths(it.pathString).firstOrNull() ?: it.pathString } + val uploadConfig = launchRequest.uploadConfig + if (uploadConfig?.ensureWorkingDirectoryExists == true && workingDirOnTarget != null) { + ensureTargetDirectoryExists(targetEnv, request, workingDirOnTarget).getOr { failure -> + targetEnv.shutdown() + return@withContext failure + } + } for (volume in targetEnv.uploadVolumes.values) { val skipUploading = uploadRoots[volume.localRoot]?.uploadVolumeExplicitly == false - if (! skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers) - volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService") + if (!skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers) + uploadVolume(volume, workingDir, uploadConfig, launchRequest.args.localFiles) } } @@ -105,9 +116,8 @@ internal suspend fun createProcessLauncherOnTarget( // exe path is always fixed (pre-presolved) promise. It can't be obtained directly because of Targets API limitation exePath = commandLineBuilder.exePath.localValue.blockingGet(1000) ?: error("Exe path not set: $binOnTarget is broken") // Map working directory through upload volumes if it's a local path - binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { workingDir -> + if (workingDirOnTarget != null) { // Try to resolve through upload volumes (in case workingDir is a local path that needs mapping) - val workingDirOnTarget = targetEnv.getTargetPaths(workingDir.pathString).firstOrNull() ?: workingDir.pathString commandLineBuilder.setWorkingDirectory(workingDirOnTarget) } launchRequest.usePty?.let { @@ -134,6 +144,74 @@ internal suspend fun createProcessLauncherOnTarget( downloadConfig))) } +private fun uploadVolume( + volume: TargetEnvironment.UploadableVolume, + workingDir: Path?, + uploadConfig: UploadConfig?, + localFiles: List, +) { + if (uploadConfig != null && workingDir != null && volume.localRoot == workingDir) { + val localFileRelativePaths = localFiles + .filter { it.parent == workingDir } + .map { workingDir.relativize(it).pathString } + val pathsToUpload = (uploadConfig.relativePaths + localFileRelativePaths).distinct() + for (path in pathsToUpload) { + volume.uploadMeasureTime(path, TargetProgressIndicator.EMPTY, "execService") + } + } + else { + volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService") + } +} + +private fun ensureTargetDirectoryExists( + targetEnv: TargetEnvironment, + request: TargetEnvironmentRequest, + directory: FullPathOnTarget, +): Result { + when (request.targetPlatform.platform) { + Platform.UNIX -> Unit + Platform.WINDOWS -> { + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.unsupported", directory)) + return Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } + } + + val cmdLine = createMkdirCommandLine(request, directory) + val process = try { + targetEnv.createProcess(cmdLine) + } + catch (e: ExecutionException) { + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.error", directory, e.localizedMessage)) + return Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } + + val exitCode = try { + process.waitFor() + } + catch (_: InterruptedException) { + process.destroyForcibly() + Thread.currentThread().interrupt() + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.interrupted", directory)) + return Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } + + return if (exitCode == 0) { + Result.success(Unit) + } + else { + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.exitCode", directory, exitCode)) + Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } +} + +private fun createMkdirCommandLine(request: TargetEnvironmentRequest, directory: FullPathOnTarget): TargetedCommandLine { + val commandLineBuilder = TargetedCommandLineBuilder(request) + commandLineBuilder.setExePath("/bin/mkdir") + commandLineBuilder.addParameters("-p", directory) + return commandLineBuilder.build() +} + private class TargetProcessCommands( override val scopeToBind: CoroutineScope, private val exePath: FullPathOnTarget, diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt b/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt index e16c59fd5899..13c425ca77ff 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt @@ -7,4 +7,6 @@ internal sealed interface ExecOptionsBase { val env: Map val processDescription: @Nls String? val tty: TtySize? + val uploadBeforeExecution: UploadConfig? + val downloadAfterExecution: DownloadConfig? } \ No newline at end of file