PY-52688 support selective uploads for target execution

Let target-backed process launches create their working directory and upload only requested project metadata. Reuse persistent deployment mappings without scheduling their remote roots for cleanup, while keeping temporary upload roots disposable. (cherry picked from commit 746c082e3ad2d8ac4f30d1f95d433b8305f2b73d) IJ-MR-215198 GitOrigin-RevId: 29e84212fee02f037800535e264abe1d17b720cc
2026-09-27 10:03:11 +07:00 · 2026-07-24 21:01:10 +00:00
parent d9505303c4
commit 4e5531e4c5
7 changed files with 154 additions and 26 deletions
@@ -8,5 +8,9 @@ py.exec.error.not.zero=Exit code is not zero
 py.exec.error.unexpected.output=Unexpected output {0}
 py.exec.target.name.default="Local"
 py.exec.error.not.trusted=Could not start {0}: project is not trusted
+py.exec.target.working.directory.create.error=Could not create target working directory {0}: {1}
+py.exec.target.working.directory.create.interrupted=Interrupted while creating target working directory {0}
+py.exec.target.working.directory.create.exitCode=Could not create target working directory {0}: mkdir exited with {1}
+py.exec.target.working.directory.create.unsupported=Could not create target working directory {0}: Windows targets are not supported

 py.exec.target.binaries.are.not.supported=Target binaries are not supported
@@ -61,8 +61,16 @@ data class BinOnEel(val path: Path, internal val workDir: Path? = null) : Binary
 * Legacy Targets-based approach. Do not use it, unless you know what you are doing
 * if [target] "local" target is used
 */
-data class BinOnTarget(internal val configureTargetCmdLine: (TargetedCommandLineBuilder) -> Unit, val target: TargetEnvironmentConfiguration, val workingDir: Path? = null) : BinaryToExec {
-  constructor(exePath: FullPathOnTarget, target: TargetEnvironmentConfiguration, workingDir: Path? = null) : this({ it.setExePath(exePath) }, target, workingDir)
+data class BinOnTarget(
+  internal val configureTargetCmdLine: (TargetedCommandLineBuilder) -> Unit,
+  val target: TargetEnvironmentConfiguration,
+  val workingDir: Path? = null,
+) : BinaryToExec {
+  constructor(
+    exePath: FullPathOnTarget,
+    target: TargetEnvironmentConfiguration,
+    workingDir: Path? = null,
+  ) : this({ it.setExePath(exePath) }, target, workingDir)

  @RequiresBackgroundThread
  fun getLocalExePath(): Lazy<FullPathOnTarget> = lazy {
@@ -249,12 +257,26 @@ data class DownloadConfig(
  val relativePaths: List<RelativePathOnTarget> = emptyList(),
 )

+/**
+ * Configuration for uploading files before command execution.
+ * Uses existing upload volume mappings (from web deployment).
+ *
+ * @param relativePaths Relative paths to upload from the working directory.
+ *                      Empty list means upload no files.
+ * @param ensureWorkingDirectoryExists Creates the target working directory before execution, even when no files are uploaded.
+ */
+data class UploadConfig(
+  val relativePaths: List<RelativePathOnTarget> = emptyList(),
+  val ensureWorkingDirectoryExists: Boolean = true,
+)
+
 /**
 * @property[env] Environment variables to be applied with the process run
 * @property[timeout] Process gets killed after this timeout
 * @property[processDescription] optional description to be displayed to user
 * @property[tty] Much like [com.intellij.platform.eel.EelExecApi.Pty]
 * @property[weight] use it to limit the number of concurrent processes not to exhaust user resources, see [ConcurrentProcessWeight]
+ * @property[uploadBeforeExecution] configuration for uploading files before command execution (Target-based execution only)
 * @property[downloadAfterExecution] configuration for downloading files after command execution (Target-based execution only)
 */
 data class ExecOptions(
@@ -263,7 +285,8 @@ data class ExecOptions(
  val timeout: Duration = 5.minutes,
  override val tty: TtySize? = null,
  val weight: ConcurrentProcessWeight = ConcurrentProcessWeight.LIGHT,
-  val downloadAfterExecution: DownloadConfig? = null,
+  override val uploadBeforeExecution: UploadConfig? = null,
+  override val downloadAfterExecution: DownloadConfig? = null,
 ) : ExecOptionsBase


@@ -275,6 +298,8 @@ data class ExecGetProcessOptions(
  override val env: Map<String, String> = emptyMap(),
  override val processDescription: @Nls String? = null,
  override val tty: TtySize? = null,
+  override val uploadBeforeExecution: UploadConfig? = null,
+  override val downloadAfterExecution: DownloadConfig? = null,
 ) : ExecOptionsBase

 data class TtySize(val rows: UShort, val cols: UShort)
@@ -45,7 +45,12 @@ internal class ExecServiceImpl private constructor() : ExecService {
    }
  }

-  override suspend fun executeGetProcess(binary: BinaryToExec, args: Args, scopeToBind: CoroutineScope?, options: ExecGetProcessOptions): Result<Process, ExecuteGetProcessError<*>> {
+  override suspend fun executeGetProcess(
+    binary: BinaryToExec,
+    args: Args,
+    scopeToBind: CoroutineScope?,
+    options: ExecGetProcessOptions,
+  ): Result<Process, ExecuteGetProcessError<*>> {
    val launcher = create(binary, args, options, scopeToBind).getOr { return it }
    val process = launcher.start().getOr {
      val createExecError = launcher.createExecError(options.processDescription ?: "", it.error).error
@@ -54,10 +59,14 @@ internal class ExecServiceImpl private constructor() : ExecService {
    return Result.success(process)
  }

-  private suspend fun create(binary: BinaryToExec, args: Args, options: ExecOptionsBase, scopeToBind: CoroutineScope? = null): Result<ProcessLauncher, ExecuteGetProcessError.EnvironmentError> {
+  private suspend fun create(
+    binary: BinaryToExec,
+    args: Args,
+    options: ExecOptionsBase,
+    scopeToBind: CoroutineScope? = null,
+  ): Result<ProcessLauncher, ExecuteGetProcessError.EnvironmentError> {
    val scope = scopeToBind ?: ApplicationManager.getApplication().service<MyService>().scope
-    val downloadConfig = (options as? ExecOptions)?.downloadAfterExecution
-    val request = LaunchRequest(scope, args, options.env, options.tty, downloadConfig)
+    val request = LaunchRequest(scope, args, options.env, options.tty, options.uploadBeforeExecution, options.downloadAfterExecution)
    return Result.success(
      when (binary) {
        is BinOnEel -> createProcessLauncherOnEel(binary, request)
@@ -70,7 +79,12 @@ internal class ExecServiceImpl private constructor() : ExecService {
      })
  }

-  override suspend fun <T> executeAdvanced(binary: BinaryToExec, args: Args, options: ExecOptions, processInteractiveHandler: ProcessInteractiveHandler<T>): PyResult<T> {
+  override suspend fun <T> executeAdvanced(
+    binary: BinaryToExec,
+    args: Args,
+    options: ExecOptions,
+    processInteractiveHandler: ProcessInteractiveHandler<T>,
+  ): PyResult<T> {
    return coroutineScope {
      val processLauncher = create(binary, args, options, this).getOr {
        return@coroutineScope it.asPyError()
@@ -89,7 +103,8 @@ internal class ExecServiceImpl private constructor() : ExecService {
    processInteractiveHandler: ProcessInteractiveHandler<T>,
  ): Result<T, ExecError> {
    val description = options.processDescription
-                      ?: PyExecBundle.message("py.exec.defaultName.process", (listOf(processLauncher.exeForError.toString()) + processLauncher.args).joinToString(" "))
+                      ?: PyExecBundle.message("py.exec.defaultName.process",
+                                              (listOf(processLauncher.exeForError.toString()) + processLauncher.args).joinToString(" "))
    val process = processLauncher.start(options.weight).getOr {
      val message = PyExecBundle.message("py.exec.start.error", description, it.error.cantExecProcessError, it.error.errNo
                                                                                                            ?: "unknown")
@@ -1,11 +1,11 @@
 // Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.community.execService.impl

+import com.intellij.execution.target.FullPathOnTarget
 import com.intellij.execution.target.TargetEnvironment
 import com.intellij.execution.target.TargetEnvironmentRequest
 import com.intellij.openapi.extensions.ExtensionPointName
 import com.intellij.openapi.progress.coroutineToIndicator
-import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler.Companion.mapUploadRoots
 import com.jetbrains.python.run.target.HelpersAwareTargetEnvironmentRequest
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
@@ -22,11 +22,10 @@ abstract class TargetEnvironmentRequestHandler<T : TargetEnvironmentRequest>(pri
  private suspend fun mapUploadRootsIfValid(
    request: TargetEnvironmentRequest,
    localDirs: Set<Path>,
-    workingDirToDownload: Path?,
  ): UploadInfo? =
    if (reqClass.isInstance(request)) {
      @Suppress("UNCHECKED_CAST") // Checked one line above
-      mapUploadRootsImpl(request as T, localDirs, workingDirToDownload)
+      mapUploadRootsImpl(request as T, localDirs)
    }
    else {
      null
@@ -38,7 +37,6 @@ abstract class TargetEnvironmentRequestHandler<T : TargetEnvironmentRequest>(pri
  protected abstract suspend fun mapUploadRootsImpl(
    request: T,
    localDirs: Set<Path>,
-    workingDirToDownload: Path?,
  ): UploadInfo


@@ -67,7 +65,7 @@ abstract class TargetEnvironmentRequestHandler<T : TargetEnvironmentRequest>(pri
      localDirs: Set<Path>,
      workingDirToDownload: Path?,
    ): Map<Path, UploadRootWithExplicitUploadInfo> = withContext(Dispatchers.IO) {
-      val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs, workingDirToDownload) }
+      val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs) }
                       ?: error("No implementation of [${TargetEnvironmentRequestHandler::class.java}] is found for $request, broken bundle? " +
                                "If you are in tests, set `@TestApplicationWithEel(useLegacyTargets=true)`")
      val localToRemoteHelpersRoots = coroutineToIndicator {
@@ -86,10 +84,13 @@ abstract class TargetEnvironmentRequestHandler<T : TargetEnvironmentRequest>(pri
          )
        }
        else {
-          // Just a random temp path, but we try to preserve location of workDir (most probably projDir) is set.
+          // Either we have a persistent path mapping, or it's just a random temp path, but we try to preserve location of
+          // workDir (most probably projDir) is set.
+          val (targetPath, removeAtShutdown) = uploadInfo.persistentPaths[localPath]?.let { TargetEnvironment.TargetPath.Persistent(it) to false }
+                                               ?: (TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString) to true)
          LocalPathToTargetResult(
-            targetPath = TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString),
-            removeAtShutdown = true,
+            targetPath = targetPath,
+            removeAtShutdown = removeAtShutdown,
            uploadVolumeExplicitly = true
          )
        }
@@ -113,6 +114,7 @@ abstract class TargetEnvironmentRequestHandler<T : TargetEnvironmentRequest>(pri
     * Each inheritor provides it so we can access the helpers.
     */
    val helpersAware: HelpersAwareTargetEnvironmentRequest,
+    val persistentPaths: Map<Path, FullPathOnTarget> = emptyMap(),
  )


@@ -7,6 +7,7 @@ import com.intellij.python.community.execService.Args
 import com.intellij.python.community.execService.ConcurrentProcessWeight
 import com.intellij.python.community.execService.DownloadConfig
 import com.intellij.python.community.execService.TtySize
+import com.intellij.python.community.execService.UploadConfig
 import com.intellij.python.community.execService.impl.LoggingProcess
 import com.jetbrains.python.Result
 import com.jetbrains.python.TraceContext
@@ -61,5 +62,6 @@ internal data class LaunchRequest(
  val args: Args,
  val env: Map<String, String>,
  val usePty: TtySize?,
+  val uploadConfig: UploadConfig? = null,
  val downloadConfig: DownloadConfig? = null,
 )
@@ -4,9 +4,11 @@
 package com.intellij.python.community.execService.impl.processLaunchers

 import com.intellij.execution.ExecutionException
+import com.intellij.execution.Platform
 import com.intellij.execution.process.LocalPtyOptions
 import com.intellij.execution.target.FullPathOnTarget
 import com.intellij.execution.target.TargetEnvironment
+import com.intellij.execution.target.TargetEnvironmentRequest
 import com.intellij.execution.target.TargetProgressIndicator
 import com.intellij.execution.target.TargetedCommandLine
 import com.intellij.execution.target.TargetedCommandLineBuilder
@@ -22,6 +24,7 @@ import com.intellij.platform.eel.impl.base.bindProcessToScopeImpl
 import com.intellij.python.community.execService.BinOnTarget
 import com.intellij.python.community.execService.DownloadConfig
 import com.intellij.python.community.execService.ExecuteGetProcessError
+import com.intellij.python.community.execService.UploadConfig
 import com.intellij.python.community.execService.impl.PyExecBundle
 import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler
 import com.intellij.remoteServer.util.ServerRuntimeException
@@ -59,21 +62,21 @@ internal suspend fun createProcessLauncherOnTarget(
    }
  }

-  // Broken Targets API can only upload the whole directory
+  // Targets API maps local roots as directories; callers may still restrict which files are uploaded below.
+  val workingDir = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }
  val dirsToMap = buildSet {
    addAll(launchRequest.args.localFiles.map { it.parent })
-    binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.also {
+    workingDir?.also {
      add(it)
    }
  }
-  val uploadRoots =
-    TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() })
+  val uploadRoots = TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, workingDir)
  request.uploadVolumes.addAll(uploadRoots.map { it.value.root })

  // Setup download roots if download is requested
  val downloadConfig = launchRequest.downloadConfig
  if (downloadConfig != null) {
-    val localDirsToDownload = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { setOf(it) } ?: emptySet()
+    val localDirsToDownload = workingDir?.let { setOf(it) } ?: emptySet()
    val downloadRoots = mapDownloadRoots(request.uploadVolumes, localDirsToDownload)
    request.downloadVolumes.addAll(downloadRoots)
  }
@@ -89,10 +92,18 @@ internal suspend fun createProcessLauncherOnTarget(
    fileLogger().warn("Failed to start $target", e) // TODO: i18n
    return@withContext Result.failure(ExecuteGetProcessError.EnvironmentError(MessageError("Failed to start environment due to ${e.localizedMessage}")))
  }
+  val workingDirOnTarget = workingDir?.let { targetEnv.getTargetPaths(it.pathString).firstOrNull() ?: it.pathString }
+  val uploadConfig = launchRequest.uploadConfig
+  if (uploadConfig?.ensureWorkingDirectoryExists == true && workingDirOnTarget != null) {
+    ensureTargetDirectoryExists(targetEnv, request, workingDirOnTarget).getOr { failure ->
+      targetEnv.shutdown()
+      return@withContext failure
+    }
+  }
  for (volume in targetEnv.uploadVolumes.values) {
    val skipUploading = uploadRoots[volume.localRoot]?.uploadVolumeExplicitly == false
-    if (! skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers)
-      volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService")
+    if (!skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers)
+      uploadVolume(volume, workingDir, uploadConfig, launchRequest.args.localFiles)
    }
  }

@@ -105,9 +116,8 @@ internal suspend fun createProcessLauncherOnTarget(
    // exe path is always fixed (pre-presolved) promise. It can't be obtained directly because of Targets API limitation
    exePath = commandLineBuilder.exePath.localValue.blockingGet(1000) ?: error("Exe path not set: $binOnTarget is broken")
    // Map working directory through upload volumes if it's a local path
-    binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { workingDir ->
+    if (workingDirOnTarget != null) {
      // Try to resolve through upload volumes (in case workingDir is a local path that needs mapping)
-      val workingDirOnTarget = targetEnv.getTargetPaths(workingDir.pathString).firstOrNull() ?: workingDir.pathString
      commandLineBuilder.setWorkingDirectory(workingDirOnTarget)
    }
    launchRequest.usePty?.let {
@@ -134,6 +144,74 @@ internal suspend fun createProcessLauncherOnTarget(
                                                                                            downloadConfig)))
 }

+private fun uploadVolume(
+  volume: TargetEnvironment.UploadableVolume,
+  workingDir: Path?,
+  uploadConfig: UploadConfig?,
+  localFiles: List<Path>,
+) {
+  if (uploadConfig != null && workingDir != null && volume.localRoot == workingDir) {
+    val localFileRelativePaths = localFiles
+      .filter { it.parent == workingDir }
+      .map { workingDir.relativize(it).pathString }
+    val pathsToUpload = (uploadConfig.relativePaths + localFileRelativePaths).distinct()
+    for (path in pathsToUpload) {
+      volume.uploadMeasureTime(path, TargetProgressIndicator.EMPTY, "execService")
+    }
+  }
+  else {
+    volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService")
+  }
+}
+
+private fun ensureTargetDirectoryExists(
+  targetEnv: TargetEnvironment,
+  request: TargetEnvironmentRequest,
+  directory: FullPathOnTarget,
+): Result<Unit, ExecuteGetProcessError.EnvironmentError> {
+  when (request.targetPlatform.platform) {
+    Platform.UNIX -> Unit
+    Platform.WINDOWS -> {
+      val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.unsupported", directory))
+      return Result.failure(ExecuteGetProcessError.EnvironmentError(error))
+    }
+  }
+
+  val cmdLine = createMkdirCommandLine(request, directory)
+  val process = try {
+    targetEnv.createProcess(cmdLine)
+  }
+  catch (e: ExecutionException) {
+    val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.error", directory, e.localizedMessage))
+    return Result.failure(ExecuteGetProcessError.EnvironmentError(error))
+  }
+
+  val exitCode = try {
+    process.waitFor()
+  }
+  catch (_: InterruptedException) {
+    process.destroyForcibly()
+    Thread.currentThread().interrupt()
+    val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.interrupted", directory))
+    return Result.failure(ExecuteGetProcessError.EnvironmentError(error))
+  }
+
+  return if (exitCode == 0) {
+    Result.success(Unit)
+  }
+  else {
+    val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.exitCode", directory, exitCode))
+    Result.failure(ExecuteGetProcessError.EnvironmentError(error))
+  }
+}
+
+private fun createMkdirCommandLine(request: TargetEnvironmentRequest, directory: FullPathOnTarget): TargetedCommandLine {
+  val commandLineBuilder = TargetedCommandLineBuilder(request)
+  commandLineBuilder.setExePath("/bin/mkdir")
+  commandLineBuilder.addParameters("-p", directory)
+  return commandLineBuilder.build()
+}
+
 private class TargetProcessCommands(
  override val scopeToBind: CoroutineScope,
  private val exePath: FullPathOnTarget,
@@ -7,4 +7,6 @@ internal sealed interface ExecOptionsBase {
  val env: Map<String, String>
  val processDescription: @Nls String?
  val tty: TtySize?
+  val uploadBeforeExecution: UploadConfig?
+  val downloadAfterExecution: DownloadConfig?
 }