PY-85897 PY-88188: Speed-up SSH/WSL file uploading.

We used to upload helpers on every single `ExecService` call.

We now:
1. Add `measureUploadTime` to measure upload time.
2. Fix `TargetEnvironmentRequestHandler` to delegate/bridge uploading process to `HelpersAwareTargetEnvironmentRequest`. These classes return mapping for WSL and upload helpers to SSH.
3. Add cache to SSH not to access the remote server too often.

(cherry picked from commit 24503ef19c78ce678cd34801b72620c95537644b)

GitOrigin-RevId: 651682cbfb8a8c3dc028435e913011bb3cd1aab1

python/python-exec-service/BUILD.bazel

@@ -44,6 +44,7 @@ jvm_library(
     "//libraries/guava",
     "//python/python-process-output/common",
     "//python/impl.helperLocator:community-helpersLocator",
+    "//platform/execution-impl",
   ]
 )
 
@@ -82,6 +83,7 @@ jvm_library(
     "//libraries/guava",
     "//python/python-process-output/common",
     "//python/impl.helperLocator:community-helpersLocator",
+    "//platform/execution-impl",
   ]
 )
 ### auto-generated section `build intellij.python.community.execService` end

python/python-exec-service/intellij.python.community.execService.iml

@@ -55,5 +55,6 @@
     <orderEntry type="module" module-name="intellij.libraries.guava" />
     <orderEntry type="module" module-name="intellij.python.processOutput.common" />
     <orderEntry type="module" module-name="intellij.python.community.helpersLocator" />
+    <orderEntry type="module" module-name="intellij.platform.execution.impl" />
   </component>
 </module>

python/python-exec-service/resources/intellij.python.community.execService.xml

@@ -1,24 +1,18 @@
 <idea-plugin visibility="public">
   <dependencies>
-    <module name="intellij.python.community"/>
     <module name="intellij.python.community.helpersLocator"/>
+    <!-- region Generated dependencies - run `Generate Product Layouts` to regenerate -->
+    <module name="intellij.python.community"/>
     <module name="intellij.python.processOutput.common"/>
+    <!-- endregion -->
   </dependencies>
 
   <extensionPoints>
     <extensionPoint qualifiedName="Pythonid.execService.targetEnvironmentRequestHandler"
-                    interface="com.intellij.python.community.execService.spi.TargetEnvironmentRequestHandler"
+                    interface="com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler"
                     dynamic="true"/>
 
   </extensionPoints>
-
-  <extensions defaultExtensionNs="Pythonid">
-    <execService.targetEnvironmentRequestHandler
-      implementation="com.intellij.python.community.execService.impl.processLaunchers.DefaultTargetEnvironmentRequestHandler"
-      order="last"
-    />
-  </extensions>
-
   <extensions defaultExtensionNs="com.intellij">
     <registryKey defaultValue="2"
                  description="How many heavy processes to execute concurrently"

python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt

@@ -0,0 +1,135 @@
+// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.community.execService.impl
+
+import com.intellij.execution.target.TargetEnvironment
+import com.intellij.execution.target.TargetEnvironmentRequest
+import com.intellij.openapi.extensions.ExtensionPointName
+import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler.Companion.mapUploadRoots
+import com.jetbrains.python.run.target.HelpersAwareTargetEnvironmentRequest
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import org.jetbrains.annotations.ApiStatus
+import java.nio.file.Path
+import kotlin.io.path.pathString
+
+/**
+ * Yet another abstraction on top of Targets API. Call [mapUploadRoots].
+ */
+@ApiStatus.Internal
+abstract class TargetEnvironmentRequestHandler<T : TargetEnvironmentRequest>(private val reqClass: Class<T>) {
+
+  private suspend fun mapUploadRootsIfValid(
+    request: TargetEnvironmentRequest,
+    localDirs: Set<Path>,
+    workingDirToDownload: Path?,
+  ): UploadInfo? =
+    if (reqClass.isInstance(request)) {
+      @Suppress("UNCHECKED_CAST") // Checked one line above
+      mapUploadRootsImpl(request as T, localDirs, workingDirToDownload)
+    }
+    else {
+      null
+    }
+
+  /**
+   * See [mapUploadRoots]
+   */
+  protected abstract suspend fun mapUploadRootsImpl(
+    request: T,
+    localDirs: Set<Path>,
+    workingDirToDownload: Path?,
+  ): UploadInfo
+
+
+  internal companion object {
+    private val EP_NAME: ExtensionPointName<TargetEnvironmentRequestHandler<*>> = ExtensionPointName.create(
+      "Pythonid.execService.targetEnvironmentRequestHandler"
+    )
+
+    data class UploadRootWithExplicitUploadInfo(
+      /**
+       * Root to add to upload volumes
+       */
+      val root: TargetEnvironment.UploadRoot,
+      /**
+       * If true, call [com.intellij.python.community.execService.impl.processLaunchers.measureUploadTime]
+       */
+      val uploadVolumeExplicitly: Boolean,
+    )
+
+    /**
+     * for [request] that needs access to  [localDirs] (with [workingDirToDownload] is a working directory) returns
+     * mapping info for each [localDirs]. This info is used to fill upload volumes, and possibly upload them.
+     */
+    suspend fun mapUploadRoots(
+      request: TargetEnvironmentRequest,
+      localDirs: Set<Path>,
+      workingDirToDownload: Path?,
+    ): Map<Path, UploadRootWithExplicitUploadInfo> = withContext(Dispatchers.IO) {
+      val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs, workingDirToDownload) }
+                       ?: error("No implementation of [${TargetEnvironmentRequestHandler::class.java}] is found for $request, broken bundle? " +
+                                "If you are in tests, set `@TestApplicationWithEel(useLegacyTargets=true)`")
+      val localToRemoteHelpersRoots =
+        uploadInfo.helpersAware.preparePyCharmHelpers().helpers.associate { it.localPath to it.targetPathFun.value }
+
+      fun localPathToTarget(localPath: Path): LocalPathToTargetResult {
+        val mightBeHelperRemoteRoot = localToRemoteHelpersRoots.entries.firstOrNull { localPath.startsWith(it.key) }?.value
+        return if (mightBeHelperRemoteRoot != null) {
+          // This is helper. They should never be uploaded explicitly, as handlers (inheritors) take care of them.
+          LocalPathToTargetResult(
+            targetPath = TargetEnvironment.TargetPath.Persistent(mightBeHelperRemoteRoot),
+            removeAndShutDown = false,
+            uploadVolumeExplicitly = false
+          )
+        }
+        else {
+          // This path was explicitly marked as persistent by inheritor
+          val explicitlyPersistent = uploadInfo.explicitlyPersistent[localPath]
+          if (explicitlyPersistent != null) {
+            LocalPathToTargetResult(
+              targetPath = explicitlyPersistent,
+              removeAndShutDown = true,
+              uploadVolumeExplicitly = false
+            )
+          }
+          else {
+            // Just a random temp path, but we try to preserve location of workDir (most probably projDir) is set.
+            LocalPathToTargetResult(
+              targetPath = TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString),
+              removeAndShutDown = true,
+              uploadVolumeExplicitly = true
+            )
+          }
+        }
+      }
+      localDirs.associateWith { localPath ->
+        val localPathToTargetRes = localPathToTarget(localPath)
+        val root = TargetEnvironment.UploadRoot(
+          localRootPath = localPath,
+          targetRootPath = localPathToTargetRes.targetPath,
+          removeAtShutdown = localPathToTargetRes.removeAndShutDown
+        )
+        UploadRootWithExplicitUploadInfo(root, uploadVolumeExplicitly = localPathToTargetRes.uploadVolumeExplicitly)
+      }
+    }
+  }
+
+  protected data class UploadInfo(
+    /**
+     * Each inheritor provides it so we can access the helpers.
+     */
+    val helpersAware: HelpersAwareTargetEnvironmentRequest,
+    /**
+     * These paths will be converted to persistent volumes (temporary otherwise)
+     */
+    val explicitlyPersistent: Map<Path, TargetEnvironment.TargetPath.Persistent> = emptyMap(),
+  )
+
+
+}
+
+private data class LocalPathToTargetResult(
+  val targetPath: TargetEnvironment.TargetPath,
+  val removeAndShutDown: Boolean,
+  val uploadVolumeExplicitly: Boolean,
+)

python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/DefaultTargetEnvironmentRequestHandler.kt

@@ -1,26 +0,0 @@
-// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.intellij.python.community.execService.impl.processLaunchers
-
-import com.intellij.execution.target.TargetEnvironment
-import com.intellij.execution.target.TargetEnvironmentRequest
-import com.intellij.python.community.execService.spi.TargetEnvironmentRequestHandler
-import java.nio.file.Path
-
-class DefaultTargetEnvironmentRequestHandler : TargetEnvironmentRequestHandler {
-  override fun mapUploadRoots(
-    request: TargetEnvironmentRequest,
-    localDirs: Set<Path>,
-    workingDirToDownload: Path?,
-  ): List<TargetEnvironment.UploadRoot> {
-    val result = localDirs.map { localDir ->
-      TargetEnvironment.UploadRoot(
-        localRootPath = localDir,
-        targetRootPath = TargetEnvironment.TargetPath.Temporary(),
-        removeAtShutdown = true
-      )
-    }
-    return result
-  }
-
-  override fun isApplicable(request: TargetEnvironmentRequest): Boolean = true
-}

python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt

@@ -11,19 +11,21 @@ import com.intellij.execution.target.TargetProgressIndicator
 import com.intellij.execution.target.TargetedCommandLine
 import com.intellij.execution.target.TargetedCommandLineBuilder
 import com.intellij.execution.target.getTargetPaths
-import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
 import com.intellij.execution.target.local.LocalTargetPtyOptions
+import com.intellij.openapi.diagnostic.debug
 import com.intellij.openapi.diagnostic.fileLogger
 import com.intellij.openapi.progress.coroutineToIndicator
 import com.intellij.openapi.project.ProjectManager
+import com.intellij.openapi.util.NlsSafe
 import com.intellij.platform.eel.provider.utils.ProcessFunctions
 import com.intellij.platform.eel.provider.utils.bindProcessToScopeImpl
 import com.intellij.python.community.execService.BinOnTarget
 import com.intellij.python.community.execService.DownloadConfig
 import com.intellij.python.community.execService.ExecuteGetProcessError
 import com.intellij.python.community.execService.impl.PyExecBundle
-import com.intellij.python.community.execService.spi.TargetEnvironmentRequestHandler
+import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler
 import com.intellij.remoteServer.util.ServerRuntimeException
+import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
 import com.jetbrains.python.Result
 import com.jetbrains.python.errorProcessing.Exe
 import com.jetbrains.python.errorProcessing.ExecErrorReason
@@ -32,9 +34,12 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.withContext
+import org.jetbrains.annotations.ApiStatus
 import java.io.IOException
+import java.nio.file.Path
 import kotlin.io.path.pathString
 import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.measureTime
 
 private val logger = fileLogger()
 
@@ -44,7 +49,7 @@ internal suspend fun createProcessLauncherOnTarget(
 ): Result<ProcessLauncher, ExecuteGetProcessError.EnvironmentError> = withContext(Dispatchers.IO) {
   val target = binOnTarget.target
 
-  val request = if (target != null) {
+  val request = run {
     val projectMan = ProjectManager.getInstance() // Broken Targets API doesn't work without project
     try {
       target.createEnvironmentRequest(projectMan.openProjects.firstOrNull() ?: projectMan.defaultProject)
@@ -53,7 +58,6 @@ internal suspend fun createProcessLauncherOnTarget(
       return@withContext Result.failure(ExecuteGetProcessError.EnvironmentError(MessageError(e.localizedMessage)))
     }
   }
-  else LocalTargetEnvironmentRequest()
 
   // Broken Targets API can only upload the whole directory
   val dirsToMap = buildSet {
@@ -62,15 +66,15 @@ internal suspend fun createProcessLauncherOnTarget(
       add(it)
     }
   }
-  val handler = TargetEnvironmentRequestHandler.getHandler(request)
-  val uploadRoots = handler.mapUploadRoots(request, dirsToMap, binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() })
-  request.uploadVolumes.addAll(uploadRoots)
+  val uploadRoots =
+    TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() })
+  request.uploadVolumes.addAll(uploadRoots.map { it.value.root })
 
   // Setup download roots if download is requested
   val downloadConfig = launchRequest.downloadConfig
   if (downloadConfig != null) {
     val localDirsToDownload = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { setOf(it) } ?: emptySet()
-    val downloadRoots = handler.mapDownloadRoots(request, request.uploadVolumes, localDirsToDownload)
+    val downloadRoots = mapDownloadRoots(request.uploadVolumes, localDirsToDownload)
     request.downloadVolumes.addAll(downloadRoots)
   }
 
@@ -85,8 +89,11 @@ internal suspend fun createProcessLauncherOnTarget(
     fileLogger().warn("Failed to start $target", e) // TODO: i18n
     return@withContext Result.failure(ExecuteGetProcessError.EnvironmentError(MessageError("Failed to start environment due to ${e.localizedMessage}")))
   }
-  targetEnv.uploadVolumes.forEach { _, volume ->
-    volume.upload(".", TargetProgressIndicator.EMPTY)
+  for (volume in targetEnv.uploadVolumes.values) {
+    val skipUploading = uploadRoots[volume.localRoot]?.uploadVolumeExplicitly == false
+    if (! skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers)
+      volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService")
+    }
   }
 
   val args = launchRequest.args.getArgs { localFile ->
@@ -192,3 +199,60 @@ private class TargetProcessCommands(
 
 private fun ExecutionException.asCantStart(): Result.Failure<ExecErrorReason.CantStart> =
   Result.failure(ExecErrorReason.CantStart(null, localizedMessage))
+
+/**
+ * Temporary function to [upload] from [localRoot] + [relativePath] to [targetRoot], much like [upload]
+ * but also reports time using [logger] if `debug` enabled.
+ */
+@ApiStatus.Internal
+@Throws(IOException::class)
+@RequiresBackgroundThread
+fun TargetEnvironment.UploadableVolume.uploadMeasureTime(
+  relativePath: String,
+  targetProgressIndicator: TargetProgressIndicator,
+  @NlsSafe reason: String,
+) {
+  measureUploadTime(
+    upload = {
+      upload(relativePath, targetProgressIndicator)
+    },
+    genMessage = {
+      "$reason: ${this.localRoot} -> ${targetRoot}"
+    }
+  )
+}
+
+/**
+ * Measures time for [upload] and reports it along with [genMessage] if `debug` enabled using [logger]
+ */
+@ApiStatus.Internal
+@RequiresBackgroundThread
+fun measureUploadTime(@RequiresBackgroundThread upload: () -> Unit, genMessage: () -> @NlsSafe String) {
+  val duration = measureTime { upload() }
+  logger.debug { "upload ${genMessage()} : $duration" }
+}
+
+/**
+ * Maps download roots using existing upload roots.
+ * This allows downloading files modified on the target back to the local machine.
+ *
+ * @param uploadRoots set of upload roots
+ * @param localDirs local directories that were uploaded and may need to be downloaded
+ * @return list of download roots, empty by default
+ */
+private fun mapDownloadRoots(
+  uploadRoots: Set<TargetEnvironment.UploadRoot>,
+  localDirs: Set<Path>,
+): List<TargetEnvironment.DownloadRoot> = localDirs.mapNotNull { localDir ->
+  val matchingUpload = uploadRoots.find { localDir.startsWith(it.localRootPath) }
+
+  if (matchingUpload != null) {
+    TargetEnvironment.DownloadRoot(
+      localRootPath = localDir,
+      targetRootPath = matchingUpload.targetRootPath,
+    )
+  }
+  else {
+    null // No matching upload, skip download
+  }
+}

python/python-exec-service/src/com/intellij/python/community/execService/spi/TargetEnvironmentRequestHandler.kt

@@ -1,62 +0,0 @@
-// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.intellij.python.community.execService.spi
-
-import com.intellij.execution.target.TargetEnvironment
-import com.intellij.execution.target.TargetEnvironmentRequest
-import com.intellij.openapi.extensions.ExtensionPointName
-import org.jetbrains.annotations.ApiStatus
-import java.nio.file.Path
-
-@ApiStatus.Internal
-interface TargetEnvironmentRequestHandler {
-
-  fun isApplicable(request: TargetEnvironmentRequest): Boolean
-
-  fun mapUploadRoots(
-    request: TargetEnvironmentRequest,
-    localDirs: Set<Path>,
-    workingDirToDownload: Path?
-  ): List<TargetEnvironment.UploadRoot>
-
-  /**
-   * Maps download roots using existing upload roots.
-   * This allows downloading files modified on the target back to the local machine.
-   *
-   * @param request the target environment request
-   * @param uploadRoots set of upload roots
-   * @param localDirs local directories that were uploaded and may need to be downloaded
-   * @return list of download roots, empty by default
-   */
-  fun mapDownloadRoots(
-    request: TargetEnvironmentRequest,
-    uploadRoots: Set<TargetEnvironment.UploadRoot>,
-    localDirs: Set<Path>,
-  ): List<TargetEnvironment.DownloadRoot> = localDirs.mapNotNull { localDir ->
-    val matchingUpload = uploadRoots.find { localDir.startsWith(it.localRootPath) }
-
-    if (matchingUpload != null) {
-      TargetEnvironment.DownloadRoot(
-        localRootPath = localDir,
-        targetRootPath = matchingUpload.targetRootPath,
-      )
-    }
-    else {
-      null // No matching upload, skip download
-    }
-  }
-
-  companion object {
-    @JvmField
-    val EP_NAME: ExtensionPointName<TargetEnvironmentRequestHandler> = ExtensionPointName.create<TargetEnvironmentRequestHandler>(
-      "Pythonid.execService.targetEnvironmentRequestHandler"
-    )
-
-    @JvmStatic
-    @ApiStatus.Internal
-    fun getHandler(request: TargetEnvironmentRequest): TargetEnvironmentRequestHandler {
-      val handler = EP_NAME.extensionList.firstOrNull { it.isApplicable(request) }
-                    ?: error("No implementation of [${TargetEnvironmentRequestHandler::class.java}] is found for $request, bundle is broken?")
-      return handler
-    }
-  }
-}

python/src/com/jetbrains/python/run/target/HelpersAwareLocalTargetEnvironmentRequest.kt

@@ -4,10 +4,12 @@ package com.jetbrains.python.run.target
 import com.intellij.execution.target.TargetEnvironmentRequest
 import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
 import com.intellij.execution.target.value.constantExplicit
+import org.jetbrains.annotations.ApiStatus
 import java.nio.file.Path
 import kotlin.io.path.absolutePathString
 
-class HelpersAwareLocalTargetEnvironmentRequest : HelpersAwareTargetEnvironmentRequest {
+@ApiStatus.Internal
+internal class HelpersAwareLocalTargetEnvironmentRequest : HelpersAwareTargetEnvironmentRequest {
   override val targetEnvironmentRequest: TargetEnvironmentRequest = LocalTargetEnvironmentRequest()
 
   override fun preparePyCharmHelpers(): PythonHelpersMappings =

python/src/com/jetbrains/python/sdk/PyTargetsRemoteSourcesRefresher.kt

@@ -21,6 +21,7 @@ import com.intellij.openapi.project.rootManager
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.python.community.execService.impl.processLaunchers.uploadMeasureTime
 import com.intellij.remote.RemoteSdkProperties
 import com.intellij.util.PathMappingSettings
 import com.intellij.util.PathUtil
@@ -108,7 +109,7 @@ class PyTargetsRemoteSourcesRefresher(val sdk: Sdk, private val project: Project
     val environment = targetEnvRequest.prepareEnvironment(targetIndicator)
     try {
       // XXX Make it automatic
-      environment.uploadVolumes.values.forEach { it.upload(".", targetIndicator) }
+      environment.uploadVolumes.values.forEach { it.uploadMeasureTime(".", targetIndicator, "sourceRefresher") }
 
       val cmd = execution.buildTargetedCommandLine(environment, sdk, emptyList())
       cmd.execute(environment, indicator)

python/src/com/jetbrains/python/sdk/skeletons/PyTargetsSkeletonGenerator.kt

@@ -18,6 +18,7 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ProjectManager
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.util.io.FileUtil
+import com.intellij.python.community.execService.impl.processLaunchers.uploadMeasureTime
 import com.jetbrains.python.PythonHelper
 import com.jetbrains.python.run.PythonInterpreterTargetEnvironmentFactory
 import com.jetbrains.python.run.buildTargetedCommandLine
@@ -122,7 +123,7 @@ class PyTargetsSkeletonGenerator(skeletonPath: String, pySdk: Sdk, currentFolder
       try {
 
         // XXX Make it automatic
-        targetEnvironment.uploadVolumes.values.forEach { it.upload(".", TargetProgressIndicator.EMPTY) }
+        targetEnvironment.uploadVolumes.values.forEach { it.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "skeleton") }
 
         val targetedCommandLine = generatorScriptExecution.buildTargetedCommandLine(targetEnvironment, sdk, emptyList())
         val process = targetEnvironment.createProcess(targetedCommandLine, EmptyProgressIndicator())