diff --git a/python/python-exec-service/resources/messages/PyExecBundle.properties b/python/python-exec-service/resources/messages/PyExecBundle.properties index d860a4351573..8d3302daca90 100644 --- a/python/python-exec-service/resources/messages/PyExecBundle.properties +++ b/python/python-exec-service/resources/messages/PyExecBundle.properties @@ -8,5 +8,9 @@ py.exec.error.not.zero=Exit code is not zero py.exec.error.unexpected.output=Unexpected output {0} py.exec.target.name.default="Local" py.exec.error.not.trusted=Could not start {0}: project is not trusted +py.exec.target.working.directory.create.error=Could not create target working directory {0}: {1} +py.exec.target.working.directory.create.interrupted=Interrupted while creating target working directory {0} +py.exec.target.working.directory.create.exitCode=Could not create target working directory {0}: mkdir exited with {1} +py.exec.target.working.directory.create.unsupported=Could not create target working directory {0}: Windows targets are not supported py.exec.target.binaries.are.not.supported=Target binaries are not supported diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/api.kt b/python/python-exec-service/src/com/intellij/python/community/execService/api.kt index 41201109cfe7..0a750336f4df 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/api.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/api.kt @@ -61,8 +61,16 @@ data class BinOnEel(val path: Path, internal val workDir: Path? = null) : Binary * Legacy Targets-based approach. Do not use it, unless you know what you are doing * if [target] "local" target is used */ -data class BinOnTarget(internal val configureTargetCmdLine: (TargetedCommandLineBuilder) -> Unit, val target: TargetEnvironmentConfiguration, val workingDir: Path? = null) : BinaryToExec { - constructor(exePath: FullPathOnTarget, target: TargetEnvironmentConfiguration, workingDir: Path? = null) : this({ it.setExePath(exePath) }, target, workingDir) +data class BinOnTarget( + internal val configureTargetCmdLine: (TargetedCommandLineBuilder) -> Unit, + val target: TargetEnvironmentConfiguration, + val workingDir: Path? = null, +) : BinaryToExec { + constructor( + exePath: FullPathOnTarget, + target: TargetEnvironmentConfiguration, + workingDir: Path? = null, + ) : this({ it.setExePath(exePath) }, target, workingDir) @RequiresBackgroundThread fun getLocalExePath(): Lazy = lazy { @@ -249,12 +257,26 @@ data class DownloadConfig( val relativePaths: List = emptyList(), ) +/** + * Configuration for uploading files before command execution. + * Uses existing upload volume mappings (from web deployment). + * + * @param relativePaths Relative paths to upload from the working directory. + * Empty list means upload no files. + * @param ensureWorkingDirectoryExists Creates the target working directory before execution, even when no files are uploaded. + */ +data class UploadConfig( + val relativePaths: List = emptyList(), + val ensureWorkingDirectoryExists: Boolean = true, +) + /** * @property[env] Environment variables to be applied with the process run * @property[timeout] Process gets killed after this timeout * @property[processDescription] optional description to be displayed to user * @property[tty] Much like [com.intellij.platform.eel.EelExecApi.Pty] * @property[weight] use it to limit the number of concurrent processes not to exhaust user resources, see [ConcurrentProcessWeight] + * @property[uploadBeforeExecution] configuration for uploading files before command execution (Target-based execution only) * @property[downloadAfterExecution] configuration for downloading files after command execution (Target-based execution only) */ data class ExecOptions( @@ -263,7 +285,8 @@ data class ExecOptions( val timeout: Duration = 5.minutes, override val tty: TtySize? = null, val weight: ConcurrentProcessWeight = ConcurrentProcessWeight.LIGHT, - val downloadAfterExecution: DownloadConfig? = null, + override val uploadBeforeExecution: UploadConfig? = null, + override val downloadAfterExecution: DownloadConfig? = null, ) : ExecOptionsBase @@ -275,6 +298,8 @@ data class ExecGetProcessOptions( override val env: Map = emptyMap(), override val processDescription: @Nls String? = null, override val tty: TtySize? = null, + override val uploadBeforeExecution: UploadConfig? = null, + override val downloadAfterExecution: DownloadConfig? = null, ) : ExecOptionsBase data class TtySize(val rows: UShort, val cols: UShort) diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt index 75dab2228cb2..c0132d9ca5fa 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt @@ -45,7 +45,12 @@ internal class ExecServiceImpl private constructor() : ExecService { } } - override suspend fun executeGetProcess(binary: BinaryToExec, args: Args, scopeToBind: CoroutineScope?, options: ExecGetProcessOptions): Result> { + override suspend fun executeGetProcess( + binary: BinaryToExec, + args: Args, + scopeToBind: CoroutineScope?, + options: ExecGetProcessOptions, + ): Result> { val launcher = create(binary, args, options, scopeToBind).getOr { return it } val process = launcher.start().getOr { val createExecError = launcher.createExecError(options.processDescription ?: "", it.error).error @@ -54,10 +59,14 @@ internal class ExecServiceImpl private constructor() : ExecService { return Result.success(process) } - private suspend fun create(binary: BinaryToExec, args: Args, options: ExecOptionsBase, scopeToBind: CoroutineScope? = null): Result { + private suspend fun create( + binary: BinaryToExec, + args: Args, + options: ExecOptionsBase, + scopeToBind: CoroutineScope? = null, + ): Result { val scope = scopeToBind ?: ApplicationManager.getApplication().service().scope - val downloadConfig = (options as? ExecOptions)?.downloadAfterExecution - val request = LaunchRequest(scope, args, options.env, options.tty, downloadConfig) + val request = LaunchRequest(scope, args, options.env, options.tty, options.uploadBeforeExecution, options.downloadAfterExecution) return Result.success( when (binary) { is BinOnEel -> createProcessLauncherOnEel(binary, request) @@ -70,7 +79,12 @@ internal class ExecServiceImpl private constructor() : ExecService { }) } - override suspend fun executeAdvanced(binary: BinaryToExec, args: Args, options: ExecOptions, processInteractiveHandler: ProcessInteractiveHandler): PyResult { + override suspend fun executeAdvanced( + binary: BinaryToExec, + args: Args, + options: ExecOptions, + processInteractiveHandler: ProcessInteractiveHandler, + ): PyResult { return coroutineScope { val processLauncher = create(binary, args, options, this).getOr { return@coroutineScope it.asPyError() @@ -89,7 +103,8 @@ internal class ExecServiceImpl private constructor() : ExecService { processInteractiveHandler: ProcessInteractiveHandler, ): Result { val description = options.processDescription - ?: PyExecBundle.message("py.exec.defaultName.process", (listOf(processLauncher.exeForError.toString()) + processLauncher.args).joinToString(" ")) + ?: PyExecBundle.message("py.exec.defaultName.process", + (listOf(processLauncher.exeForError.toString()) + processLauncher.args).joinToString(" ")) val process = processLauncher.start(options.weight).getOr { val message = PyExecBundle.message("py.exec.start.error", description, it.error.cantExecProcessError, it.error.errNo ?: "unknown") diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt index b778d76115ce..1824a8514c5b 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/TargetEnvironmentRequestHandler.kt @@ -1,11 +1,11 @@ // Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.execService.impl +import com.intellij.execution.target.FullPathOnTarget import com.intellij.execution.target.TargetEnvironment import com.intellij.execution.target.TargetEnvironmentRequest import com.intellij.openapi.extensions.ExtensionPointName import com.intellij.openapi.progress.coroutineToIndicator -import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler.Companion.mapUploadRoots import com.jetbrains.python.run.target.HelpersAwareTargetEnvironmentRequest import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -22,11 +22,10 @@ abstract class TargetEnvironmentRequestHandler(pri private suspend fun mapUploadRootsIfValid( request: TargetEnvironmentRequest, localDirs: Set, - workingDirToDownload: Path?, ): UploadInfo? = if (reqClass.isInstance(request)) { @Suppress("UNCHECKED_CAST") // Checked one line above - mapUploadRootsImpl(request as T, localDirs, workingDirToDownload) + mapUploadRootsImpl(request as T, localDirs) } else { null @@ -38,7 +37,6 @@ abstract class TargetEnvironmentRequestHandler(pri protected abstract suspend fun mapUploadRootsImpl( request: T, localDirs: Set, - workingDirToDownload: Path?, ): UploadInfo @@ -67,7 +65,7 @@ abstract class TargetEnvironmentRequestHandler(pri localDirs: Set, workingDirToDownload: Path?, ): Map = withContext(Dispatchers.IO) { - val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs, workingDirToDownload) } + val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs) } ?: error("No implementation of [${TargetEnvironmentRequestHandler::class.java}] is found for $request, broken bundle? " + "If you are in tests, set `@TestApplicationWithEel(useLegacyTargets=true)`") val localToRemoteHelpersRoots = coroutineToIndicator { @@ -86,10 +84,13 @@ abstract class TargetEnvironmentRequestHandler(pri ) } else { - // Just a random temp path, but we try to preserve location of workDir (most probably projDir) is set. + // Either we have a persistent path mapping, or it's just a random temp path, but we try to preserve location of + // workDir (most probably projDir) is set. + val (targetPath, removeAtShutdown) = uploadInfo.persistentPaths[localPath]?.let { TargetEnvironment.TargetPath.Persistent(it) to false } + ?: (TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString) to true) LocalPathToTargetResult( - targetPath = TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString), - removeAtShutdown = true, + targetPath = targetPath, + removeAtShutdown = removeAtShutdown, uploadVolumeExplicitly = true ) } @@ -113,6 +114,7 @@ abstract class TargetEnvironmentRequestHandler(pri * Each inheritor provides it so we can access the helpers. */ val helpersAware: HelpersAwareTargetEnvironmentRequest, + val persistentPaths: Map = emptyMap(), ) diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt index 83b6b0ce5e30..75621b97553a 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/ProcessLauncher.kt @@ -7,6 +7,7 @@ import com.intellij.python.community.execService.Args import com.intellij.python.community.execService.ConcurrentProcessWeight import com.intellij.python.community.execService.DownloadConfig import com.intellij.python.community.execService.TtySize +import com.intellij.python.community.execService.UploadConfig import com.intellij.python.community.execService.impl.LoggingProcess import com.jetbrains.python.Result import com.jetbrains.python.TraceContext @@ -61,5 +62,6 @@ internal data class LaunchRequest( val args: Args, val env: Map, val usePty: TtySize?, + val uploadConfig: UploadConfig? = null, val downloadConfig: DownloadConfig? = null, ) diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt index 9a1821e6860c..8f95488179c6 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt @@ -4,9 +4,11 @@ package com.intellij.python.community.execService.impl.processLaunchers import com.intellij.execution.ExecutionException +import com.intellij.execution.Platform import com.intellij.execution.process.LocalPtyOptions import com.intellij.execution.target.FullPathOnTarget import com.intellij.execution.target.TargetEnvironment +import com.intellij.execution.target.TargetEnvironmentRequest import com.intellij.execution.target.TargetProgressIndicator import com.intellij.execution.target.TargetedCommandLine import com.intellij.execution.target.TargetedCommandLineBuilder @@ -22,6 +24,7 @@ import com.intellij.platform.eel.impl.base.bindProcessToScopeImpl import com.intellij.python.community.execService.BinOnTarget import com.intellij.python.community.execService.DownloadConfig import com.intellij.python.community.execService.ExecuteGetProcessError +import com.intellij.python.community.execService.UploadConfig import com.intellij.python.community.execService.impl.PyExecBundle import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler import com.intellij.remoteServer.util.ServerRuntimeException @@ -59,21 +62,21 @@ internal suspend fun createProcessLauncherOnTarget( } } - // Broken Targets API can only upload the whole directory + // Targets API maps local roots as directories; callers may still restrict which files are uploaded below. + val workingDir = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() } val dirsToMap = buildSet { addAll(launchRequest.args.localFiles.map { it.parent }) - binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.also { + workingDir?.also { add(it) } } - val uploadRoots = - TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }) + val uploadRoots = TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, workingDir) request.uploadVolumes.addAll(uploadRoots.map { it.value.root }) // Setup download roots if download is requested val downloadConfig = launchRequest.downloadConfig if (downloadConfig != null) { - val localDirsToDownload = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { setOf(it) } ?: emptySet() + val localDirsToDownload = workingDir?.let { setOf(it) } ?: emptySet() val downloadRoots = mapDownloadRoots(request.uploadVolumes, localDirsToDownload) request.downloadVolumes.addAll(downloadRoots) } @@ -89,10 +92,18 @@ internal suspend fun createProcessLauncherOnTarget( fileLogger().warn("Failed to start $target", e) // TODO: i18n return@withContext Result.failure(ExecuteGetProcessError.EnvironmentError(MessageError("Failed to start environment due to ${e.localizedMessage}"))) } + val workingDirOnTarget = workingDir?.let { targetEnv.getTargetPaths(it.pathString).firstOrNull() ?: it.pathString } + val uploadConfig = launchRequest.uploadConfig + if (uploadConfig?.ensureWorkingDirectoryExists == true && workingDirOnTarget != null) { + ensureTargetDirectoryExists(targetEnv, request, workingDirOnTarget).getOr { failure -> + targetEnv.shutdown() + return@withContext failure + } + } for (volume in targetEnv.uploadVolumes.values) { val skipUploading = uploadRoots[volume.localRoot]?.uploadVolumeExplicitly == false - if (! skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers) - volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService") + if (!skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers) + uploadVolume(volume, workingDir, uploadConfig, launchRequest.args.localFiles) } } @@ -105,9 +116,8 @@ internal suspend fun createProcessLauncherOnTarget( // exe path is always fixed (pre-presolved) promise. It can't be obtained directly because of Targets API limitation exePath = commandLineBuilder.exePath.localValue.blockingGet(1000) ?: error("Exe path not set: $binOnTarget is broken") // Map working directory through upload volumes if it's a local path - binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { workingDir -> + if (workingDirOnTarget != null) { // Try to resolve through upload volumes (in case workingDir is a local path that needs mapping) - val workingDirOnTarget = targetEnv.getTargetPaths(workingDir.pathString).firstOrNull() ?: workingDir.pathString commandLineBuilder.setWorkingDirectory(workingDirOnTarget) } launchRequest.usePty?.let { @@ -134,6 +144,74 @@ internal suspend fun createProcessLauncherOnTarget( downloadConfig))) } +private fun uploadVolume( + volume: TargetEnvironment.UploadableVolume, + workingDir: Path?, + uploadConfig: UploadConfig?, + localFiles: List, +) { + if (uploadConfig != null && workingDir != null && volume.localRoot == workingDir) { + val localFileRelativePaths = localFiles + .filter { it.parent == workingDir } + .map { workingDir.relativize(it).pathString } + val pathsToUpload = (uploadConfig.relativePaths + localFileRelativePaths).distinct() + for (path in pathsToUpload) { + volume.uploadMeasureTime(path, TargetProgressIndicator.EMPTY, "execService") + } + } + else { + volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService") + } +} + +private fun ensureTargetDirectoryExists( + targetEnv: TargetEnvironment, + request: TargetEnvironmentRequest, + directory: FullPathOnTarget, +): Result { + when (request.targetPlatform.platform) { + Platform.UNIX -> Unit + Platform.WINDOWS -> { + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.unsupported", directory)) + return Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } + } + + val cmdLine = createMkdirCommandLine(request, directory) + val process = try { + targetEnv.createProcess(cmdLine) + } + catch (e: ExecutionException) { + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.error", directory, e.localizedMessage)) + return Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } + + val exitCode = try { + process.waitFor() + } + catch (_: InterruptedException) { + process.destroyForcibly() + Thread.currentThread().interrupt() + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.interrupted", directory)) + return Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } + + return if (exitCode == 0) { + Result.success(Unit) + } + else { + val error = MessageError(PyExecBundle.message("py.exec.target.working.directory.create.exitCode", directory, exitCode)) + Result.failure(ExecuteGetProcessError.EnvironmentError(error)) + } +} + +private fun createMkdirCommandLine(request: TargetEnvironmentRequest, directory: FullPathOnTarget): TargetedCommandLine { + val commandLineBuilder = TargetedCommandLineBuilder(request) + commandLineBuilder.setExePath("/bin/mkdir") + commandLineBuilder.addParameters("-p", directory) + return commandLineBuilder.build() +} + private class TargetProcessCommands( override val scopeToBind: CoroutineScope, private val exePath: FullPathOnTarget, diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt b/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt index e16c59fd5899..13c425ca77ff 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/implInterfaces.kt @@ -7,4 +7,6 @@ internal sealed interface ExecOptionsBase { val env: Map val processDescription: @Nls String? val tty: TtySize? + val uploadBeforeExecution: UploadConfig? + val downloadAfterExecution: DownloadConfig? } \ No newline at end of file