diff --git a/python/python-exec-service/BUILD.bazel b/python/python-exec-service/BUILD.bazel index 25a35dc4cb3f..a5276fe428e0 100644 --- a/python/python-exec-service/BUILD.bazel +++ b/python/python-exec-service/BUILD.bazel @@ -44,6 +44,7 @@ jvm_library( "//libraries/guava", "//python/python-process-output/common", "//python/impl.helperLocator:community-helpersLocator", + "//platform/execution-impl", ] ) @@ -82,6 +83,7 @@ jvm_library( "//libraries/guava", "//python/python-process-output/common", "//python/impl.helperLocator:community-helpersLocator", + "//platform/execution-impl", ] ) ### auto-generated section `build intellij.python.community.execService` end diff --git a/python/python-exec-service/intellij.python.community.execService.iml b/python/python-exec-service/intellij.python.community.execService.iml index 67f480362709..b971cdc87217 100644 --- a/python/python-exec-service/intellij.python.community.execService.iml +++ b/python/python-exec-service/intellij.python.community.execService.iml @@ -55,5 +55,6 @@ + \ No newline at end of file diff --git a/python/python-exec-service/resources/intellij.python.community.execService.xml b/python/python-exec-service/resources/intellij.python.community.execService.xml index 788cb36131c8..4d0ab32f02e2 100644 --- a/python/python-exec-service/resources/intellij.python.community.execService.xml +++ b/python/python-exec-service/resources/intellij.python.community.execService.xml @@ -1,24 +1,18 @@ - + + + - - - - - (private val reqClass: Class) { + + private suspend fun mapUploadRootsIfValid( + request: TargetEnvironmentRequest, + localDirs: Set, + workingDirToDownload: Path?, + ): UploadInfo? = + if (reqClass.isInstance(request)) { + @Suppress("UNCHECKED_CAST") // Checked one line above + mapUploadRootsImpl(request as T, localDirs, workingDirToDownload) + } + else { + null + } + + /** + * See [mapUploadRoots] + */ + protected abstract suspend fun mapUploadRootsImpl( + request: T, + localDirs: Set, + workingDirToDownload: Path?, + ): UploadInfo + + + internal companion object { + private val EP_NAME: ExtensionPointName> = ExtensionPointName.create( + "Pythonid.execService.targetEnvironmentRequestHandler" + ) + + data class UploadRootWithExplicitUploadInfo( + /** + * Root to add to upload volumes + */ + val root: TargetEnvironment.UploadRoot, + /** + * If true, call [com.intellij.python.community.execService.impl.processLaunchers.measureUploadTime] + */ + val uploadVolumeExplicitly: Boolean, + ) + + /** + * for [request] that needs access to [localDirs] (with [workingDirToDownload] is a working directory) returns + * mapping info for each [localDirs]. This info is used to fill upload volumes, and possibly upload them. + */ + suspend fun mapUploadRoots( + request: TargetEnvironmentRequest, + localDirs: Set, + workingDirToDownload: Path?, + ): Map = withContext(Dispatchers.IO) { + val uploadInfo = EP_NAME.extensionList.firstNotNullOfOrNull { it.mapUploadRootsIfValid(request, localDirs, workingDirToDownload) } + ?: error("No implementation of [${TargetEnvironmentRequestHandler::class.java}] is found for $request, broken bundle? " + + "If you are in tests, set `@TestApplicationWithEel(useLegacyTargets=true)`") + val localToRemoteHelpersRoots = + uploadInfo.helpersAware.preparePyCharmHelpers().helpers.associate { it.localPath to it.targetPathFun.value } + + fun localPathToTarget(localPath: Path): LocalPathToTargetResult { + val mightBeHelperRemoteRoot = localToRemoteHelpersRoots.entries.firstOrNull { localPath.startsWith(it.key) }?.value + return if (mightBeHelperRemoteRoot != null) { + // This is helper. They should never be uploaded explicitly, as handlers (inheritors) take care of them. + LocalPathToTargetResult( + targetPath = TargetEnvironment.TargetPath.Persistent(mightBeHelperRemoteRoot), + removeAndShutDown = false, + uploadVolumeExplicitly = false + ) + } + else { + // This path was explicitly marked as persistent by inheritor + val explicitlyPersistent = uploadInfo.explicitlyPersistent[localPath] + if (explicitlyPersistent != null) { + LocalPathToTargetResult( + targetPath = explicitlyPersistent, + removeAndShutDown = true, + uploadVolumeExplicitly = false + ) + } + else { + // Just a random temp path, but we try to preserve location of workDir (most probably projDir) is set. + LocalPathToTargetResult( + targetPath = TargetEnvironment.TargetPath.Temporary(hint = workingDirToDownload?.pathString), + removeAndShutDown = true, + uploadVolumeExplicitly = true + ) + } + } + } + localDirs.associateWith { localPath -> + val localPathToTargetRes = localPathToTarget(localPath) + val root = TargetEnvironment.UploadRoot( + localRootPath = localPath, + targetRootPath = localPathToTargetRes.targetPath, + removeAtShutdown = localPathToTargetRes.removeAndShutDown + ) + UploadRootWithExplicitUploadInfo(root, uploadVolumeExplicitly = localPathToTargetRes.uploadVolumeExplicitly) + } + } + } + + protected data class UploadInfo( + /** + * Each inheritor provides it so we can access the helpers. + */ + val helpersAware: HelpersAwareTargetEnvironmentRequest, + /** + * These paths will be converted to persistent volumes (temporary otherwise) + */ + val explicitlyPersistent: Map = emptyMap(), + ) + + +} + +private data class LocalPathToTargetResult( + val targetPath: TargetEnvironment.TargetPath, + val removeAndShutDown: Boolean, + val uploadVolumeExplicitly: Boolean, +) \ No newline at end of file diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/DefaultTargetEnvironmentRequestHandler.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/DefaultTargetEnvironmentRequestHandler.kt deleted file mode 100644 index 80ca01b96901..000000000000 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/DefaultTargetEnvironmentRequestHandler.kt +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.intellij.python.community.execService.impl.processLaunchers - -import com.intellij.execution.target.TargetEnvironment -import com.intellij.execution.target.TargetEnvironmentRequest -import com.intellij.python.community.execService.spi.TargetEnvironmentRequestHandler -import java.nio.file.Path - -class DefaultTargetEnvironmentRequestHandler : TargetEnvironmentRequestHandler { - override fun mapUploadRoots( - request: TargetEnvironmentRequest, - localDirs: Set, - workingDirToDownload: Path?, - ): List { - val result = localDirs.map { localDir -> - TargetEnvironment.UploadRoot( - localRootPath = localDir, - targetRootPath = TargetEnvironment.TargetPath.Temporary(), - removeAtShutdown = true - ) - } - return result - } - - override fun isApplicable(request: TargetEnvironmentRequest): Boolean = true -} \ No newline at end of file diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt index 0e86b7d8a3d1..f82a42a0dd8a 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/impl/processLaunchers/targets.kt @@ -11,19 +11,21 @@ import com.intellij.execution.target.TargetProgressIndicator import com.intellij.execution.target.TargetedCommandLine import com.intellij.execution.target.TargetedCommandLineBuilder import com.intellij.execution.target.getTargetPaths -import com.intellij.execution.target.local.LocalTargetEnvironmentRequest import com.intellij.execution.target.local.LocalTargetPtyOptions +import com.intellij.openapi.diagnostic.debug import com.intellij.openapi.diagnostic.fileLogger import com.intellij.openapi.progress.coroutineToIndicator import com.intellij.openapi.project.ProjectManager +import com.intellij.openapi.util.NlsSafe import com.intellij.platform.eel.provider.utils.ProcessFunctions import com.intellij.platform.eel.provider.utils.bindProcessToScopeImpl import com.intellij.python.community.execService.BinOnTarget import com.intellij.python.community.execService.DownloadConfig import com.intellij.python.community.execService.ExecuteGetProcessError import com.intellij.python.community.execService.impl.PyExecBundle -import com.intellij.python.community.execService.spi.TargetEnvironmentRequestHandler +import com.intellij.python.community.execService.impl.TargetEnvironmentRequestHandler import com.intellij.remoteServer.util.ServerRuntimeException +import com.intellij.util.concurrency.annotations.RequiresBackgroundThread import com.jetbrains.python.Result import com.jetbrains.python.errorProcessing.Exe import com.jetbrains.python.errorProcessing.ExecErrorReason @@ -32,9 +34,12 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext +import org.jetbrains.annotations.ApiStatus import java.io.IOException +import java.nio.file.Path import kotlin.io.path.pathString import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.measureTime private val logger = fileLogger() @@ -44,7 +49,7 @@ internal suspend fun createProcessLauncherOnTarget( ): Result = withContext(Dispatchers.IO) { val target = binOnTarget.target - val request = if (target != null) { + val request = run { val projectMan = ProjectManager.getInstance() // Broken Targets API doesn't work without project try { target.createEnvironmentRequest(projectMan.openProjects.firstOrNull() ?: projectMan.defaultProject) @@ -53,7 +58,6 @@ internal suspend fun createProcessLauncherOnTarget( return@withContext Result.failure(ExecuteGetProcessError.EnvironmentError(MessageError(e.localizedMessage))) } } - else LocalTargetEnvironmentRequest() // Broken Targets API can only upload the whole directory val dirsToMap = buildSet { @@ -62,15 +66,15 @@ internal suspend fun createProcessLauncherOnTarget( add(it) } } - val handler = TargetEnvironmentRequestHandler.getHandler(request) - val uploadRoots = handler.mapUploadRoots(request, dirsToMap, binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }) - request.uploadVolumes.addAll(uploadRoots) + val uploadRoots = + TargetEnvironmentRequestHandler.mapUploadRoots(request, dirsToMap, binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }) + request.uploadVolumes.addAll(uploadRoots.map { it.value.root }) // Setup download roots if download is requested val downloadConfig = launchRequest.downloadConfig if (downloadConfig != null) { val localDirsToDownload = binOnTarget.workingDir?.takeIf { it.pathString.isNotBlank() }?.let { setOf(it) } ?: emptySet() - val downloadRoots = handler.mapDownloadRoots(request, request.uploadVolumes, localDirsToDownload) + val downloadRoots = mapDownloadRoots(request.uploadVolumes, localDirsToDownload) request.downloadVolumes.addAll(downloadRoots) } @@ -85,8 +89,11 @@ internal suspend fun createProcessLauncherOnTarget( fileLogger().warn("Failed to start $target", e) // TODO: i18n return@withContext Result.failure(ExecuteGetProcessError.EnvironmentError(MessageError("Failed to start environment due to ${e.localizedMessage}"))) } - targetEnv.uploadVolumes.forEach { _, volume -> - volume.upload(".", TargetProgressIndicator.EMPTY) + for (volume in targetEnv.uploadVolumes.values) { + val skipUploading = uploadRoots[volume.localRoot]?.uploadVolumeExplicitly == false + if (! skipUploading) { // Volume explicitly marked as non-uploadable, i.e.: helpers (they are uploaded by handlers) + volume.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "execService") + } } val args = launchRequest.args.getArgs { localFile -> @@ -192,3 +199,60 @@ private class TargetProcessCommands( private fun ExecutionException.asCantStart(): Result.Failure = Result.failure(ExecErrorReason.CantStart(null, localizedMessage)) + +/** + * Temporary function to [upload] from [localRoot] + [relativePath] to [targetRoot], much like [upload] + * but also reports time using [logger] if `debug` enabled. + */ +@ApiStatus.Internal +@Throws(IOException::class) +@RequiresBackgroundThread +fun TargetEnvironment.UploadableVolume.uploadMeasureTime( + relativePath: String, + targetProgressIndicator: TargetProgressIndicator, + @NlsSafe reason: String, +) { + measureUploadTime( + upload = { + upload(relativePath, targetProgressIndicator) + }, + genMessage = { + "$reason: ${this.localRoot} -> ${targetRoot}" + } + ) +} + +/** + * Measures time for [upload] and reports it along with [genMessage] if `debug` enabled using [logger] + */ +@ApiStatus.Internal +@RequiresBackgroundThread +fun measureUploadTime(@RequiresBackgroundThread upload: () -> Unit, genMessage: () -> @NlsSafe String) { + val duration = measureTime { upload() } + logger.debug { "upload ${genMessage()} : $duration" } +} + +/** + * Maps download roots using existing upload roots. + * This allows downloading files modified on the target back to the local machine. + * + * @param uploadRoots set of upload roots + * @param localDirs local directories that were uploaded and may need to be downloaded + * @return list of download roots, empty by default + */ +private fun mapDownloadRoots( + uploadRoots: Set, + localDirs: Set, +): List = localDirs.mapNotNull { localDir -> + val matchingUpload = uploadRoots.find { localDir.startsWith(it.localRootPath) } + + if (matchingUpload != null) { + TargetEnvironment.DownloadRoot( + localRootPath = localDir, + targetRootPath = matchingUpload.targetRootPath, + ) + } + else { + null // No matching upload, skip download + } +} \ No newline at end of file diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/spi/TargetEnvironmentRequestHandler.kt b/python/python-exec-service/src/com/intellij/python/community/execService/spi/TargetEnvironmentRequestHandler.kt deleted file mode 100644 index 8939a96c41bd..000000000000 --- a/python/python-exec-service/src/com/intellij/python/community/execService/spi/TargetEnvironmentRequestHandler.kt +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -package com.intellij.python.community.execService.spi - -import com.intellij.execution.target.TargetEnvironment -import com.intellij.execution.target.TargetEnvironmentRequest -import com.intellij.openapi.extensions.ExtensionPointName -import org.jetbrains.annotations.ApiStatus -import java.nio.file.Path - -@ApiStatus.Internal -interface TargetEnvironmentRequestHandler { - - fun isApplicable(request: TargetEnvironmentRequest): Boolean - - fun mapUploadRoots( - request: TargetEnvironmentRequest, - localDirs: Set, - workingDirToDownload: Path? - ): List - - /** - * Maps download roots using existing upload roots. - * This allows downloading files modified on the target back to the local machine. - * - * @param request the target environment request - * @param uploadRoots set of upload roots - * @param localDirs local directories that were uploaded and may need to be downloaded - * @return list of download roots, empty by default - */ - fun mapDownloadRoots( - request: TargetEnvironmentRequest, - uploadRoots: Set, - localDirs: Set, - ): List = localDirs.mapNotNull { localDir -> - val matchingUpload = uploadRoots.find { localDir.startsWith(it.localRootPath) } - - if (matchingUpload != null) { - TargetEnvironment.DownloadRoot( - localRootPath = localDir, - targetRootPath = matchingUpload.targetRootPath, - ) - } - else { - null // No matching upload, skip download - } - } - - companion object { - @JvmField - val EP_NAME: ExtensionPointName = ExtensionPointName.create( - "Pythonid.execService.targetEnvironmentRequestHandler" - ) - - @JvmStatic - @ApiStatus.Internal - fun getHandler(request: TargetEnvironmentRequest): TargetEnvironmentRequestHandler { - val handler = EP_NAME.extensionList.firstOrNull { it.isApplicable(request) } - ?: error("No implementation of [${TargetEnvironmentRequestHandler::class.java}] is found for $request, bundle is broken?") - return handler - } - } -} diff --git a/python/src/com/jetbrains/python/run/target/HelpersAwareLocalTargetEnvironmentRequest.kt b/python/src/com/jetbrains/python/run/target/HelpersAwareLocalTargetEnvironmentRequest.kt index 0d07b7a8309a..8fb4de945c30 100644 --- a/python/src/com/jetbrains/python/run/target/HelpersAwareLocalTargetEnvironmentRequest.kt +++ b/python/src/com/jetbrains/python/run/target/HelpersAwareLocalTargetEnvironmentRequest.kt @@ -4,10 +4,12 @@ package com.jetbrains.python.run.target import com.intellij.execution.target.TargetEnvironmentRequest import com.intellij.execution.target.local.LocalTargetEnvironmentRequest import com.intellij.execution.target.value.constantExplicit +import org.jetbrains.annotations.ApiStatus import java.nio.file.Path import kotlin.io.path.absolutePathString -class HelpersAwareLocalTargetEnvironmentRequest : HelpersAwareTargetEnvironmentRequest { +@ApiStatus.Internal +internal class HelpersAwareLocalTargetEnvironmentRequest : HelpersAwareTargetEnvironmentRequest { override val targetEnvironmentRequest: TargetEnvironmentRequest = LocalTargetEnvironmentRequest() override fun preparePyCharmHelpers(): PythonHelpersMappings = diff --git a/python/src/com/jetbrains/python/sdk/PyTargetsRemoteSourcesRefresher.kt b/python/src/com/jetbrains/python/sdk/PyTargetsRemoteSourcesRefresher.kt index d8ebd40656a7..a2192de055a3 100644 --- a/python/src/com/jetbrains/python/sdk/PyTargetsRemoteSourcesRefresher.kt +++ b/python/src/com/jetbrains/python/sdk/PyTargetsRemoteSourcesRefresher.kt @@ -21,6 +21,7 @@ import com.intellij.openapi.project.rootManager import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.util.Disposer import com.intellij.openapi.vfs.LocalFileSystem +import com.intellij.python.community.execService.impl.processLaunchers.uploadMeasureTime import com.intellij.remote.RemoteSdkProperties import com.intellij.util.PathMappingSettings import com.intellij.util.PathUtil @@ -108,7 +109,7 @@ class PyTargetsRemoteSourcesRefresher(val sdk: Sdk, private val project: Project val environment = targetEnvRequest.prepareEnvironment(targetIndicator) try { // XXX Make it automatic - environment.uploadVolumes.values.forEach { it.upload(".", targetIndicator) } + environment.uploadVolumes.values.forEach { it.uploadMeasureTime(".", targetIndicator, "sourceRefresher") } val cmd = execution.buildTargetedCommandLine(environment, sdk, emptyList()) cmd.execute(environment, indicator) diff --git a/python/src/com/jetbrains/python/sdk/skeletons/PyTargetsSkeletonGenerator.kt b/python/src/com/jetbrains/python/sdk/skeletons/PyTargetsSkeletonGenerator.kt index d92bb4650cfa..a5b152266d71 100644 --- a/python/src/com/jetbrains/python/sdk/skeletons/PyTargetsSkeletonGenerator.kt +++ b/python/src/com/jetbrains/python/sdk/skeletons/PyTargetsSkeletonGenerator.kt @@ -18,6 +18,7 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.project.ProjectManager import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.util.io.FileUtil +import com.intellij.python.community.execService.impl.processLaunchers.uploadMeasureTime import com.jetbrains.python.PythonHelper import com.jetbrains.python.run.PythonInterpreterTargetEnvironmentFactory import com.jetbrains.python.run.buildTargetedCommandLine @@ -122,7 +123,7 @@ class PyTargetsSkeletonGenerator(skeletonPath: String, pySdk: Sdk, currentFolder try { // XXX Make it automatic - targetEnvironment.uploadVolumes.values.forEach { it.upload(".", TargetProgressIndicator.EMPTY) } + targetEnvironment.uploadVolumes.values.forEach { it.uploadMeasureTime(".", TargetProgressIndicator.EMPTY, "skeleton") } val targetedCommandLine = generatorScriptExecution.buildTargetedCommandLine(targetEnvironment, sdk, emptyList()) val process = targetEnvironment.createProcess(targetedCommandLine, EmptyProgressIndicator())