PY-82666 Fix undetached process cancellation when running packaging tool

Pip packaging tool used legacy process handler, which led to undetached
processes being spawned and even process leaks if they are really long
running. This led to flaky tests that were trying to delete temporary
directories after the test was finished, and at the same time process
was still running and creating __pycache__ directories there. That's
where a DirectoryNotEmptyException was thrown leading to a test failure.

The whole problem was fixed by using a new ExecService instead of a
legacy process handler. Also unused register keys `python.packaging.tool.use.project.location.as.working.dir` and `python.packaging.tool.upload.project` have been removed.


Merge-request: IJ-MR-176080
Merged-by: Alexey Katsman <alexey.katsman@jetbrains.com>

GitOrigin-RevId: 31c02fc8c25abd5941518958e7e2651f3771bad1

python/pluginResources/intellij.python.community.impl.xml

@@ -572,10 +572,6 @@
 
     <!-- Targets API -->
     <registryKey key="enable.conda.on.targets" defaultValue="false" description="Enables Conda configuration on targets."/>
-    <registryKey key="python.packaging.tool.use.project.location.as.working.dir" defaultValue="false"
-                 description="Use project location as a working directory for the packaging tool execution."/>
-    <registryKey key="python.packaging.tool.upload.project" defaultValue="false"
-                 description="Upload project files and use this location as a working directory for the packaging tool execution."/>
 
     <notificationGroup id="Python Debugger" displayType="TOOL_WINDOW" toolWindowId="Debug" bundle="messages.PyBundle"
                        key="debug.notification.group"/>

python/python-exec-service/src/com/intellij/python/community/execService/api.kt

@@ -185,7 +185,7 @@ open class ZeroCodeStdoutParserTransformer<T>(val stdoutParser: (String) -> Resu
  * @property[env] Environment variables to be applied with the process run
  * @property[timeout] Process gets killed after this timeout
  * @property[processDescription] optional description to be displayed to user
- * [tty] Much like [com.intellij.platform.eel.EelExecApi.Pty]
+ * @property[tty] Much like [com.intellij.platform.eel.EelExecApi.Pty]
  */
 data class ExecOptions(
   override val env: Map<String, String> = emptyMap(),

python/src/com/jetbrains/python/PythonHelper.java

@@ -22,13 +22,14 @@ import java.util.stream.Collectors;
 import static com.intellij.python.community.helpersLocator.PythonHelpersLocator.*;
 import static com.intellij.python.community.impl.venv.VenvKt.LEGACY_VIRTUALENV_ZIPAPP_NAME;
 import static com.intellij.python.community.impl.venv.VenvKt.VIRTUALENV_ZIPAPP_NAME;
+import static com.jetbrains.python.packaging.pip.PipPackageManagerEngine.PACKAGING_TOOL_NAME;
 
 public enum PythonHelper implements HelperPackage {
   GENERATOR3("generator3/__main__.py"),
   REMOTE_SYNC("remote_sync.py"),
 
   // Packaging tools
-  PACKAGING_TOOL("packaging_tool.py"),
+  PACKAGING_TOOL(PACKAGING_TOOL_NAME),
   VIRTUALENV_ZIPAPP(VIRTUALENV_ZIPAPP_NAME),
   LEGACY_VIRTUALENV_ZIPAPP(LEGACY_VIRTUALENV_ZIPAPP_NAME), // virtualenv used to create virtual environments for python 2.7, 3.6, 3.7
 

python/src/com/jetbrains/python/packaging/management/PyPackageProcessHandler.kt

@@ -1,41 +0,0 @@
-// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.packaging.management
-
-import com.intellij.execution.process.CapturingAnsiEscapesAwareProcessHandler
-import com.intellij.execution.process.CapturingProcessAdapter
-import com.intellij.execution.process.ProcessOutput
-import com.intellij.execution.process.ProcessOutputTypes
-import com.intellij.openapi.util.Key
-
-internal class PyPackageProcessHandler : CapturingAnsiEscapesAwareProcessHandler {
-  constructor(process: Process, commandLine: String) : super(process, commandLine)
-
-  var lastLine: String = ""
-    private set
-
-  var lastLineNotifier: ((String) -> Unit)? = null
-
-  override fun createProcessAdapter(processOutput: ProcessOutput): CapturingProcessAdapter? {
-    return PackageProcessAdapter(processOutput)
-  }
-
-  private inner class PackageProcessAdapter(processOutput: ProcessOutput) : AnsiEscapesAwareAdapter(processOutput) {
-    override fun addToOutput(text: String, outputType: Key<*>) {
-      super.addToOutput(text, outputType)
-
-      when {
-        outputType === ProcessOutputTypes.STDOUT -> {
-          lastLine = output.stdoutLines.lastOrNull { it.isNotBlank() } ?: ""
-
-        }
-        outputType === ProcessOutputTypes.STDERR -> {
-          lastLine = output.stderrLines.lastOrNull { it.isNotBlank() } ?: ""
-        }
-      }
-      lastLine = lastLine.trim()
-      if (lastLine.isBlank())
-        return
-      lastLineNotifier?.invoke(lastLine)
-    }
-  }
-}

python/src/com/jetbrains/python/packaging/management/PythonPackageManagerRunner.kt

@@ -1,32 +0,0 @@
-// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.packaging.management
-
-import com.intellij.execution.process.ProcessOutput
-import com.intellij.openapi.progress.ProgressManager
-import com.intellij.openapi.progress.coroutineToIndicator
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.withContext
-import org.jetbrains.annotations.ApiStatus
-
-@ApiStatus.Internal
-internal object PythonPackageManagerRunner {
-  suspend fun runProcess(
-    process: Process,
-    command: String,
-  ): ProcessOutput {
-    val handler = PyPackageProcessHandler(process, command)
-
-    return runProcessInternal(handler)
-  }
-
-  private suspend fun runProcessInternal(processHandler: PyPackageProcessHandler): ProcessOutput = withContext(Dispatchers.IO) {
-    coroutineToIndicator {
-      val progressIndicator = ProgressManager.getInstance().progressIndicator
-      processHandler.lastLineNotifier = { line: String ->
-        @Suppress("HardCodedStringLiteral")
-        progressIndicator.text = line.trim()
-      }
-      processHandler.runProcessWithProgressIndicator(progressIndicator, 10 * 60 * 1000)
-    }
-  }
-}

python/src/com/jetbrains/python/packaging/pip/PipPackageManagerEngine.kt

@@ -1,23 +1,14 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.jetbrains.python.packaging.pip
 
-import com.intellij.execution.target.TargetProgressIndicator
-import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
-import com.intellij.execution.target.value.targetPath
 import com.intellij.openapi.diagnostic.thisLogger
-import com.intellij.openapi.progress.EmptyProgressIndicator
-import com.intellij.openapi.progress.ProgressManager
 import com.intellij.openapi.project.Project
-import com.intellij.openapi.project.guessProjectDir
 import com.intellij.openapi.projectRoots.Sdk
-import com.intellij.openapi.util.registry.Registry
 import com.intellij.openapi.vfs.VirtualFile
-import com.intellij.util.io.IdeUtilIoBundle
-import com.jetbrains.python.sdk.impl.PySdkBundle
-import com.jetbrains.python.PythonHelper
+import com.intellij.python.community.execService.ExecService
+import com.intellij.python.community.execService.python.HelperName
 import com.jetbrains.python.errorProcessing.PyResult
 import com.jetbrains.python.onFailure
-import com.jetbrains.python.packaging.PyExecutionException
 import com.jetbrains.python.packaging.PyPIPackageUtil
 import com.jetbrains.python.packaging.common.PythonOutdatedPackage
 import com.jetbrains.python.packaging.common.PythonPackage
@@ -25,16 +16,11 @@ import com.jetbrains.python.packaging.common.PythonRepositoryPackageSpecificatio
 import com.jetbrains.python.packaging.management.PythonPackageInstallRequest
 import com.jetbrains.python.packaging.management.PythonPackageManager
 import com.jetbrains.python.packaging.management.PythonPackageManagerEngine
-import com.jetbrains.python.packaging.management.PythonPackageManagerRunner
 import com.jetbrains.python.packaging.utils.PyProxyUtils
-import com.jetbrains.python.run.PythonInterpreterTargetEnvironmentFactory
-import com.jetbrains.python.run.buildTargetedCommandLine
-import com.jetbrains.python.run.ensureProjectSdkAndModuleDirsAreOnTarget
-import com.jetbrains.python.run.prepareHelperScriptExecution
+import com.jetbrains.python.sdk.executeHelper
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import org.jetbrains.annotations.ApiStatus
-import kotlin.math.min
 
 
 @ApiStatus.Internal
@@ -95,91 +81,22 @@ class PipPackageManagerEngine(
     return PyResult.success(packages)
   }
 
-  @ApiStatus.Internal
   suspend fun runPackagingTool(operation: String, arguments: List<String>): PyResult<String> = withContext(Dispatchers.IO) {
-    // todo[akniazev]: check for package management tools
-    val helpersAwareTargetRequest = PythonInterpreterTargetEnvironmentFactory.findPythonTargetInterpreter(sdk, project)
-    val targetEnvironmentRequest = helpersAwareTargetRequest.targetEnvironmentRequest
-    val pythonExecution = prepareHelperScriptExecution(PythonHelper.PACKAGING_TOOL, helpersAwareTargetRequest)
-
-    if (targetEnvironmentRequest is LocalTargetEnvironmentRequest) {
-      if (Registry.`is`("python.packaging.tool.use.project.location.as.working.dir")) {
-        project.guessProjectDir()?.toNioPath()?.let {
-          pythonExecution.workingDir = targetPath(it)
-        }
-      }
-    }
-    else {
-      if (Registry.`is`("python.packaging.tool.upload.project")) {
-        project.guessProjectDir()?.toNioPath()?.let {
-          targetEnvironmentRequest.ensureProjectSdkAndModuleDirsAreOnTarget(project)
-          pythonExecution.workingDir = targetPath(it)
-        }
-      }
-    }
-
-    pythonExecution.addParameter(operation)
+    val parameters = mutableListOf(operation)
     if (operation == "install") {
       PyProxyUtils.proxyString?.let {
-        pythonExecution.addParameter("--proxy")
-        pythonExecution.addParameter(it)
+        parameters += "--proxy"
+        parameters += it
       }
     }
-
-    arguments.forEach(pythonExecution::addParameter)
-
-    // // todo[akniazev]: add extra args to package specification
-
-    val targetProgressIndicator = TargetProgressIndicator.EMPTY
-    val targetEnvironment = targetEnvironmentRequest.prepareEnvironment(targetProgressIndicator)
-
-    targetEnvironment.uploadVolumes.entries.forEach { (_, value) ->
-      value.upload(".", targetProgressIndicator)
-    }
-
-    val targetedCommandLine = pythonExecution.buildTargetedCommandLine(targetEnvironment, sdk, emptyList())
-
-    val indicator = ProgressManager.getInstance().progressIndicator ?: EmptyProgressIndicator()
-    // from targets package manager
-    // TODO [targets] Apply environment variables: setPythonUnbuffered(...), setPythonDontWriteBytecode(...), resetHomePathChanges(...)
-    // TODO [targets] Apply flavor from PythonSdkFlavor.getFlavor(mySdk)
-    // TODO [targets] check askForSudo
-
-    val process = targetEnvironment.createProcess(targetedCommandLine, indicator)
-
-    val commandLine = targetedCommandLine.collectCommandsSynchronously()
-    val commandLineString = commandLine.joinToString(" ")
+    parameters += arguments
 
     thisLogger().debug("Running python packaging tool. Operation: $operation")
-
-    val result = PythonPackageManagerRunner.runProcess(
-      process,
-      commandLineString
+    ExecService().executeHelper(
+      sdk,
+      PACKAGING_TOOL_NAME,
+      parameters,
     )
-    if (result.isCancelled) {
-      return@withContext PyResult.localizedError(IdeUtilIoBundle.message("run.canceled.by.user.message"))
-    }
-
-    result.checkSuccess(thisLogger())
-    val exitCode = result.exitCode
-    val helperPath = commandLine.firstOrNull() ?: ""
-    val args: List<String> = commandLine.subList(min(1, commandLine.size), commandLine.size)
-    if (exitCode != 0) {
-      val message = if (result.stdout.isBlank() && result.stderr.isBlank()) PySdkBundle.message(
-        "python.conda.permission.denied")
-      else PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode)
-      PyExecutionException(message, commandLine[0], args, result).let {
-        return@withContext PyResult.failure(it.pyError)
-      }
-    }
-
-    if (result.isTimeout) {
-      PyExecutionException.createForTimeout(PySdkBundle.message("python.sdk.packaging.timed.out"), helperPath, args).let {
-        return@withContext PyResult.failure(it.pyError)
-      }
-    }
-
-    return@withContext PyResult.success(result.stdout)
   }
 
 
@@ -235,4 +152,8 @@ class PipPackageManagerEngine(
 
     return PyResult.success(Unit)
   }
-}
+
+  companion object {
+    const val PACKAGING_TOOL_NAME: HelperName = "packaging_tool.py"
+  }
+}