Python: separate advancedApi.kt (low-level) from simple (api.kt) in ExecService.

Consider using `api.kt` whenever possible.

GitOrigin-RevId: fc08d3ffdcce1b68cfb568c9a4c94381c1f58042

python/python-exec-service/src/com/intellij/python/community/execService/advancedApi.kt

@@ -6,9 +6,39 @@ import com.intellij.platform.eel.channels.EelSendChannel
 import com.intellij.platform.eel.provider.utils.EelProcessExecutionResult
 import com.intellij.python.community.execService.impl.ProcessSemiInteractiveHandlerImpl
 import com.jetbrains.python.Result
+import com.jetbrains.python.errorProcessing.PyExecResult
 import kotlinx.coroutines.Deferred
+import org.jetbrains.annotations.ApiStatus
+import org.jetbrains.annotations.CheckReturnValue
 import org.jetbrains.annotations.Nls
-import java.io.IOException
+
+
+// This is an advanced API, consider using basic api.kt
+
+/**
+ * Service is a thin wrapper over [com.intellij.platform.eel.EelApi] to execute python tools on local or remote Eel.
+ * to obtain service, use function with same name.
+ *
+ * For all APIs but full interactive mode (which is a very low-level custom mode) stdout/stderr is reported as progress.
+ */
+@ApiStatus.Internal
+interface ExecService {
+
+  /**
+   * Execute code in a so-called "interactive" mode.
+   * This is a quite advanced mode where *you* are responsible for converting a process to output.
+   * You must listen for process stdout/stderr e.t.c.
+   * Use it if you need to get some info from a process before it ends or to interact (i.e. write into stdin).
+   * See [ProcessInteractiveHandler] and [processSemiInteractiveHandler]
+   */
+  @CheckReturnValue
+  suspend fun <T> execute(
+    whatToExec: WhatToExec,
+    args: List<String> = emptyList(),
+    options: ExecOptions = ExecOptions(),
+    processInteractiveHandler: ProcessInteractiveHandler<T>,
+  ): PyExecResult<T>
+}
 
 
 /**
@@ -33,11 +63,11 @@ fun interface ProcessInteractiveHandler<T> {
 /**
  * Process stdout -> result
  */
-typealias ProcessSemiInteractiveFun<T> = suspend (EelSendChannel, Deferred<Int>) -> Result<T, CustomErrorMessage?>
+typealias ProcessSemiInteractiveFun<T> = suspend (EelSendChannel, Deferred<EelProcessExecutionResult>) -> Result<T, CustomErrorMessage?>
 
 /**
  * [ProcessInteractiveHandler], but you do not have to collect output by yourself. You only have access to stdout and exit code.
  * Function collects output lines and reports them to [pyProcessListener] if set
  * So, you can only *write* something to process.
  */
-fun <T> processSemiInteractiveHandler(pyProcessListener: PyProcessListener? = null, code: ProcessSemiInteractiveFun<T>): ProcessInteractiveHandler<T> = ProcessSemiInteractiveHandlerImpl(pyProcessListener, code)
+fun <T> processSemiInteractiveHandler(pyProcessListener: PyProcessListener? = null, code: ProcessSemiInteractiveFun<T>): ProcessInteractiveHandler<T> = ProcessSemiInteractiveHandlerImpl(pyProcessListener, code)

python/python-exec-service/src/com/intellij/python/community/execService/api.kt

@@ -14,15 +14,99 @@ import com.jetbrains.python.Result
 import com.jetbrains.python.errorProcessing.ExecError
 import com.jetbrains.python.errorProcessing.PyExecResult
 import com.jetbrains.python.errorProcessing.PyResult
-import org.jetbrains.annotations.ApiStatus
 import org.jetbrains.annotations.CheckReturnValue
 import org.jetbrains.annotations.Nls
 import java.nio.file.Path
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.minutes
 
+
 /**
- * Error is an optional additionalMessage, that will be used instead of a default one for the [ExecError] in the [com.jetbrains.python.execution.PyExecutionFailure].
+ * Default service implementation
+ */
+fun ExecService(): ExecService = ExecServiceImpl
+
+
+/**
+ * Execute [binary] right directly on the eel it resides on.
+ */
+suspend fun ExecService.execGetStdout(
+  binary: Path,
+  args: List<String> = emptyList(),
+  options: ExecOptions = ExecOptions(),
+  procListener: PyProcessListener? = null,
+): PyExecResult<String> = execGetStdout(WhatToExec.Binary(binary), args, options, procListener)
+
+/**
+ * Execute [binaryName] on [eelApi].
+ * This [binaryName] will be searched in `PATH`
+ */
+suspend fun ExecService.execGetStdout(
+  eelApi: EelApi,
+  binaryName: String,
+  args: List<String> = emptyList(),
+  options: ExecOptions = ExecOptions(),
+  procListener: PyProcessListener? = null,
+): PyResult<String> {
+  val whatToExec = WhatToExec.Binary.fromRelativeName(eelApi, binaryName)
+                   ?: return PyResult.localizedError(PyExecBundle.message("py.exec.fileNotFound", binaryName, eelApi.descriptor.userReadableDescription))
+  return execGetStdout(whatToExec, args, options, procListener)
+}
+
+
+/**
+ * Execute [commandForShell] on [eelApi].
+ * Shell is `cmd` for Windows and Bourne Shell for POSIX.
+ */
+suspend fun ExecService.execGetStdoutInShell(
+  eelApi: EelApi,
+  commandForShell: String,
+  args: List<String> = emptyList(),
+  options: ExecOptions = ExecOptions(),
+  procListener: PyProcessListener? = null,
+): PyExecResult<String> {
+  val (shell, arg) = eelApi.exec.getShell()
+  return execGetStdout(WhatToExec.Binary(shell.asNioPath()), listOf(arg, commandForShell) + args, options, procListener)
+}
+
+/**
+ * Execute [whatToExec] with [args] and get both stdout/stderr outputs if `errorCode != 0`, returns error otherwise.
+ * Function collects output lines and reports them to [procListener] if set
+ *
+ * @param[args] command line arguments
+ * @param[options]  customizable process run options like timeout or environment variables to use
+ * @return stdout or error. It is recommended to put this error into [com.jetbrains.python.errorProcessing.ErrorSink], but feel free to match and process it.
+ */
+@CheckReturnValue
+suspend fun <T> ExecService.execute(
+  whatToExec: WhatToExec,
+  args: List<String> = emptyList(),
+  options: ExecOptions = ExecOptions(),
+  procListener: PyProcessListener? = null,
+  processOutputTransformer: ProcessOutputTransformer<T>,
+): PyExecResult<T> = execute(whatToExec, args, options, processSemiInteractiveHandler(procListener) { _, result ->
+  processOutputTransformer(result.await())
+})
+
+/**
+ * See [ExecService.execute]
+ */
+@CheckReturnValue
+suspend fun ExecService.execGetStdout(
+  whatToExec: WhatToExec,
+  args: List<String> = emptyList(),
+  options: ExecOptions = ExecOptions(),
+  procListener: PyProcessListener? = null,
+): PyExecResult<String> = execute(
+  whatToExec = whatToExec,
+  args = args,
+  options = options,
+  processOutputTransformer = ZeroCodeStdoutTransformer,
+  procListener = procListener
+)
+
+/**
+ * Error is an optional additionalMessage, that will be used instead of a default one for the [ExecError]
  */
 typealias ProcessOutputTransformer<T> = (EelProcessExecutionResult) -> Result<T, @NlsSafe String?>
 
@@ -31,48 +115,6 @@ object ZeroCodeStdoutTransformer : ProcessOutputTransformer<String> {
     if (processOutput.exitCode == 0) Result.success(processOutput.stdoutString.trim()) else Result.failure(null)
 }
 
-/**
- * Service is a thin wrapper over [EelApi] to execute python tools on local or remote Eel.
- * to obtain service, use function with same name.
- *
- * For all APIs but full interactive mode (which is a very low-level custom mode) stdout/stderr is reported as a progress.
- */
-@ApiStatus.Internal
-interface ExecService {
-
-  /**
-   * Execute code in a so-called "interactive" mode.
-   * This is a quite advanced mode where *you* are responsible for converting a process to output.
-   * You must listen for process stdout/stderr e.t.c.
-   * Use it if you need to get some info from a process before it ends or to interact (i.e write into stdin).
-   * See [ProcessInteractiveHandler] and [processSemiInteractiveHandler]
-   */
-  @CheckReturnValue
-  suspend fun <T> executeInteractive(
-    whatToExec: WhatToExec,
-    args: List<String> = emptyList(),
-    options: ExecOptions = ExecOptions(),
-    processInteractiveHandler: ProcessInteractiveHandler<T>,
-  ): PyExecResult<T>
-
-  /**
-   * Execute [whatToExec] with [args] and get both stdout/stderr outputs if `errorCode != 0`, returns error otherwise.
-   * Function collects output lines and reports them to [procListener] if set
-   *
-   * @param[args] command line arguments
-   * @param[options]  customizable process run options like timeout or environment variables to use
-   * @return stdout or error. It is recommended to put this error into [com.jetbrains.python.errorProcessing.ErrorSink], but feel free to match and process it.
-   */
-  @CheckReturnValue
-  suspend fun <T> execute(
-    whatToExec: WhatToExec,
-    args: List<String> = emptyList(),
-    options: ExecOptions = ExecOptions(),
-    procListener: PyProcessListener? = null,
-    processOutputTransformer: ProcessOutputTransformer<T>,
-  ): PyExecResult<T>
-
-}
 
 /**
  * @property[workingDirectory] Directory where to run the process (PWD)
@@ -107,69 +149,3 @@ sealed interface WhatToExec {
    */
   data class Helper(val python: PythonBinary, val helper: HelperName) : WhatToExec
 }
-
-/**
- * Default server implementation
- */
-fun ExecService(): ExecService = ExecServiceImpl
-
-/**
- * See [ExecService.execute]
- */
-@CheckReturnValue
-suspend fun ExecService.execGetStdout(
-  whatToExec: WhatToExec,
-  args: List<String> = emptyList(),
-  options: ExecOptions = ExecOptions(),
-  procListener: PyProcessListener? = null,
-): PyExecResult<String> = execute(
-  whatToExec = whatToExec,
-  args = args,
-  options = options,
-  processOutputTransformer = ZeroCodeStdoutTransformer,
-  procListener = procListener
-)
-
-/**
- * Execute [binaryName] on [eelApi].
- * This [binaryName] will be searched in `PATH`
- */
-@CheckReturnValue
-suspend fun ExecService.execGetStdout(
-  eelApi: EelApi,
-  binaryName: String,
-  args: List<String> = emptyList(),
-  options: ExecOptions = ExecOptions(),
-  procListener: PyProcessListener? = null,
-): PyResult<String> {
-  val whatToExec = WhatToExec.Binary.fromRelativeName(eelApi, binaryName)
-                   ?: return PyResult.localizedError(PyExecBundle.message("py.exec.fileNotFound", binaryName, eelApi.descriptor.userReadableDescription))
-  return execGetStdout(whatToExec, args, options, procListener)
-}
-
-/**
- * Execute [binary] right directly on the eel it resides on.
- */
-@CheckReturnValue
-suspend fun ExecService.execGetStdout(
-  binary: Path,
-  args: List<String> = emptyList(),
-  options: ExecOptions = ExecOptions(),
-  procListener: PyProcessListener? = null,
-): PyExecResult<String> = execGetStdout(WhatToExec.Binary(binary), args, options, procListener)
-
-/**
- * Execute [commandForShell] on [eelApi].
- * Shell is `cmd` for Windows and Bourne Shell for POSIX.
- */
-@CheckReturnValue
-suspend fun ExecService.execGetStdoutInShell(
-  eelApi: EelApi,
-  commandForShell: String,
-  args: List<String> = emptyList(),
-  options: ExecOptions = ExecOptions(),
-  procListener: PyProcessListener? = null,
-): PyExecResult<String> {
-  val (shell, arg) = eelApi.exec.getShell()
-  return execGetStdout(WhatToExec.Binary(shell.asNioPath()), listOf(arg, commandForShell) + args, options, procListener)
-}

python/python-exec-service/src/com/intellij/python/community/execService/impl/ExecServiceImpl.kt

@@ -9,7 +9,10 @@ import com.intellij.platform.eel.provider.asEelPath
 import com.intellij.platform.eel.provider.getEelDescriptor
 import com.intellij.platform.eel.provider.utils.EelPathUtils
 import com.intellij.platform.eel.spawnProcess
-import com.intellij.python.community.execService.*
+import com.intellij.python.community.execService.ExecOptions
+import com.intellij.python.community.execService.ExecService
+import com.intellij.python.community.execService.ProcessInteractiveHandler
+import com.intellij.python.community.execService.WhatToExec
 import com.jetbrains.python.PythonHelpersLocator
 import com.jetbrains.python.Result
 import com.jetbrains.python.errorProcessing.ExecError
@@ -27,7 +30,7 @@ import kotlin.time.Duration
 
 
 internal object ExecServiceImpl : ExecService {
-  override suspend fun <T> executeInteractive(
+  override suspend fun <T> execute(
     whatToExec: WhatToExec,
     args: List<String>,
     options: ExecOptions,
@@ -53,32 +56,6 @@ internal object ExecServiceImpl : ExecService {
 
     return result
   }
-
-  override suspend fun <T> execute(
-    whatToExec: WhatToExec,
-    args: List<String>,
-    options: ExecOptions,
-    procListener: PyProcessListener?,
-    processOutputTransformer: ProcessOutputTransformer<T>,
-  ): PyExecResult<T> {
-    val executableProcess = whatToExec.buildExecutableProcess(args, options)
-    val eelProcess = executableProcess.run().getOr { return it }
-
-    procListener?.emit(ProcessEvent.ProcessStarted(whatToExec, args))
-    val eelProcessExecutionResult = try {
-      withTimeout(options.timeout) { eelProcess.awaitWithReporting(procListener) }
-    }
-    catch (_: TimeoutCancellationException) {
-      return executableProcess.killProcessAndFailAsTimeout(eelProcess, options.timeout)
-    }
-
-    val processOutput = eelProcessExecutionResult
-    procListener?.emit(ProcessEvent.ProcessEnded(eelProcessExecutionResult.exitCode))
-    val transformerSuccess = processOutputTransformer.invoke(processOutput).getOr { failure ->
-      return executableProcess.failAsExecutionFailed(ExecErrorReason.UnexpectedProcessTermination(processOutput), failure.error)
-    }
-    return Result.success(transformerSuccess)
-  }
 }
 
 private data class EelExecutableProcess(

python/python-exec-service/src/com/intellij/python/community/execService/impl/interactiveHandlersImpl.kt

@@ -17,7 +17,7 @@ internal class ProcessSemiInteractiveHandlerImpl<T>(
     coroutineScope {
       pyProcessListener?.emit(ProcessEvent.ProcessStarted(whatToExec, args))
       val processOutput = async { process.awaitWithReporting(pyProcessListener) }
-      val result = code(process.stdin, process.exitCode)
+      val result = code(process.stdin, processOutput)
       pyProcessListener?.emit(ProcessEvent.ProcessEnded(process.exitCode.await()))
       return@coroutineScope result.mapError { customErrorMessage ->
         Pair(processOutput.await(), customErrorMessage)

python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ExecServiceShowCaseTest.kt

@@ -32,26 +32,39 @@ import kotlin.time.Duration.Companion.minutes
  */
 @TestApplicationWithEel(osesMayNotHaveRemoteEels = [OS.WINDOWS, OS.LINUX, OS.MAC])
 class ExecServiceShowCaseTest {
+  enum class SimpleApiExecType { IN_SHELL, RELATIVE, FULL_PATH }
 
   @CartesianTest
   fun testExecSimpleApi(
     @EelSource eelHolder: EelHolder,
     @CartesianTest.Values(booleans = [true, false]) rainyDay: Boolean,
-    @CartesianTest.Values(booleans = [true, false]) useShell: Boolean,
+    @CartesianTest.Enum execType: SimpleApiExecType,
   ): Unit = timeoutRunBlocking(5.minutes) {
     val eel = eelHolder.eel
     val sut = ExecService()
     val hello = "hello"
 
-    val r = if (useShell) {
-      sut.execGetStdoutInShell(eel, if (rainyDay) "abc123" else "echo $hello")
+    val (binary, args) = when (eel.platform) {
+      is EelPlatform.Windows -> Pair("cmd.exe", arrayOf("/C", "echo $hello\r\nexit\r\n"))
+      is EelPlatform.Posix -> Pair("sh", arrayOf("-c", "echo $hello && exit"))
     }
-    else {
-      val (binary, args) = when (eel.platform) {
-        is EelPlatform.Windows -> Pair("cmd.exe", arrayOf("/C", "echo $hello\r\nexit\r\n"))
-        is EelPlatform.Posix -> Pair("sh", arrayOf("-c", "echo $hello && exit"))
+
+    val r = when (execType) {
+      SimpleApiExecType.IN_SHELL -> {
+        sut.execGetStdoutInShell(eel, if (rainyDay) "abc123" else "echo $hello")
+      }
+      SimpleApiExecType.RELATIVE -> {
+        sut.execGetStdout(eel, if (rainyDay) "abc123" else binary, args.toList())
+      }
+      SimpleApiExecType.FULL_PATH -> {
+        var fullPath = eel.exec.findExeFilesInPath(binary).firstOrNull()
+                       ?: error("no $binary found on ${eel.descriptor.userReadableDescription}")
+        if (rainyDay) {
+          fullPath = fullPath.resolve("junk")
+        }
+
+        sut.execGetStdout(fullPath.asNioPath(), args.toList())
       }
-      sut.execGetStdout(eel, if (rainyDay) "abc123" else binary, args.toList())
     }
 
     when (r) {
@@ -59,7 +72,7 @@ class ExecServiceShowCaseTest {
         assertTrue(rainyDay, "unexpected error ${r.error}")
       }
       is Result.Success -> {
-        assertFalse(rainyDay)
+        assertFalse(rainyDay, "unexpected success:${r.result}")
         assertThat("No expected stdout", r.result, CoreMatchers.containsString(hello))
       }
     }
@@ -131,7 +144,7 @@ class ExecServiceShowCaseTest {
   fun testInteractive(eelHolder: EelHolder): Unit = timeoutRunBlocking {
     val string = "abc123"
     val shell = eelHolder.eel.exec.getShell().first
-    val output = ExecService().executeInteractive(WhatToExec.Binary(shell.asNioPath()), emptyList(), processInteractiveHandler = ProcessInteractiveHandler<String> { _, _, process ->
+    val output = ExecService().execute(WhatToExec.Binary(shell.asNioPath()), emptyList(), processInteractiveHandler = ProcessInteractiveHandler<String> { _, _, process ->
       val stdout = async {
         process.stdout.readWholeText()
       }
@@ -149,9 +162,9 @@ class ExecServiceShowCaseTest {
   ): Unit = timeoutRunBlocking {
     val messageToUser = "abc123"
     val shell = eelHolder.eel.exec.getShell().first
-    val result = ExecService().executeInteractive(WhatToExec.Binary(shell.asNioPath()), emptyList(), processInteractiveHandler = processSemiInteractiveHandler<Unit> { channel, exitCode ->
+    val result = ExecService().execute(WhatToExec.Binary(shell.asNioPath()), emptyList(), processInteractiveHandler = processSemiInteractiveHandler<Unit> { channel, exitCode ->
       channel.sendWholeText("exit\n")
-      assertEquals(0, exitCode.await(), "Wrong exit code")
+      assertEquals(0, exitCode.await().exitCode, "Wrong exit code")
       if (sunny) {
         Result.success(Unit)
       }
@@ -218,7 +231,7 @@ class ExecServiceShowCaseTest {
       }
     }
 
-    ExecService().executeInteractive(whatToExec, args = emptyList(), processInteractiveHandler = processSemiInteractiveHandler<Unit>(progressCapturer) { stdin, exitCode ->
+    ExecService().execute(whatToExec, args = emptyList(), processInteractiveHandler = processSemiInteractiveHandler<Unit>(progressCapturer) { stdin, exitCode ->
       for (string in text) {
         stdin.sendWholeText("echo $string\n")
         delay(500)

python/python-hatch/src/com/intellij/python/hatch/runtime/HatchRuntime.kt

@@ -60,7 +60,7 @@ class HatchRuntime(
   }
 
   internal suspend fun <T> executeInteractive(vararg arguments: String, processSemiInteractiveFun: ProcessSemiInteractiveFun<T>): PyExecResult<T> {
-    return execService.executeInteractive(hatchBinary, arguments.toList(), execOptions, processSemiInteractiveHandler(code = processSemiInteractiveFun))
+    return execService.execute(hatchBinary, arguments.toList(), execOptions, processSemiInteractiveHandler(code = processSemiInteractiveFun))
   }
 
   internal suspend fun resolvePythonVirtualEnvironment(pythonHomePath: PythonHomePath): PyResult<PythonVirtualEnvironment> {