From eb336fedc2cfcad7f0ac175e35fe2c5c71be0b54 Mon Sep 17 00:00:00 2001 From: Vitaly Legchilkin Date: Tue, 19 Aug 2025 08:09:48 +0000 Subject: [PATCH] [python] show execution command and outputs for conda runner in case of parsing failures (PY-76274) + ZeroCodeStdoutParserTransformer + ZeroCodeJsonParserTransformer + add transformer showcase Merge-request: IJ-MR-172758 Merged-by: Vitaly Legchilkin GitOrigin-RevId: 1f293e8725a986eaed2a6050a3c6a240441f09fe --- .../src/com/jetbrains/python/Result.kt | 24 ---- python/python-exec-service/BUILD.bazel | 12 ++ .../intellij.python.community.execService.iml | 23 ++++ .../python/community/execService/api.kt | 14 +++ .../community/execService/transformers.kt | 50 +++++++++ .../unit/alsoWin/ExecServiceShowCaseTest.kt | 44 +------- .../ProcessOutputTransformerShowCaseTest.kt | 104 ++++++++++++++++++ .../python/sdk/PySdkCommandRunner.kt | 36 ++++-- .../sdk/conda/execution/CondaExecutor.kt | 68 ++++++++---- 9 files changed, 280 insertions(+), 95 deletions(-) create mode 100644 python/python-exec-service/src/com/intellij/python/community/execService/transformers.kt create mode 100644 python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ProcessOutputTransformerShowCaseTest.kt diff --git a/python/openapi/src/com/jetbrains/python/Result.kt b/python/openapi/src/com/jetbrains/python/Result.kt index c69c4abaa53b..a13bc59a73c5 100644 --- a/python/openapi/src/com/jetbrains/python/Result.kt +++ b/python/openapi/src/com/jetbrains/python/Result.kt @@ -5,10 +5,6 @@ import com.intellij.openapi.diagnostic.Logger import com.jetbrains.python.Result.Failure import com.jetbrains.python.Result.Success import com.jetbrains.python.errorProcessing.MessageError -import com.jetbrains.python.errorProcessing.PyResult -import com.jetbrains.python.packaging.PyExecutionException -import kotlinx.coroutines.CancellationException -import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.Nls /** @@ -114,29 +110,9 @@ sealed class Result { fun success(value: S): Success = Success(value) fun failure(error: E): Failure = Failure(error) fun localizedError(message: @Nls String): Failure = failure(MessageError(message)) - - @ApiStatus.Internal - inline fun runCatching(body: () -> T): PyResult = pyRunCatching(body) } } -@Suppress("UsagesOfObsoleteApi") -@ApiStatus.Internal -inline fun pyRunCatching(body: () -> T): PyResult { - return try { - PyResult.success(body()) - } - catch (t: PyExecutionException) { - PyResult.failure(t.pyError) - } - catch (t: CancellationException) { - throw t - } - catch (t: IllegalArgumentException) { - //Parse deserialization exceptions - PyResult.localizedError(t.localizedMessage) - } -} /** * Maps success result to another one with same error diff --git a/python/python-exec-service/BUILD.bazel b/python/python-exec-service/BUILD.bazel index 6f3383c9c3a8..3d664f65b65b 100644 --- a/python/python-exec-service/BUILD.bazel +++ b/python/python-exec-service/BUILD.bazel @@ -1,6 +1,12 @@ ### auto-generated section `build intellij.python.community.execService` start +load("//build:compiler-options.bzl", "create_kotlinc_options") load("@rules_jvm//:jvm.bzl", "jvm_library", "jvm_resources") +create_kotlinc_options( + name = "custom_community-execService", + opt_in = ["kotlin.time.ExperimentalTime"] +) + jvm_resources( name = "community-execService_resources", files = glob(["resources/**/*"]), @@ -12,6 +18,7 @@ jvm_library( module_name = "intellij.python.community.execService", visibility = ["//visibility:public"], srcs = glob(["src/**/*.kt", "src/**/*.java"], allow_empty = True), + kotlinc_opts = ":custom_community-execService", deps = [ "@lib//:kotlin-stdlib", "@lib//:jetbrains-annotations", @@ -25,6 +32,8 @@ jvm_library( "//platform/util/progress", "//platform/execution", "//platform/projectModel-api:projectModel", + "@lib//:kotlinx-serialization-json", + "@lib//:kotlinx-serialization-core", ], runtime_deps = [":community-execService_resources"] ) @@ -33,6 +42,7 @@ jvm_library( name = "community-execService_test_lib", visibility = ["//visibility:public"], srcs = glob(["tests/**/*.kt", "tests/**/*.java"], allow_empty = True), + kotlinc_opts = ":custom_community-execService", associates = [":community-execService"], deps = [ "@lib//:kotlin-stdlib", @@ -57,6 +67,8 @@ jvm_library( "//platform/util/progress", "//platform/execution", "//platform/projectModel-api:projectModel", + "@lib//:kotlinx-serialization-json", + "@lib//:kotlinx-serialization-core", ], runtime_deps = [":community-execService_resources"] ) diff --git a/python/python-exec-service/intellij.python.community.execService.iml b/python/python-exec-service/intellij.python.community.execService.iml index d3d1e32a76b7..1cd0cabcc56c 100644 --- a/python/python-exec-service/intellij.python.community.execService.iml +++ b/python/python-exec-service/intellij.python.community.execService.iml @@ -1,5 +1,26 @@ + + + + + + + + + + + + + + $KOTLIN_BUNDLED$/lib/kotlinx-serialization-compiler-plugin.jar + + + + + + @@ -28,5 +49,7 @@ + + \ No newline at end of file diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/api.kt b/python/python-exec-service/src/com/intellij/python/community/execService/api.kt index e3bd53697007..2f46b7cd42c5 100644 --- a/python/python-exec-service/src/com/intellij/python/community/execService/api.kt +++ b/python/python-exec-service/src/com/intellij/python/community/execService/api.kt @@ -164,6 +164,20 @@ object ZeroCodeStdoutTransformer : ProcessOutputTransformer { if (processOutput.exitCode == 0) Result.success(processOutput.stdoutString.trim()) else Result.failure(null) } +/** + * A process output transformer that parses standard output using a provided parser function. + * + * @param T The type of the result produced by the transformer. + * @param stdoutParser A function that takes a string (standard output) and parses it into a [Result] containing + * either a successfully parsed result of type [T], or a failure with an optional [NlsSafe] error message. + */ +open class ZeroCodeStdoutParserTransformer(val stdoutParser: (String) -> Result) : ProcessOutputTransformer { + override fun invoke(processOutput: EelProcessExecutionResult): Result { + val data = ZeroCodeStdoutTransformer.invoke(processOutput).getOr { return it } + return stdoutParser(data) + } +} + /** * @property[env] Environment variables to be applied with the process run diff --git a/python/python-exec-service/src/com/intellij/python/community/execService/transformers.kt b/python/python-exec-service/src/com/intellij/python/community/execService/transformers.kt new file mode 100644 index 000000000000..166581fe463c --- /dev/null +++ b/python/python-exec-service/src/com/intellij/python/community/execService/transformers.kt @@ -0,0 +1,50 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.community.execService + +import com.intellij.serialization.SerializationException +import com.jetbrains.python.Result +import kotlinx.serialization.json.Json + + +/** + * A functional interface defining a contract for parsing JSON strings into instances of type [T]. + * + * Potential exceptions: + * - [IllegalArgumentException]: Thrown if the input JSON string is invalid or improper. + * - [SerializationException]: Thrown if there is a failure during the deserialization process. + * + * @param T The type of object expected as the result of JSON parsing. + */ +fun interface JsonParser { + fun parseJson(rawJson: String): T +} + + +/** + * Parses the JSON from the process execution result using the provided parser function. + * [jsonParser] may throw [SerializationException] or [IllegalArgumentException] if output is invalid, + * in this case the parsing exception localized message will be returned. + * + * + * @param jsonParser Function that converts stdout string to type [T]. + * @return A [Result] containing either the successfully parsed [T] object, or a parsing failure message + */ +class ZeroCodeJsonParserTransformer(jsonParser: JsonParser) : ZeroCodeStdoutParserTransformer( + { + try { + Result.success(jsonParser.parseJson(it)) + } + catch (t: SerializationException) { + Result.failure(t.localizedMessage) + } + catch (t: IllegalArgumentException) { + Result.failure(t.localizedMessage) + } + } +) { + companion object { + inline operator fun invoke(json: Json): ZeroCodeJsonParserTransformer { + return ZeroCodeJsonParserTransformer(JsonParser { json.decodeFromString(it) }) + } + } +} \ No newline at end of file diff --git a/python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ExecServiceShowCaseTest.kt b/python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ExecServiceShowCaseTest.kt index bcf2460629cd..fe1e6433ce77 100644 --- a/python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ExecServiceShowCaseTest.kt +++ b/python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ExecServiceShowCaseTest.kt @@ -4,7 +4,10 @@ package com.intellij.python.junit5Tests.unit.alsoWin import com.intellij.platform.eel.EelPlatform import com.intellij.platform.eel.getShell import com.intellij.platform.eel.provider.asNioPath -import com.intellij.platform.eel.provider.utils.* +import com.intellij.platform.eel.provider.utils.asEelChannel +import com.intellij.platform.eel.provider.utils.consumeAsEelChannel +import com.intellij.platform.eel.provider.utils.readWholeText +import com.intellij.platform.eel.provider.utils.sendWholeText import com.intellij.platform.testFramework.junit5.eel.params.api.EelHolder import com.intellij.platform.testFramework.junit5.eel.params.api.EelSource import com.intellij.platform.testFramework.junit5.eel.params.api.TestApplicationWithEel @@ -97,45 +100,6 @@ class ExecServiceShowCaseTest { } } - - @ParameterizedTest - @EelSource - fun testDataTransformer(eelHolder: EelHolder): Unit = timeoutRunBlocking { - val eel = eelHolder.eel - - data class Record(val name: String, val age: Int) - - val (shell, execArg) = eel.exec.getShell() - val args = Args(execArg, "echo Alice,25 && echo Bob,48") - - val records = ExecService().execute((BinOnEel(shell.asNioPath())), args) { output -> - val stdout = output.stdoutString.trim() - when { - output.exitCode == 123 -> { - Result.success(emptyList()) - } - output.exitCode != 0 -> { - Result.failure(null) - } - stdout == "SOME_BUSINESS_ERROR" -> { - Result.failure("My Business Error Description") - } - else -> { - val records = stdout.lines().map { it.trim() }.map { - val (name, age) = it.split(',') - Record(name, age.trim().toInt()) - } - Result.success(records) - } - } - } - - assertEquals( - listOf(Record("Alice", 25), Record("Bob", 48)), - records.getOrThrow() - ) - } - @ParameterizedTest @EelSource fun testSunnyDay(eelHolder: EelHolder): Unit = timeoutRunBlocking { diff --git a/python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ProcessOutputTransformerShowCaseTest.kt b/python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ProcessOutputTransformerShowCaseTest.kt new file mode 100644 index 000000000000..14483f7f93a6 --- /dev/null +++ b/python/python-exec-service/tests/com/intellij/python/junit5Tests/unit/alsoWin/ProcessOutputTransformerShowCaseTest.kt @@ -0,0 +1,104 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.junit5Tests.unit.alsoWin + +import com.intellij.platform.eel.EelPlatform +import com.intellij.platform.eel.getShell +import com.intellij.platform.eel.provider.asNioPath +import com.intellij.platform.eel.provider.utils.stdoutString +import com.intellij.platform.testFramework.junit5.eel.params.api.EelHolder +import com.intellij.platform.testFramework.junit5.eel.params.api.EelSource +import com.intellij.platform.testFramework.junit5.eel.params.api.TestApplicationWithEel +import com.intellij.python.community.execService.* +import com.intellij.testFramework.common.timeoutRunBlocking +import com.jetbrains.python.Result +import com.jetbrains.python.getOrThrow +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.condition.DisabledOnOs +import org.junit.jupiter.api.condition.OS +import org.junit.jupiter.params.ParameterizedTest + +/** + * How to use [ProcessOutputTransformer] inheritors. + */ +@TestApplicationWithEel(osesMayNotHaveRemoteEels = [OS.WINDOWS, OS.LINUX, OS.MAC]) +class ProcessOutputTransformerShowCaseTest { + + @ParameterizedTest + @EelSource + fun testProcessOutputTransformer(eelHolder: EelHolder): Unit = timeoutRunBlocking { + val eel = eelHolder.eel + + data class Record(val name: String, val age: Int) + + val (shell, execArg) = eel.exec.getShell() + val args = Args(execArg, "echo Alice,25 && echo Bob,48") + + val records = ExecService().execute((BinOnEel(shell.asNioPath())), args) { output -> + val stdout = output.stdoutString.trim() + when { + output.exitCode == 123 -> { + Result.success(emptyList()) + } + output.exitCode != 0 -> { + Result.failure(null) + } + stdout == "SOME_BUSINESS_ERROR" -> { + Result.failure("My Business Error Description") + } + else -> { + val records = stdout.lines().map { it.trim() }.map { + val (name, age) = it.split(',') + Record(name, age.trim().toInt()) + } + Result.success(records) + } + } + } + + assertEquals( + listOf(Record("Alice", 25), Record("Bob", 48)), + records.getOrThrow() + ) + } + + @ParameterizedTest + @EelSource + @DisabledOnOs(OS.WINDOWS, disabledReason = "echo command creates extra escaping on Windows") + fun testZeroCodeJsonParserTransformer(eelHolder: EelHolder): Unit = timeoutRunBlocking { + val eel = eelHolder.eel + + @Serializable + data class Record(val name: String, val age: Int) + + val testData = listOf(Record("Alice", 25), Record("Bob", 48)) + + val (shell, execArg) = eel.exec.getShell() + + val json = Json { ignoreUnknownKeys = true } + val serialized = json.encodeToString(testData) + val args = Args( + execArg, + if (eel.platform is EelPlatform.Windows) "echo $serialized" else "echo '$serialized'" + ) + + val recordsViaManualDecode = ExecService().execute( + binary = (BinOnEel(shell.asNioPath())), + args = args, + processOutputTransformer = ZeroCodeJsonParserTransformer { jsonString -> + json.decodeFromString>(jsonString) + } + ) + + assertEquals(testData, recordsViaManualDecode.getOrThrow()) + + val recordsViaGenericTransform = ExecService().execute( + binary = (BinOnEel(shell.asNioPath())), + args = args, + processOutputTransformer = ZeroCodeJsonParserTransformer>(json) + ) + + assertEquals(testData, recordsViaGenericTransform.getOrThrow()) + } +} \ No newline at end of file diff --git a/python/src/com/jetbrains/python/sdk/PySdkCommandRunner.kt b/python/src/com/jetbrains/python/sdk/PySdkCommandRunner.kt index 4fc07ef1a099..34cf3ae8a6ff 100644 --- a/python/src/com/jetbrains/python/sdk/PySdkCommandRunner.kt +++ b/python/src/com/jetbrains/python/sdk/PySdkCommandRunner.kt @@ -1,11 +1,7 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.jetbrains.python.sdk -import com.intellij.python.community.execService.Args -import com.intellij.python.community.execService.BinOnEel -import com.intellij.python.community.execService.ExecOptions -import com.intellij.python.community.execService.ExecService -import com.intellij.python.community.execService.execGetStdout +import com.intellij.python.community.execService.* import com.jetbrains.python.Result import com.jetbrains.python.errorProcessing.PyResult import org.jetbrains.annotations.ApiStatus.Internal @@ -13,6 +9,16 @@ import java.nio.file.Path import kotlin.time.Duration import kotlin.time.Duration.Companion.minutes +@Internal +suspend fun runExecutableWithProgress( + executable: Path, workDir: Path?, + timeout: Duration = 10.minutes, + env: Map = emptyMap(), + vararg args: String, +): PyResult { + return runExecutableWithProgress(executable, workDir, timeout, env, *args, transformer = ZeroCodeStdoutTransformer) +} + /** * Executes a given executable with specified arguments within an optional project directory. @@ -24,12 +30,26 @@ import kotlin.time.Duration.Companion.minutes * @return A [Result] object containing the output of the command execution. */ @Internal -suspend fun runExecutableWithProgress( +suspend fun runExecutableWithProgress( executable: Path, workDir: Path?, timeout: Duration = 10.minutes, env: Map = emptyMap(), vararg args: String, -): PyResult { + transformer: ProcessOutputTransformer, +): PyResult { val execOptions = ExecOptions(timeout = timeout, env = env) - return ExecService().execGetStdout(BinOnEel(executable, workDir), Args(*args), execOptions) + + val errorHandlerTransformer: ProcessOutputTransformer = { output -> + when { + output.exitCode == 0 -> transformer.invoke(output) + else -> Result.failure(null) + } + } + + return ExecService().execute( + binary = BinOnEel(executable, workDir), + args = Args(*args), + options = execOptions, + processOutputTransformer = errorHandlerTransformer + ) } diff --git a/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt b/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt index cdcee7c39062..1bb283f0197c 100644 --- a/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt +++ b/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt @@ -4,12 +4,14 @@ package com.jetbrains.python.sdk.conda.execution import com.intellij.openapi.diagnostic.thisLogger import com.intellij.platform.eel.isWindows import com.intellij.platform.eel.provider.getEelDescriptor +import com.intellij.python.community.execService.ProcessOutputTransformer +import com.intellij.python.community.execService.ZeroCodeJsonParserTransformer +import com.intellij.python.community.execService.ZeroCodeStdoutTransformer import com.intellij.util.ShellEnvironmentReader import com.jetbrains.python.PyBundle import com.jetbrains.python.errorProcessing.PyResult import com.jetbrains.python.packaging.common.PythonOutdatedPackage import com.jetbrains.python.packaging.common.PythonPackage -import com.jetbrains.python.pyRunCatching import com.jetbrains.python.sdk.conda.execution.models.CondaEnvInfo import com.jetbrains.python.sdk.flavors.conda.PyCondaEnvIdentity import com.jetbrains.python.sdk.runExecutableWithProgress @@ -24,62 +26,82 @@ import kotlin.time.Duration.Companion.minutes object CondaExecutor { suspend fun createNamedEnv(condaPath: Path, envName: String, pythonVersion: String): PyResult { val args = listOf("create", "-y", "-n", envName, "python=${pythonVersion}") - return runConda(condaPath, args, null).mapSuccess { } + return runConda( + condaPath, args, null + ) { PyResult.success(Unit) } } suspend fun createUnnamedEnv(condaPath: Path, envPrefix: String, pythonVersion: String): PyResult { val args = listOf("create", "-y", "-p", envPrefix, "python=${pythonVersion}") - return runConda(condaPath, args, null).mapSuccess { } + return runConda( + condaPath, args, null + ) { PyResult.success(Unit) } } suspend fun createFileEnv(condaPath: Path, environmentYaml: Path): PyResult { val args = listOf("env", "create", "-f", environmentYaml.pathString) - return runConda(condaPath, args, null).mapSuccess { } + return runConda( + condaPath, args, null + ) { PyResult.success(Unit) } } suspend fun updateFromEnvironmentFile(condaPath: Path, envYmlPath: String, envIdentity: PyCondaEnvIdentity): PyResult { val args = listOf("env", "update", "--file", envYmlPath, "--prune") - return runConda(condaPath, args, envIdentity).mapSuccess { } + return runConda( + condaPath, args, envIdentity + ) { PyResult.success(Unit) } } suspend fun listEnvs(condaPath: Path): PyResult { val args = listOf("env", "list", "--json") - val json = runConda(condaPath, args, null).getOr { return it } - return pyRunCatching { - CondaExecutionParser.parseListEnvironmentsOutput(json) - } + return runConda( + condaPath, args, null, + transformer = ZeroCodeJsonParserTransformer { CondaExecutionParser.parseListEnvironmentsOutput(it) } + ) } suspend fun exportEnvironmentFile(condaPath: Path, envIdentity: PyCondaEnvIdentity): PyResult { - return runConda(condaPath, listOf("env", "export") + listOf("--no-builds"), envIdentity) + return runConda( + condaPath, listOf("env", "export") + listOf("--no-builds"), envIdentity, + transformer = ZeroCodeStdoutTransformer + ) } suspend fun listPackages(condaPath: Path, envIdentity: PyCondaEnvIdentity): PyResult> { - return runConda(condaPath, listOf("list", "--json"), envIdentity).mapSuccess { - CondaExecutionParser.parseCondaPackageList(it) - } + return runConda( + condaPath, listOf("list", "--json"), envIdentity, + transformer = ZeroCodeJsonParserTransformer { CondaExecutionParser.parseCondaPackageList(it) } + ) } suspend fun installPackages(condaPath: Path, envIdentity: PyCondaEnvIdentity, packages: List, options: List): PyResult { - return runConda(condaPath, listOf("install") + packages + listOf("-y") + options, envIdentity).mapSuccess { } + return runConda( + condaPath, listOf("install") + packages + listOf("-y") + options, envIdentity + ) { PyResult.success(Unit) } } suspend fun uninstallPackages(condaPath: Path, envIdentity: PyCondaEnvIdentity, packages: List): PyResult { - return runConda(condaPath, listOf("uninstall") + packages + "-y", envIdentity).mapSuccess { } + return runConda( + condaPath, listOf("uninstall") + packages + "-y", envIdentity + ) { PyResult.success(Unit) } } suspend fun listOutdatedPackages(condaPath: Path, envIdentity: PyCondaEnvIdentity): PyResult> { - val jsonPyResult = runConda(condaPath, listOf("update", "--dry-run", "--all", "--json"), envIdentity).getOr { - return it - } - return pyRunCatching { - CondaExecutionParser.parseOutdatedOutputs(jsonPyResult) - } + return runConda( + condaPath, listOf("update", "--dry-run", "--all", "--json"), envIdentity, + transformer = ZeroCodeJsonParserTransformer { CondaExecutionParser.parseOutdatedOutputs(it) } + ) } - private suspend fun runConda(condaPath: Path, args: List, condaEnvIdentity: PyCondaEnvIdentity?, timeout: Duration = 15.minutes): PyResult { + private suspend fun runConda( + condaPath: Path, + args: List, + condaEnvIdentity: PyCondaEnvIdentity?, + timeout: Duration = 15.minutes, + transformer: ProcessOutputTransformer, + ): PyResult { val condaEnv = when (condaEnvIdentity) { is PyCondaEnvIdentity.UnnamedEnv -> { if (condaEnvIdentity.isBase) @@ -98,7 +120,7 @@ object CondaExecutor { } val runArgs = (args + condaEnv).toTypedArray() - return runExecutableWithProgress(condaPath, null, timeout, env = envs, *runArgs) + return runExecutableWithProgress(condaPath, null, timeout, env = envs, *runArgs, transformer = transformer) } private fun getFixedEnvs(condaPath: Path): PyResult> {