From ddc3157c601027ad0b2172ee654b402ba90a40bb Mon Sep 17 00:00:00 2001 From: Alexey Katsman Date: Thu, 27 Nov 2025 15:28:18 +0100 Subject: [PATCH] PY-85903 Implement a cache for conda environments GitOrigin-RevId: 78338ac3ffc7f25a6ba980cd017f0c8c0602bc6b --- .../conda/PyEnvironmentYmlSdkConfiguration.kt | 2 +- .../intellij.python.community.impl.xml | 4 +- .../python/sdk/PySdkCommandRunner.kt | 3 +- .../conda/CondaExistingEnvironmentSelector.kt | 2 +- .../python/sdk/add/v2/conda/CondaViewModel.kt | 10 +-- .../sdk/conda/execution/CondaExecutor.kt | 14 ++- .../python/sdk/flavors/conda/PyCondaEnv.kt | 73 ++++++++------- .../sdk/flavors/conda/PyCondaEnvProvider.kt | 88 +++++++++++++++++++ 8 files changed, 147 insertions(+), 49 deletions(-) create mode 100644 python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnvProvider.kt diff --git a/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/conda/PyEnvironmentYmlSdkConfiguration.kt b/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/conda/PyEnvironmentYmlSdkConfiguration.kt index ba8ae1b27cac..e6ff726870cd 100644 --- a/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/conda/PyEnvironmentYmlSdkConfiguration.kt +++ b/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/conda/PyEnvironmentYmlSdkConfiguration.kt @@ -199,7 +199,7 @@ internal class PyEnvironmentYmlSdkConfiguration : PyProjectSdkConfigurationExten private suspend fun createCondaEnv(project: Project, condaExecutable: String, environmentYml: String): PyResult { val binaryToExec = BinOnEel(Path.of(condaExecutable)) - val existingEnvs = PyCondaEnv.getEnvs(binaryToExec).getOrNull() ?: emptyList() + val existingEnvs = PyCondaEnv.getEnvs(binaryToExec, forceRefresh = true).getOrNull() ?: emptyList() val existingSdks = PyConfigurableInterpreterList.getInstance(project).model.sdks val newCondaEnvInfo = NewCondaEnvRequest.LocalEnvByLocalEnvironmentFile(Path.of(environmentYml), diff --git a/python/pluginResources/intellij.python.community.impl.xml b/python/pluginResources/intellij.python.community.impl.xml index 753019dfb6ba..42b63ccd893f 100644 --- a/python/pluginResources/intellij.python.community.impl.xml +++ b/python/pluginResources/intellij.python.community.impl.xml @@ -35,7 +35,9 @@ implementationClass="com.jetbrains.python.sdk.poetry.PoetryPackageVersionsInspection" key="INSP.poetry.package.versions.display.name" bundle="messages.PyBundle" groupKey="INSP.GROUP.python" suppressId="PoetryPackageVersions" shortName="PoetryPackageVersionsInspection"/> - + + + runExecutableWithProgress( env: Map = emptyMap(), vararg args: String, transformer: ProcessOutputTransformer, + execService: ExecService = ExecService(), processWeight: ConcurrentProcessWeight = ConcurrentProcessWeight.LIGHT ): PyResult { val execOptions = ExecOptions(timeout = timeout, env = env, weight = processWeight) @@ -37,7 +38,7 @@ suspend fun runExecutableWithProgress( } } - return ExecService().execute( + return execService.execute( binary = binaryToExec, args = Args(*args), options = execOptions, diff --git a/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaExistingEnvironmentSelector.kt b/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaExistingEnvironmentSelector.kt index 717cf4b47c84..e2f376413ad1 100644 --- a/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaExistingEnvironmentSelector.kt +++ b/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaExistingEnvironmentSelector.kt @@ -122,7 +122,7 @@ internal class CondaExistingEnvironmentSelector

(model: PythonAdd reloadLink.action = object : AbstractAction(message("sdk.create.custom.conda.refresh.envs")) { override fun actionPerformed(e: ActionEvent?) { - model.condaViewModel.detectCondaEnvironments() + model.condaViewModel.detectCondaEnvironments(forceRefresh = true) } } diff --git a/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaViewModel.kt b/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaViewModel.kt index 7e8d511083ff..ed98148681d9 100644 --- a/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaViewModel.kt +++ b/python/src/com/jetbrains/python/sdk/add/v2/conda/CondaViewModel.kt @@ -89,15 +89,15 @@ class CondaViewModel

( baseCondaEnv.set(null) if (condaExecutable?.validationResult?.successOrNull != null) { - detectCondaEnvironments() + detectCondaEnvironments(forceRefresh = false) } } } - fun detectCondaEnvironments() { + fun detectCondaEnvironments(forceRefresh: Boolean) { condaEnvironmentsLoading.value = true scope.launch(Dispatchers.UI) { - condaEnvironmentsResult.value = updateCondaEnvironments() + condaEnvironmentsResult.value = updateCondaEnvironments(forceRefresh) }.invokeOnCompletion { condaEnvironmentsLoading.value = false } @@ -133,13 +133,13 @@ class CondaViewModel

( path.refreshAndFindVirtualFileOrDirectory()?.takeIf { virtualFile -> virtualFile.isFile } } - private suspend fun updateCondaEnvironments(): PyResult> = withContext(Dispatchers.IO) { + private suspend fun updateCondaEnvironments(forceRefresh: Boolean): PyResult> = withContext(Dispatchers.IO) { val executable = condaExecutable.get() if (executable == null) return@withContext PyResult.localizedError(message("python.sdk.conda.no.exec")) executable.validationResult.getOr { return@withContext it } val binaryToExec = executable.pathHolder?.let { fileSystem.getBinaryToExec(it) }!! - val environments = PyCondaEnv.getEnvs(binaryToExec).getOr { return@withContext it } + val environments = PyCondaEnv.getEnvs(binaryToExec, forceRefresh).getOr { return@withContext it } val baseConda = environments.find { env -> env.envIdentity.let { it is PyCondaEnvIdentity.UnnamedEnv && it.isBase } } withContext(Dispatchers.UI) { diff --git a/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt b/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt index 047923225227..90edcdb08ef3 100644 --- a/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt +++ b/python/src/com/jetbrains/python/sdk/conda/execution/CondaExecutor.kt @@ -57,10 +57,11 @@ object CondaExecutor { ) { PyResult.success(Unit) } } - suspend fun listEnvs(binaryToExec: BinaryToExec): PyResult { + suspend fun listEnvs(binaryToExec: BinaryToExec, execService: ExecService = ExecService()): PyResult { val args = listOf("env", "list", "--json") return runConda( binaryToExec, args, null, + execService = execService, transformer = ZeroCodeJsonParserTransformer { CondaExecutionParser.parseListEnvironmentsOutput(it) } ) } @@ -129,11 +130,20 @@ object CondaExecutor { args: List, condaEnvIdentity: PyCondaEnvIdentity?, timeout: Duration = 15.minutes, + execService: ExecService = ExecService(), transformer: ProcessOutputTransformer, ): PyResult { val envs = getFixedEnvs(binaryToExec).getOr { return it } val runArgs = prepareCondaRunArgs(args, emptyList(), condaEnvIdentity).toTypedArray() - return runExecutableWithProgress(binaryToExec, timeout, env = envs, *runArgs, transformer = transformer, processWeight = ConcurrentProcessWeight.HEAVY) + return runExecutableWithProgress( + binaryToExec, + timeout, + env = envs, + *runArgs, + transformer = transformer, + execService = execService, + processWeight = ConcurrentProcessWeight.HEAVY + ) } private fun getFixedEnvs(binaryToExec: BinaryToExec): PyResult> { diff --git a/python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnv.kt b/python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnv.kt index 8652e41ca374..d4b38b1b1ee3 100644 --- a/python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnv.kt +++ b/python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnv.kt @@ -4,27 +4,28 @@ package com.jetbrains.python.sdk.flavors.conda import com.intellij.execution.target.FullPathOnTarget import com.intellij.execution.target.TargetEnvironmentConfiguration import com.intellij.execution.target.TargetedCommandLineBuilder +import com.intellij.openapi.components.Service +import com.intellij.openapi.components.service import com.intellij.openapi.project.Project import com.intellij.openapi.projectRoots.Sdk -import com.intellij.python.community.execService.BinOnEel -import com.intellij.python.community.execService.BinOnTarget +import com.intellij.openapi.util.IntellijInternalApi +import com.intellij.openapi.util.registry.RegistryManager import com.intellij.python.community.execService.BinaryToExec import com.jetbrains.python.errorProcessing.PyResult import com.jetbrains.python.sdk.conda.TargetCommandExecutor import com.jetbrains.python.sdk.conda.createCondaSdkFromExistingEnv -import com.jetbrains.python.sdk.conda.execution.CondaExecutor import com.jetbrains.python.sdk.flavors.conda.PyCondaEnv.Companion.getEnvs +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async import org.jetbrains.annotations.ApiStatus -import java.nio.file.Path -import java.util.* -import kotlin.io.path.name +import kotlin.time.Duration.Companion.seconds /** * TODO: Once we get rid of [TargetCommandExecutor] and have access to [TargetEnvironmentConfiguration] use it validate conda binary in [getEnvs] * @see `PyCondaTest` */ @ApiStatus.Internal - data class PyCondaEnv( val envIdentity: PyCondaEnvIdentity, val fullCondaPathOnTarget: FullPathOnTarget, @@ -32,39 +33,20 @@ data class PyCondaEnv( companion object { /** + * The logic is the following: + * + * - If an explicit refresh is triggered, ask the cache to reload the value + * - If a value is present in the cache, a refresh is triggered if the refresh interval has passed, and the old value is returned: + * - If it's an error, let's try to reload (we may succeed this time) + * - If it's a success, return this value + * - If a value is not present in the cache, it will be calculated + * * @return list of conda environments */ @ApiStatus.Internal - suspend fun getEnvs(binaryToExec: BinaryToExec): PyResult> { - val condaPath = when (binaryToExec) { - is BinOnEel -> binaryToExec.path.toString() - is BinOnTarget -> binaryToExec.getLocalExePath().value - } - val info = CondaExecutor.listEnvs(binaryToExec).getOr { return it } - val condaPrefix = info.condaPrefix ?: condaPath.removeSuffix("/bin/conda") - val envs = info.envs.distinctBy { it.trim().lowercase(Locale.getDefault()) } - val identities = envs.map { envPath -> - // Env name is the basename for envs inside of default location - // envPath should be direct child of envs_dirs to be a NamedEnv - val isEnvName = info.envsDirs.any { - Path.of(it) == Path.of(envPath).parent - } - val envName = if (isEnvName) - Path.of(envPath).name - else - null - val base = envPath.equals(condaPrefix, ignoreCase = true) - val identity = if (envName != null) { - PyCondaEnvIdentity.NamedEnv(envName) - } - else { - PyCondaEnvIdentity.UnnamedEnv(envPath, base) - } - PyCondaEnv(identity, condaPath) - } - - return PyResult.success(identities) - } + @JvmOverloads + suspend fun getEnvs(binaryToExec: BinaryToExec, forceRefresh: Boolean = false): PyResult> = + service().getEnvs(binaryToExec, forceRefresh) suspend fun createEnv(command: PyCondaCommand, newCondaEnvInfo: NewCondaEnvRequest): PyResult { return newCondaEnvInfo.create(command.asBinaryToExec()) @@ -98,4 +80,19 @@ data class PyCondaEnv( } override fun toString(): String = "$envIdentity@$fullCondaPathOnTarget" -} \ No newline at end of file +} + +@OptIn(IntellijInternalApi::class) +@Service(Service.Level.APP) +private class CondaEnvService(scope: CoroutineScope) { + private val _condaEnvProviderImpl: Deferred = scope.async { + PyCondaEnvProvider( + refreshInterval = RegistryManager.getInstanceAsync().intValue("python.conda.envs.refresh.seconds").seconds, + ttlAfterWrite = RegistryManager.getInstanceAsync().intValue("python.conda.envs.cache.ttl.seconds").seconds, + ) + } + private suspend fun condaEnvProvider() = _condaEnvProviderImpl.await() + + suspend fun getEnvs(binaryToExec: BinaryToExec, forceRefresh: Boolean): PyResult> = + condaEnvProvider().getEnvs(binaryToExec, forceRefresh) +} diff --git a/python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnvProvider.kt b/python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnvProvider.kt new file mode 100644 index 000000000000..abd5b86a9168 --- /dev/null +++ b/python/src/com/jetbrains/python/sdk/flavors/conda/PyCondaEnvProvider.kt @@ -0,0 +1,88 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.jetbrains.python.sdk.flavors.conda + +import com.github.benmanes.caffeine.cache.AsyncLoadingCache +import com.github.benmanes.caffeine.cache.Caffeine +import com.intellij.openapi.progress.runBlockingMaybeCancellable +import com.intellij.openapi.util.IntellijInternalApi +import com.intellij.python.community.execService.BinOnEel +import com.intellij.python.community.execService.BinOnTarget +import com.intellij.python.community.execService.BinaryToExec +import com.intellij.python.community.execService.ExecService +import com.jetbrains.python.errorProcessing.PyResult +import com.jetbrains.python.isFailure +import com.jetbrains.python.isSuccess +import com.jetbrains.python.sdk.conda.execution.CondaExecutor +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.asExecutor +import kotlinx.coroutines.future.asDeferred +import org.jetbrains.annotations.ApiStatus +import java.nio.file.Path +import java.util.* +import java.util.concurrent.Executor +import kotlin.io.path.name +import kotlin.time.Duration +import kotlin.time.toJavaDuration + +@ApiStatus.Internal +@IntellijInternalApi +class PyCondaEnvProvider( + refreshInterval: Duration, + ttlAfterWrite: Duration, + executor: Executor = Dispatchers.Default.asExecutor(), + private val execService: ExecService = ExecService(), +) { + private val cache: AsyncLoadingCache>> = Caffeine.newBuilder() + .executor(executor) + .refreshAfterWrite(refreshInterval.toJavaDuration()) + .expireAfterWrite(ttlAfterWrite.toJavaDuration()) + .buildAsync { binaryToExec -> + runBlockingMaybeCancellable { + getEnvsInternal(binaryToExec) + } + } + + suspend fun getEnvs(binaryToExec: BinaryToExec, forceRefresh: Boolean): PyResult> { + if (forceRefresh) { + return cache.synchronous().refresh(binaryToExec).asDeferred().await() + } + + val currentValue = cache.getIfPresent(binaryToExec)?.asDeferred()?.await() + return when { + currentValue?.isSuccess == true -> currentValue + currentValue?.isFailure == true -> cache.synchronous().refresh(binaryToExec).asDeferred().await() + else -> cache[binaryToExec].asDeferred().await() + } + } + + private suspend fun getEnvsInternal(binaryToExec: BinaryToExec): PyResult> { + val condaPath = when (binaryToExec) { + is BinOnEel -> binaryToExec.path.toString() + is BinOnTarget -> binaryToExec.getLocalExePath().value + } + val info = CondaExecutor.listEnvs(binaryToExec, execService).getOr { return it } + val condaPrefix = info.condaPrefix ?: condaPath.removeSuffix("/bin/conda") + val envs = info.envs.distinctBy { it.trim().lowercase(Locale.getDefault()) } + val identities = envs.map { envPath -> + // Env name is the basename for envs inside of default location + // envPath should be direct child of envs_dirs to be a NamedEnv + val isEnvName = info.envsDirs.any { + Path.of(it) == Path.of(envPath).parent + } + val envName = if (isEnvName) + Path.of(envPath).name + else + null + val base = envPath.equals(condaPrefix, ignoreCase = true) + val identity = if (envName != null) { + PyCondaEnvIdentity.NamedEnv(envName) + } + else { + PyCondaEnvIdentity.UnnamedEnv(envPath, base) + } + PyCondaEnv(identity, condaPath) + } + + return PyResult.success(identities) + } +} \ No newline at end of file