diff --git a/python/services/system-python/BUILD.bazel b/python/services/system-python/BUILD.bazel index a4d8f1c4978e..c4199cd00609 100644 --- a/python/services/system-python/BUILD.bazel +++ b/python/services/system-python/BUILD.bazel @@ -47,6 +47,7 @@ jvm_library( resources = [":system-python_test_resources"], associates = [":system-python"], deps = [ + "//platform/util/coroutines", "@lib//:kotlin-stdlib", "//python/python-sdk:sdk", "//python/python-sdk:sdk_test_lib", @@ -75,6 +76,8 @@ jvm_library( "//platform/testFramework:testFramework_test_lib", "//python/python-psi-impl:psi-impl", "//python/python-exec-service/execService.python", + "//platform/testFramework/junit5/eel", + "//platform/testFramework/junit5/eel:eel_test_lib", ], exports = [ "//python/services/shared", diff --git a/python/services/system-python/intellij.python.community.services.systemPython.iml b/python/services/system-python/intellij.python.community.services.systemPython.iml index 572f3c2302a1..247d311a5b38 100644 --- a/python/services/system-python/intellij.python.community.services.systemPython.iml +++ b/python/services/system-python/intellij.python.community.services.systemPython.iml @@ -10,6 +10,7 @@ + @@ -33,5 +34,6 @@ + \ No newline at end of file diff --git a/python/services/system-python/resources/intellij.python.community.services.systemPython.xml b/python/services/system-python/resources/intellij.python.community.services.systemPython.xml index dcfcff144174..455ef18d3d69 100644 --- a/python/services/system-python/resources/intellij.python.community.services.systemPython.xml +++ b/python/services/system-python/resources/intellij.python.community.services.systemPython.xml @@ -13,6 +13,10 @@ + + + @@ -28,7 +32,7 @@ + /> \ No newline at end of file diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt index 51cdda6716fe..58e40f6533cc 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt @@ -17,11 +17,21 @@ import kotlin.time.Duration * If cache hasn't been calculated yet, this function suspends till [getDataForCache]. * Cache could be cleared with [clear]. * - * Once [startUpdate] is called (which is idempotent call) cache will be updarted on [scope] every [updateInterval]. + * Once [startUpdate] is called (which is idempotent call) cache will be updated on [scope] every [delayer]. * * Cache isn't blocked while updating: a previous version is used. */ -internal class Cache(scope: CoroutineScope, updateInterval: Duration, private val getDataForCache: suspend (K) -> List) { +internal class Cache( + scope: CoroutineScope, + delayer: UpdateCacheDelayer, + private val getDataForCache: suspend (K) -> List, +) { + constructor( + scope: CoroutineScope, + updateInterval: Duration, + getDataForCache: suspend (K) -> List, + ) : this(scope, UpdateCacheDelayer.TimeBased(updateInterval), getDataForCache) + private companion object { val logger = fileLogger() } @@ -35,16 +45,24 @@ internal class Cache(scope: CoroutineScope, updateInterval: Duration, priv private val cacheUpdateJob = lazy { scope.launch(Dispatchers.Default) { while (true) { - delay(updateInterval) - logger.debug("Updating cache") - val values = cache.keys.toList() // copy keys not to affect the current cache - for (k in values) { - updateCache(k) + val callAfterUpdate = delayer.delayBeforeUpdate() + updateCacheForAllKeys() + if (callAfterUpdate != null) { + callAfterUpdate() } } } } + private suspend fun updateCacheForAllKeys() { + val keys = cache.keys + logger.debug("Updating cache for keys ${keys.joinToString(", ")}") + val values = keys.toList() // copy keys not to affect the current cache + for (k in values) { + updateCache(k) + } + } + /** * Updates cache and suspends till finished */ @@ -95,4 +113,20 @@ internal class Cache(scope: CoroutineScope, updateInterval: Duration, priv logger.info("End update for $k") return newValue } +} + +/** + * [Cache] calls [delayBeforeUpdate] in a loop and updates cache after each call. + * [TimeBased] is an implementation that uses [delay]. + * + * Returned value (if any) is called after each cache update + */ +internal fun interface UpdateCacheDelayer { + suspend fun delayBeforeUpdate(): (() -> Unit)? + class TimeBased(private val duration: Duration) : UpdateCacheDelayer { + override suspend fun delayBeforeUpdate(): (() -> Unit)? { + delay(duration) + return null + } + } } \ No newline at end of file diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/EelDescriptorFilter.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/EelDescriptorFilter.kt new file mode 100644 index 000000000000..ca7d14846ce6 --- /dev/null +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/EelDescriptorFilter.kt @@ -0,0 +1,23 @@ +// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.community.services.systemPython.impl + +import com.intellij.openapi.extensions.ExtensionPointName +import com.intellij.platform.eel.EelDescriptor + +/** + * Control various parameters for [com.intellij.python.community.services.systemPython.SystemPythonServiceImpl] based + * on [EelDescriptor], **not** a public SPI + */ +interface EelDescriptorFilter { + + companion object { + internal val EP: ExtensionPointName = + ExtensionPointName("com.intellij.python.community.services.systemPython.impl.eelFilter") + internal val EelDescriptor.isEphemeral: Boolean get() = EP.extensionList.any { it.isEphemeral(this) } + } + + /** + * Ephemeral [eelDescriptor] should never be cached nor persisted in user settings + */ + fun isEphemeral(eelDescriptor: EelDescriptor): Boolean +} diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/AsdfSystemProvider.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/AsdfSystemProvider.kt index 598685e55c4f..08b0786ba200 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/AsdfSystemProvider.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/AsdfSystemProvider.kt @@ -44,7 +44,7 @@ internal class AsdfSystemPythonProvider : SystemPythonProvider { val paths = entries .map { versionsDir.resolve(it).resolve("bin").asNioPath() } - return@withContext collectPythonsInPaths(eelApi, paths, listOf(python3NamePattern)) + return@withContext collectPythonsInPaths( paths, listOf(python3NamePattern)) } catch (e: RuntimeException) { if (Logger.shouldRethrow(e)) throw e diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/BrewSystemPythonProvider.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/BrewSystemPythonProvider.kt index 772d613b2f83..88ad30b462c6 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/BrewSystemPythonProvider.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/BrewSystemPythonProvider.kt @@ -25,7 +25,7 @@ internal class BrewSystemPythonProvider : SystemPythonProvider { val pythons = withContext(Dispatchers.IO) { try { - return@withContext collectPythonsInPaths(eelApi, listOf(binDirectory), listOf(python3XNamePattern)) + return@withContext collectPythonsInPaths( listOf(binDirectory), listOf(python3XNamePattern)) } catch (e: RuntimeException) { LOGGER.error("failed to discover brew pythons", e) diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/MacSystemPythonProvider.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/MacSystemPythonProvider.kt index ac0e4e9ed3c8..6c0e60e201c3 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/MacSystemPythonProvider.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/MacSystemPythonProvider.kt @@ -39,7 +39,7 @@ internal class MacSystemPythonProvider : SystemPythonProvider { val pythons = withContext(Dispatchers.IO) { try { - return@withContext collectPythonsInPaths(eelApi, directories, names) + return@withContext collectPythonsInPaths( directories, names) } catch (e: RuntimeException) { LOGGER.error("Failed to discover mac system pythons", e) diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/PyenvSystemPythonProvider.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/PyenvSystemPythonProvider.kt index b9be2c4a5470..982e9b676ec3 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/PyenvSystemPythonProvider.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/PyenvSystemPythonProvider.kt @@ -44,7 +44,7 @@ internal class PyenvSystemPythonProvider : SystemPythonProvider { val paths = entries .map { versionsDir.resolve(it).resolve("bin").asNioPath() } - return@withContext collectPythonsInPaths(eelApi, paths, listOf(python3NamePattern)) + return@withContext collectPythonsInPaths( paths, listOf(python3NamePattern)) } catch (e: RuntimeException) { LOGGER.error("failed to discover pyenv pythons", e) diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/UnixSystemPythonProvider.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/UnixSystemPythonProvider.kt index a963f4594899..0fe46a6d022c 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/UnixSystemPythonProvider.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/UnixSystemPythonProvider.kt @@ -5,21 +5,21 @@ import com.intellij.openapi.diagnostic.Logger import com.intellij.platform.eel.EelApi import com.intellij.platform.eel.EelPlatform import com.intellij.platform.eel.isMac +import com.intellij.platform.eel.provider.utils.Path import com.intellij.python.community.services.systemPython.SystemPythonProvider import com.jetbrains.python.PyToolUIInfo import com.jetbrains.python.PythonBinary import com.jetbrains.python.errorProcessing.PyResult import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -import java.nio.file.Path internal class UnixSystemPythonProvider : SystemPythonProvider { private val LOGGER: Logger = Logger.getInstance(UnixSystemPythonProvider::class.java) - private val directories = listOf( - Path.of("/usr/bin"), - Path.of("/usr/local/bin")) + private val directories = arrayOf( + "/usr/bin", + "/usr/local/bin") // Patterns to match Python executable filenames private val names = listOf( @@ -36,7 +36,7 @@ internal class UnixSystemPythonProvider : SystemPythonProvider { val pythons = withContext(Dispatchers.IO) { try { - return@withContext collectPythonsInPaths(eelApi, directories, names) + return@withContext collectPythonsInPaths(directories.map { Path(it, eelApi.descriptor) }, names) } catch (e: RuntimeException) { LOGGER.error("Failed to discover unix system pythons", e) diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/Utils.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/Utils.kt index 8186120ab781..539103126a27 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/Utils.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/providers/Utils.kt @@ -1,16 +1,17 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.services.systemPython.impl.providers +import com.intellij.openapi.diagnostic.fileLogger import com.intellij.openapi.util.registry.Registry -import com.intellij.platform.eel.EelApi -import com.intellij.platform.eel.fs.EelFileSystemApi.StatError -import com.intellij.platform.eel.fs.stat -import com.intellij.platform.eel.getOrNull -import com.intellij.platform.eel.path.EelPath -import com.intellij.platform.eel.provider.asNioPath +import com.jetbrains.python.venvReader.Directory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.IOException +import java.nio.file.FileSystemNotFoundException +import java.nio.file.NotDirectoryException import java.nio.file.Path import java.util.regex.Pattern -import kotlin.io.path.pathString +import kotlin.io.path.listDirectoryEntries internal val pypyNamePattern: Pattern = Pattern.compile("pypy$") @@ -21,23 +22,28 @@ internal fun useLegacyPythonProvider(): Boolean { return Registry.`is`("python.use.system.legacy.provider") } -internal suspend fun collectPythonsInPaths(eelApi: EelApi, paths: List, names: List): Set { - val pythons = mutableSetOf() - - for (path in paths) { - val directory = EelPath.parse(path.pathString, eelApi.descriptor) - if (eelApi.fs.stat(directory).eelIt() is StatError) { - continue - } - - val entries = eelApi.fs.listDirectory(directory) - .getOrNull() - - entries - ?.map { directory.resolve(it).asNioPath() } - ?.filter { names.firstOrNull { name -> name.matcher(it.fileName.toString()).matches() } != null } - ?.let { pythons.addAll(it) } +internal suspend fun collectPythonsInPaths(paths: List, names: List): Set = + withContext(Dispatchers.IO) { + paths + .flatMap { + try { + it.listDirectoryEntries() + } + catch (_: NotDirectoryException) { + emptyList() + } + catch (e: FileSystemNotFoundException) { + // This is a temporary hack: Eel might throw this exception in tests when fs gets deregistered + // it will be removed as soon as we arrange it with eel + logger.warn("Path $it filesystem is inaccessible", e) + emptyList() + } + catch (e: IOException) { + logger.warn("Path $it is inaccessible", e) + emptyList() + } + } + .filter { child -> names.any { it.matcher(child.fileName.toString()).matches() } }.toSet() } - return pythons -} \ No newline at end of file +private val logger = fileLogger() \ No newline at end of file diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt index 305f032f2b61..c49a0c535c3c 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt @@ -1,6 +1,7 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.services.systemPython +import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.EDT import com.intellij.openapi.components.BaseState import com.intellij.openapi.components.RoamingType @@ -20,7 +21,9 @@ import com.intellij.python.community.impl.installer.PySdkToInstallManager import com.intellij.python.community.services.internal.impl.VanillaPythonWithPythonInfoImpl import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl.MyServiceState import com.intellij.python.community.services.systemPython.impl.Cache +import com.intellij.python.community.services.systemPython.impl.EelDescriptorFilter.Companion.isEphemeral import com.intellij.python.community.services.systemPython.impl.PySystemPythonBundle +import com.intellij.python.community.services.systemPython.impl.UpdateCacheDelayer import com.intellij.python.community.services.systemPython.impl.asSysPythonRegisterError import com.jetbrains.python.NON_INTERACTIVE_ROOT_TRACE_CONTEXT import com.jetbrains.python.PyToolUIInfo @@ -29,6 +32,7 @@ import com.jetbrains.python.Result import com.jetbrains.python.errorProcessing.getOr import com.jetbrains.python.getOrNull import com.jetbrains.python.sdk.installer.installBinary +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -58,16 +62,24 @@ internal suspend fun getCacheTimeout(): Duration? = @State(name = "SystemPythonService", storages = [Storage("systemPythonService.xml", roamingType = RoamingType.LOCAL)], allowLoadInTests = true) @Internal -class SystemPythonServiceImpl(scope: CoroutineScope) : SystemPythonService, - SimplePersistentStateComponent(MyServiceState()) { +class SystemPythonServiceImpl internal constructor( + scope: CoroutineScope, + createUpdateCacheDelayer: suspend () -> UpdateCacheDelayer?, +) : SystemPythonService, + SimplePersistentStateComponent(MyServiceState()) { + constructor(scope: CoroutineScope) : this(scope, { + val duration = getCacheTimeout() + if (duration != null) UpdateCacheDelayer.TimeBased(duration) else null + }) + private val findPythonsMutex = Mutex() private val _cacheImpl: CompletableDeferred?> = CompletableDeferred() private suspend fun cache() = _cacheImpl.await() init { scope.launch { - _cacheImpl.complete(getCacheTimeout()?.let { interval -> - Cache(scope, interval) { eelDescriptor -> + _cacheImpl.complete(createUpdateCacheDelayer()?.let { delayer -> + Cache(scope, delayer) { eelDescriptor -> withContext(NON_INTERACTIVE_ROOT_TRACE_CONTEXT) { searchPythonsPhysicallyNoCache(eelDescriptor.toEelApi()) } @@ -81,26 +93,34 @@ class SystemPythonServiceImpl(scope: CoroutineScope) : SystemPythonService, .getOr(PySystemPythonBundle.message("py.system.python.service.python.is.broken", pythonPath)) { return Result.failure(it.error.asSysPythonRegisterError()) } val systemPython = SystemPython.create(pythonWithLangLevel, null).getOr { return it } - state.userProvidedPythons.add(pythonPath.pathString) - cache()?.get(pythonPath.getEelDescriptor())?.add(systemPython) + + val eelDescriptor = pythonPath.getEelDescriptor() + if (!eelDescriptor.isEphemeral) { + state.userProvidedPythons.add(pythonPath.pathString) + logger.debug("Registering $pythonPath") + cache()?.get(eelDescriptor)?.add(systemPython) + } return Result.success(systemPython) } override fun getInstaller(eelApi: EelApi): PythonInstallerService? = if (eelApi == localEel) LocalPythonInstaller else null - override suspend fun findSystemPythons(eelApi: EelApi, forceRefresh: Boolean): List = - cache()?.let { cache -> + override suspend fun findSystemPythons(eelApi: EelApi, forceRefresh: Boolean): List { + val eelDescriptor = eelApi.descriptor + val cache = if (!eelDescriptor.isEphemeral) cache() else null + return cache?.let { cache -> // Cache enabled cache.startUpdate() if (forceRefresh) { logger.info("pythons refresh requested") - cache.updateCache(eelApi.descriptor) // Update cache and suspend till update finished + cache.updateCache(eelDescriptor) // Update cache and suspend till update finished } else { - cache.get(eelApi.descriptor) + cache.get(eelDescriptor) }.sortedSystemPythons() } ?: searchPythonsPhysicallyNoCache(eelApi).sortedSystemPythons() + } private fun Iterable.sortedSystemPythons(): List = sortedWith( @@ -184,4 +204,4 @@ private object LocalPythonInstaller : PythonInstallerService { } return Result.Companion.success(Unit) } -} +} \ No newline at end of file diff --git a/python/services/system-python/testResources/intellij.python.community.services.systemPython._test.xml b/python/services/system-python/testResources/intellij.python.community.services.systemPython._test.xml index 3085fe637e79..6dd0e3184784 100644 --- a/python/services/system-python/testResources/intellij.python.community.services.systemPython._test.xml +++ b/python/services/system-python/testResources/intellij.python.community.services.systemPython._test.xml @@ -1,6 +1,9 @@ + + + diff --git a/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/SystemPythonServiceStaleDescriptorTest.kt b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/SystemPythonServiceStaleDescriptorTest.kt new file mode 100644 index 000000000000..25ba3aa52bd7 --- /dev/null +++ b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/SystemPythonServiceStaleDescriptorTest.kt @@ -0,0 +1,57 @@ +// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.junit5Tests.env.systemPython.impl + +import com.intellij.platform.testFramework.junit5.eel.params.api.DockerTest +import com.intellij.platform.testFramework.junit5.eel.params.api.EelHolder +import com.intellij.platform.testFramework.junit5.eel.params.api.EelSource +import com.intellij.platform.testFramework.junit5.eel.params.api.TestApplicationWithEel +import com.intellij.platform.testFramework.junit5.eel.params.api.WslTest +import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl +import com.intellij.python.community.services.systemPython.impl.UpdateCacheDelayer +import com.intellij.python.junit5Tests.framework.applicationScope +import com.intellij.testFramework.common.timeoutRunBlocking +import kotlinx.coroutines.sync.Mutex +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.condition.OS +import org.junit.jupiter.params.ParameterizedTest +import kotlin.time.Duration.Companion.seconds + +@TestApplicationWithEel(osesMayNotHaveRemoteEels = [OS.WINDOWS]) +internal class SystemPythonServiceStaleDescriptorTest { + companion object { + private lateinit var sut: SystemPythonServiceImpl + private val startUpdate = Mutex(locked = true) + private val updateFinished = Mutex(locked = true) + private val scope = applicationScope("${SystemPythonServiceStaleDescriptorTest}") + + @BeforeAll + @JvmStatic + fun setUp() { + sut = SystemPythonServiceImpl(scope.get()) { + UpdateCacheDelayer { + startUpdate.lock() // Start cache update when lock is opened + ({ + updateFinished.unlock() // this lock will be opened as soon as cache updated + }) + } + } + } + + @AfterAll + @JvmStatic + fun makeSureNoEelLeaked(): Unit = timeoutRunBlocking(30.seconds) { + startUpdate.unlock() // Start cache update + updateFinished.lock() // Wait for its end + } + } + + @ParameterizedTest + @EelSource + @DockerTest("python:3.14.2-trixie", mandatory = false) + @WslTest("Ubuntu-22.04", mandatory = false) + fun testPythonOnDocker(eelHolder: EelHolder): Unit = timeoutRunBlocking { + Assertions.assertTrue(sut.findSystemPythons(eelHolder.eel).isNotEmpty(), "No pythons found on ${eelHolder.eel}") + } +} \ No newline at end of file