PY-87542, IJPL-234101: Do not touch removed container eel descriptor in tests.

PY-87542, IJPL-234101: (technical fix): suppress test dependencies check. There is a bug in `IdeaUltimatePackagingTest` and reports several dependencies as redundant. But they aren't. I've been told by Develar to suppress it for now. PY-87542, IJPL-234101: Support `SystemPythonService` python detection on WSL and Docker/Windows. Enable `UnixSystemPythonProvider` on Windows so it can find WSL and Docker pythons even when host is Windows. We need it to test Docker descriptor support on Windows, and it is generally good thing to have. To do that, we need to fix `directories` (`Path(""/foo/bar")` doesn't work on Windows JVM. `collectPythonsInPaths` was also broken: `Path` isn't a string: it is a path (either real or eel path) with eel descriptor inside. PY-87542, IJPL-234101: Do not touch removed container eel descriptor in tests. `SystemPythonService` caches eel descriptors and looks for pythons in background. `VirtualEnvReaderEelTest` called it for Docker, and removed the container shortly after, effectively left `SystemPythonService` with unusable descriptor that throws random (undocumented) exceptions from `toEelApi`. Eel descriptor for Docker lifetime is somewhat blur (must be discussed with Eel team) so for now we mark it "ephemeral" which means we never cache it: see `EelDescriptorFilter` Merge-request: IJ-MR-191348 Merged-by: Ilya Kazakevich <ilya.kazakevich@jetbrains.com> GitOrigin-RevId: e007e09b2e8a496105e9a8a269916981823c7f78
2026-09-18 09:34:34 +07:00 · 2026-02-14 23:16:18 +00:00
parent ad6fcb2006
commit 2cbfd1a42d
14 changed files with 205 additions and 53 deletions
@@ -47,6 +47,7 @@ jvm_library(
  resources = [":system-python_test_resources"],
  associates = [":system-python"],
  deps = [
+    "//platform/util/coroutines",
    "@lib//:kotlin-stdlib",
    "//python/python-sdk:sdk",
    "//python/python-sdk:sdk_test_lib",
@@ -75,6 +76,8 @@ jvm_library(
    "//platform/testFramework:testFramework_test_lib",
    "//python/python-psi-impl:psi-impl",
    "//python/python-exec-service/execService.python",
+    "//platform/testFramework/junit5/eel",
+    "//platform/testFramework/junit5/eel:eel_test_lib",
  ],
  exports = [
    "//python/services/shared",
@@ -10,6 +10,7 @@
      <sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
    </content>
    <orderEntry type="inheritedJdk" />
+    <orderEntry type="module" module-name="intellij.platform.util.coroutines" scope="TEST" />
    <orderEntry type="sourceFolder" forTests="false" />
    <orderEntry type="library" name="kotlin-stdlib" level="project" />
    <orderEntry type="module" module-name="intellij.python.sdk" />
@@ -33,5 +34,6 @@
    <orderEntry type="module" module-name="intellij.platform.testFramework" scope="TEST" />
    <orderEntry type="module" module-name="intellij.python.psi.impl" />
    <orderEntry type="module" module-name="intellij.python.community.execService.python" />
+    <orderEntry type="module" module-name="intellij.platform.testFramework.junit5.eel" scope="TEST" />
  </component>
 </module>
@@ -13,6 +13,10 @@
  <extensions defaultExtensionNs="com.intellij">
    <postStartupActivity implementation="com.intellij.python.community.services.systemPython.impl.SystemPythonInitialLoader"/>
  </extensions>
+  <extensionPoints>
+    <extensionPoint dynamic="false" qualifiedName="com.intellij.python.community.services.systemPython.impl.eelFilter"
+                    interface="com.intellij.python.community.services.systemPython.impl.EelDescriptorFilter"/>
+  </extensionPoints>
  <extensions defaultExtensionNs="Pythonid">
    <systemPythonProvider implementation="com.intellij.python.community.services.systemPython.impl.providers.LegacySystemPythonProvider"/>

@@ -28,7 +32,7 @@
    <systemPythonProvider implementation="com.intellij.python.community.services.systemPython.impl.providers.PyenvSystemPythonProvider"/>

    <systemPythonProvider implementation="com.intellij.python.community.services.systemPython.impl.providers.UnixSystemPythonProvider"
-                          os="unix"/>
+                          /> <!-- we need it even on Windows for WSL/Docker/ssh -->

  </extensions>
 </idea-plugin>
@@ -17,11 +17,21 @@ import kotlin.time.Duration
 * If cache hasn't been calculated yet, this function suspends till [getDataForCache].
 * Cache could be cleared with [clear].
 *
- * Once [startUpdate] is called (which is idempotent call) cache will be updarted on [scope] every [updateInterval].
+ * Once [startUpdate] is called (which is idempotent call) cache will be updated on [scope] every [delayer].
 *
 * Cache isn't blocked while updating: a previous version is used.
 */
-internal class Cache<K, V>(scope: CoroutineScope, updateInterval: Duration, private val getDataForCache: suspend (K) -> List<V>) {
+internal class Cache<K : Any, V>(
+  scope: CoroutineScope,
+  delayer: UpdateCacheDelayer,
+  private val getDataForCache: suspend (K) -> List<V>,
+) {
+  constructor(
+    scope: CoroutineScope,
+    updateInterval: Duration,
+    getDataForCache: suspend (K) -> List<V>,
+  ) : this(scope, UpdateCacheDelayer.TimeBased(updateInterval), getDataForCache)
+
  private companion object {
    val logger = fileLogger()
  }
@@ -35,16 +45,24 @@ internal class Cache<K, V>(scope: CoroutineScope, updateInterval: Duration, priv
  private val cacheUpdateJob = lazy {
    scope.launch(Dispatchers.Default) {
      while (true) {
-        delay(updateInterval)
-        logger.debug("Updating cache")
-        val values = cache.keys.toList() // copy keys not to affect the current cache
-        for (k in values) {
-          updateCache(k)
+        val callAfterUpdate = delayer.delayBeforeUpdate()
+        updateCacheForAllKeys()
+        if (callAfterUpdate != null) {
+          callAfterUpdate()
        }
      }
    }
  }

+  private suspend fun updateCacheForAllKeys() {
+    val keys = cache.keys
+    logger.debug("Updating cache for keys ${keys.joinToString(", ")}")
+    val values = keys.toList() // copy keys not to affect the current cache
+    for (k in values) {
+      updateCache(k)
+    }
+  }
+
  /**
   * Updates cache and suspends till finished
   */
@@ -95,4 +113,20 @@ internal class Cache<K, V>(scope: CoroutineScope, updateInterval: Duration, priv
    logger.info("End update for $k")
    return newValue
  }
+}
+
+/**
+ * [Cache] calls [delayBeforeUpdate] in a loop and updates cache after each call.
+ * [TimeBased] is an implementation that uses [delay].
+ *
+ * Returned value (if any) is called after each cache update
+ */
+internal fun interface UpdateCacheDelayer {
+  suspend fun delayBeforeUpdate(): (() -> Unit)?
+  class TimeBased(private val duration: Duration) : UpdateCacheDelayer {
+    override suspend fun delayBeforeUpdate(): (() -> Unit)? {
+      delay(duration)
+      return null
+    }
+  }
 }
@@ -0,0 +1,23 @@
+// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.community.services.systemPython.impl
+
+import com.intellij.openapi.extensions.ExtensionPointName
+import com.intellij.platform.eel.EelDescriptor
+
+/**
+ * Control various parameters for [com.intellij.python.community.services.systemPython.SystemPythonServiceImpl] based
+ * on [EelDescriptor], **not** a public SPI
+ */
+interface EelDescriptorFilter {
+
+  companion object {
+    internal val EP: ExtensionPointName<EelDescriptorFilter> =
+      ExtensionPointName("com.intellij.python.community.services.systemPython.impl.eelFilter")
+    internal val EelDescriptor.isEphemeral: Boolean get() = EP.extensionList.any { it.isEphemeral(this) }
+  }
+
+  /**
+   * Ephemeral [eelDescriptor] should never be cached nor persisted in user settings
+   */
+  fun isEphemeral(eelDescriptor: EelDescriptor): Boolean
+}
@@ -44,7 +44,7 @@ internal class AsdfSystemPythonProvider : SystemPythonProvider {
        val paths = entries
          .map { versionsDir.resolve(it).resolve("bin").asNioPath() }

-        return@withContext collectPythonsInPaths(eelApi, paths, listOf(python3NamePattern))
+        return@withContext collectPythonsInPaths( paths, listOf(python3NamePattern))
      }
      catch (e: RuntimeException) {
        if (Logger.shouldRethrow(e)) throw e
@@ -25,7 +25,7 @@ internal class BrewSystemPythonProvider : SystemPythonProvider {

    val pythons = withContext(Dispatchers.IO) {
      try {
-        return@withContext collectPythonsInPaths(eelApi, listOf(binDirectory), listOf(python3XNamePattern))
+        return@withContext collectPythonsInPaths( listOf(binDirectory), listOf(python3XNamePattern))
      }
      catch (e: RuntimeException) {
        LOGGER.error("failed to discover brew pythons", e)
@@ -39,7 +39,7 @@ internal class MacSystemPythonProvider : SystemPythonProvider {

    val pythons = withContext(Dispatchers.IO) {
      try {
-        return@withContext collectPythonsInPaths(eelApi, directories, names)
+        return@withContext collectPythonsInPaths( directories, names)
      }
      catch (e: RuntimeException) {
        LOGGER.error("Failed to discover mac system pythons", e)
@@ -44,7 +44,7 @@ internal class PyenvSystemPythonProvider : SystemPythonProvider {
        val paths = entries
          .map { versionsDir.resolve(it).resolve("bin").asNioPath() }

-        return@withContext collectPythonsInPaths(eelApi, paths, listOf(python3NamePattern))
+        return@withContext collectPythonsInPaths( paths, listOf(python3NamePattern))
      }
      catch (e: RuntimeException) {
        LOGGER.error("failed to discover pyenv pythons", e)
@@ -5,21 +5,21 @@ import com.intellij.openapi.diagnostic.Logger
 import com.intellij.platform.eel.EelApi
 import com.intellij.platform.eel.EelPlatform
 import com.intellij.platform.eel.isMac
+import com.intellij.platform.eel.provider.utils.Path
 import com.intellij.python.community.services.systemPython.SystemPythonProvider
 import com.jetbrains.python.PyToolUIInfo
 import com.jetbrains.python.PythonBinary
 import com.jetbrains.python.errorProcessing.PyResult
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
-import java.nio.file.Path


 internal class UnixSystemPythonProvider : SystemPythonProvider {
  private val LOGGER: Logger = Logger.getInstance(UnixSystemPythonProvider::class.java)

-  private val directories = listOf(
-    Path.of("/usr/bin"),
-    Path.of("/usr/local/bin"))
+  private val directories = arrayOf(
+    "/usr/bin",
+    "/usr/local/bin")

  // Patterns to match Python executable filenames
  private val names = listOf(
@@ -36,7 +36,7 @@ internal class UnixSystemPythonProvider : SystemPythonProvider {

    val pythons = withContext(Dispatchers.IO) {
      try {
-        return@withContext collectPythonsInPaths(eelApi, directories, names)
+        return@withContext collectPythonsInPaths(directories.map { Path(it, eelApi.descriptor) }, names)
      }
      catch (e: RuntimeException) {
        LOGGER.error("Failed to discover unix system pythons", e)
@@ -1,16 +1,17 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.community.services.systemPython.impl.providers

+import com.intellij.openapi.diagnostic.fileLogger
 import com.intellij.openapi.util.registry.Registry
-import com.intellij.platform.eel.EelApi
-import com.intellij.platform.eel.fs.EelFileSystemApi.StatError
-import com.intellij.platform.eel.fs.stat
-import com.intellij.platform.eel.getOrNull
-import com.intellij.platform.eel.path.EelPath
-import com.intellij.platform.eel.provider.asNioPath
+import com.jetbrains.python.venvReader.Directory
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import java.io.IOException
+import java.nio.file.FileSystemNotFoundException
+import java.nio.file.NotDirectoryException
 import java.nio.file.Path
 import java.util.regex.Pattern
-import kotlin.io.path.pathString
+import kotlin.io.path.listDirectoryEntries


 internal val pypyNamePattern: Pattern = Pattern.compile("pypy$")
@@ -21,23 +22,28 @@ internal fun useLegacyPythonProvider(): Boolean {
  return Registry.`is`("python.use.system.legacy.provider")
 }

-internal suspend fun collectPythonsInPaths(eelApi: EelApi, paths: List<Path>, names: List<Pattern>): Set<Path> {
-  val pythons = mutableSetOf<Path>()
-
-  for (path in paths) {
-    val directory = EelPath.parse(path.pathString, eelApi.descriptor)
-    if (eelApi.fs.stat(directory).eelIt() is StatError) {
-      continue
-    }
-
-    val entries = eelApi.fs.listDirectory(directory)
-      .getOrNull()
-
-    entries
-      ?.map { directory.resolve(it).asNioPath() }
-      ?.filter { names.firstOrNull { name -> name.matcher(it.fileName.toString()).matches() } != null }
-      ?.let { pythons.addAll(it) }
+internal suspend fun collectPythonsInPaths(paths: List<Directory>, names: List<Pattern>): Set<Path> =
+  withContext(Dispatchers.IO) {
+    paths
+      .flatMap {
+        try {
+          it.listDirectoryEntries()
+        }
+        catch (_: NotDirectoryException) {
+          emptyList()
+        }
+        catch (e: FileSystemNotFoundException) {
+          // This is a temporary hack: Eel might throw this exception in tests when fs gets deregistered
+          // it will be removed as soon as we arrange it with eel
+          logger.warn("Path $it filesystem is inaccessible", e)
+          emptyList()
+        }
+        catch (e: IOException) {
+          logger.warn("Path $it is inaccessible", e)
+          emptyList()
+        }
+      }
+      .filter { child -> names.any { it.matcher(child.fileName.toString()).matches() } }.toSet()
  }

-  return pythons
-}
+private val logger = fileLogger()
@@ -1,6 +1,7 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.community.services.systemPython

+import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.EDT
 import com.intellij.openapi.components.BaseState
 import com.intellij.openapi.components.RoamingType
@@ -20,7 +21,9 @@ import com.intellij.python.community.impl.installer.PySdkToInstallManager
 import com.intellij.python.community.services.internal.impl.VanillaPythonWithPythonInfoImpl
 import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl.MyServiceState
 import com.intellij.python.community.services.systemPython.impl.Cache
+import com.intellij.python.community.services.systemPython.impl.EelDescriptorFilter.Companion.isEphemeral
 import com.intellij.python.community.services.systemPython.impl.PySystemPythonBundle
+import com.intellij.python.community.services.systemPython.impl.UpdateCacheDelayer
 import com.intellij.python.community.services.systemPython.impl.asSysPythonRegisterError
 import com.jetbrains.python.NON_INTERACTIVE_ROOT_TRACE_CONTEXT
 import com.jetbrains.python.PyToolUIInfo
@@ -29,6 +32,7 @@ import com.jetbrains.python.Result
 import com.jetbrains.python.errorProcessing.getOr
 import com.jetbrains.python.getOrNull
 import com.jetbrains.python.sdk.installer.installBinary
+import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -58,16 +62,24 @@ internal suspend fun getCacheTimeout(): Duration? =
@State(name = "SystemPythonService", storages = [Storage("systemPythonService.xml", roamingType = RoamingType.LOCAL)],
       allowLoadInTests = true)
@Internal
-class SystemPythonServiceImpl(scope: CoroutineScope) : SystemPythonService,
-                                                       SimplePersistentStateComponent<MyServiceState>(MyServiceState()) {
+class SystemPythonServiceImpl internal constructor(
+  scope: CoroutineScope,
+  createUpdateCacheDelayer: suspend () -> UpdateCacheDelayer?,
+) : SystemPythonService,
+    SimplePersistentStateComponent<MyServiceState>(MyServiceState()) {
+  constructor(scope: CoroutineScope) : this(scope, {
+    val duration = getCacheTimeout()
+    if (duration != null) UpdateCacheDelayer.TimeBased(duration) else null
+  })
+
  private val findPythonsMutex = Mutex()
  private val _cacheImpl: CompletableDeferred<Cache<EelDescriptor, SystemPython>?> = CompletableDeferred()
  private suspend fun cache() = _cacheImpl.await()

  init {
    scope.launch {
-      _cacheImpl.complete(getCacheTimeout()?.let { interval ->
-        Cache<EelDescriptor, SystemPython>(scope, interval) { eelDescriptor ->
+      _cacheImpl.complete(createUpdateCacheDelayer()?.let { delayer ->
+        Cache(scope, delayer) { eelDescriptor ->
          withContext(NON_INTERACTIVE_ROOT_TRACE_CONTEXT) {
            searchPythonsPhysicallyNoCache(eelDescriptor.toEelApi())
          }
@@ -81,26 +93,34 @@ class SystemPythonServiceImpl(scope: CoroutineScope) : SystemPythonService,
      .getOr(PySystemPythonBundle.message("py.system.python.service.python.is.broken",
                                          pythonPath)) { return Result.failure(it.error.asSysPythonRegisterError()) }
    val systemPython = SystemPython.create(pythonWithLangLevel, null).getOr { return it }
-    state.userProvidedPythons.add(pythonPath.pathString)
-    cache()?.get(pythonPath.getEelDescriptor())?.add(systemPython)
+
+    val eelDescriptor = pythonPath.getEelDescriptor()
+    if (!eelDescriptor.isEphemeral) {
+      state.userProvidedPythons.add(pythonPath.pathString)
+      logger.debug("Registering $pythonPath")
+      cache()?.get(eelDescriptor)?.add(systemPython)
+    }
    return Result.success(systemPython)
  }

  override fun getInstaller(eelApi: EelApi): PythonInstallerService? =
    if (eelApi == localEel) LocalPythonInstaller else null

-  override suspend fun findSystemPythons(eelApi: EelApi, forceRefresh: Boolean): List<SystemPython> =
-    cache()?.let { cache ->
+  override suspend fun findSystemPythons(eelApi: EelApi, forceRefresh: Boolean): List<SystemPython> {
+    val eelDescriptor = eelApi.descriptor
+    val cache = if (!eelDescriptor.isEphemeral) cache() else null
+    return cache?.let { cache ->
      // Cache enabled
      cache.startUpdate()
      if (forceRefresh) {
        logger.info("pythons refresh requested")
-        cache.updateCache(eelApi.descriptor) // Update cache and suspend till update finished
+        cache.updateCache(eelDescriptor) // Update cache and suspend till update finished
      }
      else {
-        cache.get(eelApi.descriptor)
+        cache.get(eelDescriptor)
      }.sortedSystemPythons()
    } ?: searchPythonsPhysicallyNoCache(eelApi).sortedSystemPythons()
+  }

  private fun Iterable<SystemPython>.sortedSystemPythons(): List<SystemPython> =
    sortedWith(
@@ -184,4 +204,4 @@ private object LocalPythonInstaller : PythonInstallerService {
    }
    return Result.Companion.success(Unit)
  }
-}
+}
@@ -1,6 +1,9 @@
 <idea-plugin>
  <dependencies>
    <module name="intellij.python.community.services.systemPython"/>
+    <module name="intellij.platform.testFramework.junit5.eel._test"/>
+    <module name="intellij.platform.testFramework.junit5._test"/>
+    <module name="intellij.python.community.junit5Tests.framework._test"/>
  </dependencies>
  <extensions defaultExtensionNs="com.intellij">
    <applicationInitializedListener implementation="com.intellij.python.junit5Tests.env.systemPython.impl.SystemPythonRootsFixer"/>
@@ -0,0 +1,57 @@
+// Copyright 2000-2026 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.junit5Tests.env.systemPython.impl
+
+import com.intellij.platform.testFramework.junit5.eel.params.api.DockerTest
+import com.intellij.platform.testFramework.junit5.eel.params.api.EelHolder
+import com.intellij.platform.testFramework.junit5.eel.params.api.EelSource
+import com.intellij.platform.testFramework.junit5.eel.params.api.TestApplicationWithEel
+import com.intellij.platform.testFramework.junit5.eel.params.api.WslTest
+import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl
+import com.intellij.python.community.services.systemPython.impl.UpdateCacheDelayer
+import com.intellij.python.junit5Tests.framework.applicationScope
+import com.intellij.testFramework.common.timeoutRunBlocking
+import kotlinx.coroutines.sync.Mutex
+import org.junit.jupiter.api.AfterAll
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.condition.OS
+import org.junit.jupiter.params.ParameterizedTest
+import kotlin.time.Duration.Companion.seconds
+
+@TestApplicationWithEel(osesMayNotHaveRemoteEels = [OS.WINDOWS])
+internal class SystemPythonServiceStaleDescriptorTest {
+  companion object {
+    private lateinit var sut: SystemPythonServiceImpl
+    private val startUpdate = Mutex(locked = true)
+    private val updateFinished = Mutex(locked = true)
+    private val scope = applicationScope("${SystemPythonServiceStaleDescriptorTest}")
+
+    @BeforeAll
+    @JvmStatic
+    fun setUp() {
+      sut = SystemPythonServiceImpl(scope.get()) {
+        UpdateCacheDelayer {
+          startUpdate.lock() // Start cache update when lock is opened
+          ({
+            updateFinished.unlock() // this lock will be opened as soon as cache updated
+          })
+        }
+      }
+    }
+
+    @AfterAll
+    @JvmStatic
+    fun makeSureNoEelLeaked(): Unit = timeoutRunBlocking(30.seconds) {
+      startUpdate.unlock() // Start cache update
+      updateFinished.lock() // Wait for its end
+    }
+  }
+
+  @ParameterizedTest
+  @EelSource
+  @DockerTest("python:3.14.2-trixie", mandatory = false)
+  @WslTest("Ubuntu-22.04", mandatory = false)
+  fun testPythonOnDocker(eelHolder: EelHolder): Unit = timeoutRunBlocking {
+    Assertions.assertTrue(sut.findSystemPythons(eelHolder.eel).isNotEmpty(), "No pythons found on ${eelHolder.eel}")
+  }
+}