Python: Cache system pythons to speed up a function result.

1. Preload pythons from a project start activity
2. Cache them for some time (see `cache` variable)
3. Provide API to refresh


Merge-request: IJ-MR-158389
Merged-by: Ilya Kazakevich <ilya.kazakevich@jetbrains.com>

GitOrigin-RevId: 8b58ad3f35f144364d4103578d20a3cfc9b637f2

python/ide/impl/src/com/intellij/pycharm/community/ide/impl/miscProject/impl/lib.kt

@@ -219,7 +219,7 @@ private suspend fun getSystemPython(confirmInstallation: suspend () -> Boolean,
         }
         is Result.Success -> {
           // Find the latest python again, after installation
-          systemPythonBinary = pythonService.findSystemPythons().firstOrNull()
+          systemPythonBinary = pythonService.findSystemPythons(forceRefresh = true).firstOrNull()
         }
       }
     }

python/pluginCore/resources/META-INF/plugin.xml

@@ -61,7 +61,8 @@ The Python plug-in provides smart editing for Python scripts. The feature set of
   <extensions defaultExtensionNs="com.intellij">
     <localInspection language="TOML" enabledByDefault="true" implementationClass="com.jetbrains.python.sdk.poetry.PoetryPackageVersionsInspection" key="INSP.poetry.package.versions.display.name" bundle="messages.PyBundle" groupKey="INSP.GROUP.python" suppressId="PoetryPackageVersions" shortName="PoetryPackageVersionsInspection"/>
     <localInspection language="TOML" enabledByDefault="true" implementationClass="com.jetbrains.python.sdk.uv.UvPackageVersionsInspection" key="INSP.poetry.package.versions.display.name" bundle="messages.PyBundle" groupKey="INSP.GROUP.python" suppressId="UvPackageVersions" shortName="UvPackageVersionsInspection"/>
-
+    <registryKey defaultValue="10" description="How ofter system pythons must be refreshed (minutes)" key="python.system.refresh.minutes"/>
+    <postStartupActivity implementation="com.intellij.python.community.services.systemPython.impl.SystemPythonInitialLoader"/>
     <fileType name="Requirements.txt"
               implementationClass="com.jetbrains.python.requirements.RequirementsFileType"
               fieldName="INSTANCE"

python/services/system-python/src/com/intellij/python/community/services/systemPython/api.kt

@@ -18,9 +18,10 @@ import javax.swing.Icon
 @ApiStatus.NonExtendable
 sealed interface SystemPythonService {
   /**
+   * The result of this function might be cached. Use [forceRefresh] to reload it forcibly.
    * @return system pythons installed on OS sorted by type, then by lang.level: in order from highest (hence, the first one is usually the best one)
    */
-  suspend fun findSystemPythons(eelApi: EelApi = localEel): List<SystemPython>
+  suspend fun findSystemPythons(eelApi: EelApi = localEel, forceRefresh: Boolean = false): List<SystemPython>
 
   /**
    * When user provides a path to the python binary, use this method to the [SystemPython].

python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt

@@ -0,0 +1,98 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.community.services.systemPython.impl
+
+import com.intellij.openapi.diagnostic.fileLogger
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import java.util.concurrent.ConcurrentHashMap
+import kotlin.time.Duration
+
+/**
+ * [K]->[[V]] cache. List of [V] is calculated by [getDataForCache].
+ * [get] returns a mutable list, so you can add items there.
+ * If cache hasn't been calculated yet, this function suspends till [getDataForCache].
+ * Cache could be cleared with [clear].
+ *
+ * Once [startUpdate] is called (which is idempotent call) cache will be updarted on [scope] every [updateInterval].
+ *
+ * Cache isn't blocked while updating: a previous version is used.
+ */
+internal class Cache<K, V>(scope: CoroutineScope, updateInterval: Duration, private val getDataForCache: suspend (K) -> List<V>) {
+  private companion object {
+    val logger = fileLogger()
+  }
+
+  private val cacheUpdateMutex = Mutex()
+  private val cache = ConcurrentHashMap<K, MutableSet<V>>()
+
+  /**
+   * Laziness prevents postpones a job from creation till first [startUpdate] so a client might decide not to create a job at all
+   */
+  private val cacheUpdateJob = lazy {
+    scope.launch(Dispatchers.Default) {
+      while (true) {
+        delay(updateInterval)
+        logger.debug("Updating cache")
+        val values = cache.keys.toList() // copy keys not to affect the current cache
+        for (k in values) {
+          updateCache(k)
+        }
+      }
+    }
+  }
+
+  /**
+   * Updates cache and suspends till finished
+   */
+  suspend fun updateCache(k: K): Set<V> = cacheUpdateMutex.withLock {
+    updateCacheInternal(k)
+  }
+
+  /**
+   * Starts cache-in-background-updating process (does nothing if started already)
+   */
+  fun startUpdate() {
+    cacheUpdateJob.value
+  }
+
+  /**
+   * Removes cache for [k]. Next [get] might suspend (if a background process wouldn't fill at in advance)
+   */
+  fun clear(k: K) {
+    cache.remove(k)
+    logger.info("Cache flushed for $k")
+  }
+
+  /**
+   * Returns current cache for [k], might suspend for a while if cache hasn't been filled yet.
+   */
+  suspend fun get(k: K): MutableCollection<V> {
+    cache[k]?.let { return it } // Return current cache
+    // No need to update the cache in several coroutines
+    cacheUpdateMutex.withLock {
+      // It could be that previous coroutine updated cache already (this is why we were blocked), so check again.
+      // Kinda Double-checked locking
+      cache[k]?.let { return it }
+      logger.info("No cache for $k, will fill")
+      return updateCacheInternal(k)
+    }
+  }
+
+  /**
+   * Always call under [cacheUpdateMutex].
+   * This function can't have mutex itself because double-checking locking is used in [get] but not in [cacheUpdateJob]
+   */
+  private suspend fun updateCacheInternal(k: K): MutableSet<V> {
+    logger.info("Starting update for $k")
+    val data = getDataForCache(k)
+    val newValue = ConcurrentHashMap.newKeySet<V>(data.size)
+    newValue.addAll(data)
+    cache[k] = newValue
+    logger.info("End update for $k")
+    return newValue
+  }
+}

python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/CoreSystemPythonProvider.kt

@@ -0,0 +1,26 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.community.services.systemPython.impl
+
+import com.intellij.platform.eel.EelApi
+import com.intellij.platform.eel.provider.localEel
+import com.intellij.python.community.services.systemPython.SystemPythonProvider
+import com.jetbrains.python.PythonBinary
+import com.jetbrains.python.sdk.flavors.PythonSdkFlavor
+import kotlinx.coroutines.Dispatchers
+
+/**
+ * [SystemPythonProvider] based ob [PythonSdkFlavor] (kind of a bridge)
+ */
+internal object CoreSystemPythonProvider : SystemPythonProvider {
+  override suspend fun findSystemPythons(eelApi: EelApi): Result<Set<PythonBinary>> {
+    if (eelApi != localEel) return Result.success(emptySet())
+    with(Dispatchers.IO) {
+      val paths = PythonSdkFlavor.getApplicableFlavors(false)
+        .flatMap {
+          it.dropCaches()
+          it.suggestLocalHomePaths(null, null)
+        }
+      return Result.success(paths.toSet())
+    }
+  }
+}

python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/SystemPythonInitialLoader.kt

@@ -0,0 +1,20 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.community.services.systemPython.impl
+
+import com.intellij.openapi.diagnostic.fileLogger
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.startup.ProjectActivity
+import com.intellij.platform.eel.provider.getEelDescriptor
+import com.intellij.python.community.services.systemPython.SystemPythonService
+import com.intellij.python.community.services.systemPython.cacheTimeout
+
+private val logger = fileLogger()
+
+// Preload pythons as soon as a project gets loaded
+internal class SystemPythonInitialLoader : ProjectActivity {
+  override suspend fun execute(project: Project) {
+    if (cacheTimeout == null) return // Cache is disabled, no need to preload it
+    logger.debug("Preloading pythons for $project")
+    SystemPythonService().findSystemPythons(project.getEelDescriptor().upgrade())
+  }
+}

python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/package-info.java

@@ -0,0 +1,5 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+@ApiStatus.Internal
+package com.intellij.python.community.services.systemPython.impl;
+
+import org.jetbrains.annotations.ApiStatus;

python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt

@@ -5,84 +5,134 @@ import com.intellij.openapi.application.EDT
 import com.intellij.openapi.components.*
 import com.intellij.openapi.components.Service.Level.APP
 import com.intellij.openapi.diagnostic.fileLogger
+import com.intellij.openapi.util.registry.Registry
 import com.intellij.platform.eel.EelApi
+import com.intellij.platform.eel.EelDescriptor
 import com.intellij.platform.eel.provider.getEelDescriptor
 import com.intellij.platform.eel.provider.localEel
 import com.intellij.python.community.impl.installer.PySdkToInstallManager
 import com.intellij.python.community.services.internal.impl.PythonWithLanguageLevelImpl
 import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl.MyServiceState
+import com.intellij.python.community.services.systemPython.impl.Cache
+import com.intellij.python.community.services.systemPython.impl.CoreSystemPythonProvider
 import com.jetbrains.python.PythonBinary
 import com.jetbrains.python.Result
-import com.jetbrains.python.sdk.flavors.PythonSdkFlavor
 import com.jetbrains.python.sdk.installer.installBinary
+import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
 import org.jetbrains.annotations.ApiStatus.Internal
 import org.jetbrains.annotations.Nls
+import java.nio.file.InvalidPathException
+import java.nio.file.Path
+import kotlin.io.path.pathString
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.minutes
 
 
-// Implementation
+private val logger = fileLogger()
+
+// null means "disabled"
+internal val cacheTimeout: Duration?
+  get() = Registry.get("python.system.refresh.minutes").asInteger().let { i ->
+    if (i > 0) i.minutes else null
+  }
+
 
 @Service(APP)
 @State(name = "SystemPythonService", storages = [Storage("systemPythonService.xml", roamingType = RoamingType.LOCAL)],
        allowLoadInTests = true)
 @Internal
-internal class SystemPythonServiceImpl : SystemPythonService, SimplePersistentStateComponent<MyServiceState>(MyServiceState()) {
+internal class SystemPythonServiceImpl(scope: CoroutineScope) : SystemPythonService, SimplePersistentStateComponent<MyServiceState>(MyServiceState()) {
+  private val findPythonsMutex = Mutex()
+  private val cache: Cache<EelDescriptor, SystemPython>? = cacheTimeout?.let { interval ->
+    Cache(scope, interval) { eelDescriptor ->
+      searchPythonsPhysicallyNoCache(eelDescriptor.upgrade())
+    }
+  }
+
 
   override suspend fun registerSystemPython(pythonPath: PythonBinary): Result<SystemPython, @Nls String> {
-    val impl = PythonWithLanguageLevelImpl.createByPythonBinary(pythonPath).getOr { return it }
-    state.userProvidedPythons.add(pythonPath)
-    return Result.success(SystemPython(impl, null))
+    val pythonWithLangLevel = PythonWithLanguageLevelImpl.createByPythonBinary(pythonPath).getOr { return it }
+    val systemPython = SystemPython(pythonWithLangLevel, null)
+    state.userProvidedPythons.add(pythonPath.pathString)
+    cache?.get(pythonPath.getEelDescriptor())?.add(systemPython)
+    return Result.success(systemPython)
   }
 
   override fun getInstaller(eelApi: EelApi): PythonInstallerService? =
     if (eelApi == localEel) LocalPythonInstaller else null
 
-  override suspend fun findSystemPythons(eelApi: EelApi): List<SystemPython> = withContext(Dispatchers.IO) {
-    val corePythons = if (eelApi == localEel)
-      PythonSdkFlavor.getApplicableFlavors(false)
-        .flatMap {
-          it.dropCaches()
-          it.suggestLocalHomePaths(null, null)
+  override suspend fun findSystemPythons(eelApi: EelApi, forceRefresh: Boolean): List<SystemPython> =
+    if (cache != null) {
+      // Cache enabled
+      cache.startUpdate()
+      if (forceRefresh) {
+        logger.info("pythons refresh requested")
+        cache.updateCache(eelApi.descriptor) // Update cache and suspend till update finished
+      }
+      else {
+        cache.get(eelApi.descriptor)
+      }.sorted()
+    }
+    else {
+      // Cache disabled
+      searchPythonsPhysicallyNoCache(eelApi)
+    }
+
+  class MyServiceState : BaseState() {
+    // Only strings are supported by serializer
+    var userProvidedPythons by list<String>()
+    val userProvidedPythonsAsPath: Collection<Path>
+      get() = userProvidedPythons.filterNotNull().mapNotNull {
+        try {
+          Path.of(it)
         }
-    else emptyList()
-
-    val pythonsUi = mutableMapOf<PythonBinary, UICustomization>()
-
-    val pythonsFromExtensions = SystemPythonProvider.EP
-      .extensionList
-      .flatMap { provider ->
-        val pythons = provider.findSystemPythons(eelApi).getOrNull() ?: emptyList()
-        val ui = provider.uiCustomization
-        if (ui != null) {
-          pythons.forEach { pythonsUi[it] = ui }
+        catch (_: InvalidPathException) {
+          logger.warn("invalid path $it")
+          null
         }
-        pythons
-      }.filter { it.getEelDescriptor().upgrade() == eelApi }
-
-    val badPythons = mutableSetOf<PythonBinary>()
-    val pythons = corePythons + pythonsFromExtensions + state.userProvidedPythons.filter { it.getEelDescriptor() == eelApi.descriptor }
-
-    val result = PythonWithLanguageLevelImpl.createByPythonBinaries(pythons.toSet())
-      .mapNotNull { (python, r) ->
-        when (r) {
-          is Result.Success -> SystemPython(r.result, pythonsUi[r.result.pythonBinary])
-          is Result.Failure -> {
-            fileLogger().info("Skipping $python : ${r.error}")
-            badPythons.add(python)
-            null
-          }
-        }
-
-      }.toSet()
-    // Remove stale pythons from cache
-    state.userProvidedPythons.removeAll(badPythons)
-    return@withContext result.sorted()
+      }
   }
 
 
-  class MyServiceState : BaseState() {
-    val userProvidedPythons: MutableCollection<PythonBinary> by list()
+  private suspend fun searchPythonsPhysicallyNoCache(eelApi: EelApi): List<SystemPython> = withContext(Dispatchers.IO) {
+    findPythonsMutex.withLock {
+      val pythonsUi = mutableMapOf<PythonBinary, UICustomization>()
+
+      val pythonsFromExtensions = (SystemPythonProvider.EP
+                                     .extensionList + listOf(CoreSystemPythonProvider))
+        .flatMap { provider ->
+          val pythons = provider.findSystemPythons(eelApi).getOrNull() ?: emptyList()
+          val ui = provider.uiCustomization
+          if (ui != null) {
+            pythons.forEach { pythonsUi[it] = ui }
+          }
+          pythons
+        }
+
+      val badPythons = mutableSetOf<PythonBinary>()
+      val pythons = pythonsFromExtensions + state.userProvidedPythonsAsPath.filter { it.getEelDescriptor() == eelApi.descriptor }
+
+      val result = PythonWithLanguageLevelImpl.createByPythonBinaries(pythons.toSet())
+        .mapNotNull { (python, r) ->
+          when (r) {
+            is Result.Success -> SystemPython(r.result, pythonsUi[r.result.pythonBinary])
+            is Result.Failure -> {
+              fileLogger().warn("Skipping $python : ${r.error}")
+              badPythons.add(python)
+              null
+            }
+          }
+
+        }.toSet()
+      // Remove stale pythons from the cache
+      state.userProvidedPythons.removeAll(badPythons.map { it.pathString })
+      logger.info("pythons refreshed")
+      return@withContext result.sorted()
+    }
   }
 }
 
@@ -97,8 +147,8 @@ private object LocalPythonInstaller : PythonInstallerService {
       installBinary(pythonToInstall, null) {
       }
     }.getOrElse {
-      return Result.failure(it.message ?: it.toString())
+      return Result.Companion.failure(it.message ?: it.toString())
     }
-    return Result.success(Unit)
+    return Result.Companion.success(Unit)
   }
 }

python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/SystemPythonServiceShowCaseTest.kt

@@ -1,6 +1,8 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.junit5Tests.env.systemPython
 
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.diagnostic.fileLogger
 import com.intellij.openapi.util.SystemInfo
 import com.intellij.platform.eel.executeProcess
@@ -8,16 +10,24 @@ import com.intellij.platform.eel.getOrThrow
 import com.intellij.platform.eel.provider.getEelDescriptor
 import com.intellij.platform.eel.provider.utils.readWholeText
 import com.intellij.python.community.impl.venv.createVenv
+import com.intellij.python.community.services.systemPython.SystemPythonProvider
 import com.intellij.python.community.services.systemPython.SystemPythonService
+import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl
 import com.intellij.python.junit5Tests.assertFail
 import com.intellij.python.junit5Tests.framework.env.PyEnvTestCase
 import com.intellij.python.junit5Tests.framework.env.PythonBinaryPath
 import com.intellij.python.junit5Tests.framework.winLockedFile.deleteCheckLocking
 import com.intellij.python.junit5Tests.randomBinary
 import com.intellij.testFramework.common.timeoutRunBlocking
+import com.intellij.testFramework.junit5.RegistryKey
+import com.intellij.testFramework.junit5.TestDisposable
+import com.intellij.testFramework.registerExtension
 import com.jetbrains.python.getOrThrow
 import com.jetbrains.python.sdk.flavors.PythonSdkFlavor
 import com.jetbrains.python.venvReader.VirtualEnvReader
+import io.mockk.coEvery
+import io.mockk.coVerify
+import io.mockk.mockk
 import kotlinx.coroutines.async
 import org.hamcrest.MatcherAssert.assertThat
 import org.hamcrest.Matchers.hasItem
@@ -34,7 +44,7 @@ class SystemPythonServiceShowCaseTest {
 
   @Test
   fun testListPythons(): Unit = timeoutRunBlocking {
-    for (systemPython in SystemPythonService().findSystemPythons()) {
+    for (systemPython in SystemPythonService().findSystemPythons(forceRefresh = true)) {
       fileLogger().info("Python found: $systemPython")
       val eelApi = systemPython.pythonBinary.getEelDescriptor().upgrade()
       val process = eelApi.exec.executeProcess(systemPython.pythonBinary.pathString, "--version").getOrThrow()
@@ -61,11 +71,42 @@ class SystemPythonServiceShowCaseTest {
     assertThat("No newly registered python returned", allPythons, hasItem(newPython))
     python.deleteExisting()
 
-    allPythons = SystemPythonService().findSystemPythons()
+    allPythons = SystemPythonService().findSystemPythons(forceRefresh = true)
     assertThat("Broken python returned", allPythons, not(hasItem(newPython)))
 
     if (SystemInfo.isWindows) {
       deleteCheckLocking(venvPath)
     }
   }
+
+  @Test
+  fun testRefresh(@TestDisposable disposable: Disposable): Unit = timeoutRunBlocking {
+    val mockProvider = mockk<SystemPythonProvider>()
+    coEvery { mockProvider.findSystemPythons(any()) } returns Result.failure(java.lang.AssertionError("..."))
+    coEvery { mockProvider.uiCustomization } returns null
+    val sut = SystemPythonService()
+    sut.findSystemPythons()
+    ApplicationManager.getApplication().registerExtension(SystemPythonProvider.EP, mockProvider, disposable)
+    repeat(10) {
+      sut.findSystemPythons()
+    }
+    coVerify(exactly = 0) { mockProvider.findSystemPythons(any()) }
+    sut.findSystemPythons(forceRefresh = true)
+    coVerify(atLeast = 1) { mockProvider.findSystemPythons(any()) }
+  }
+
+  @RegistryKey("python.system.refresh.minutes", "0")
+  @Test
+  fun testDisableCache(@TestDisposable disposable: Disposable): Unit = timeoutRunBlocking {
+    val timesToRepeat = 5
+    val mockProvider = mockk<SystemPythonProvider>()
+    coEvery { mockProvider.findSystemPythons(any()) } returns Result.success(emptySet())
+    coEvery { mockProvider.uiCustomization } returns null
+    val sut = SystemPythonServiceImpl(this)
+    ApplicationManager.getApplication().registerExtension(SystemPythonProvider.EP, mockProvider, disposable)
+    repeat(timesToRepeat) {
+      sut.findSystemPythons()
+    }
+    coVerify(exactly = timesToRepeat) { mockProvider.findSystemPythons(any()) }
+  }
 }

python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/CacheTest.kt

@@ -0,0 +1,93 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.junit5Tests.env.systemPython.impl
+
+import com.intellij.python.community.services.systemPython.impl.Cache
+import com.intellij.testFramework.common.timeoutRunBlocking
+import kotlinx.coroutines.*
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ValueSource
+import java.util.concurrent.atomic.AtomicInteger
+import kotlin.time.Duration.Companion.days
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.seconds
+
+class CacheTest {
+  private val producerCalled = AtomicInteger()
+
+  private fun createSut(scope: CoroutineScope) = Cache<String, String>(scope, 1.milliseconds) {
+    producerCalled.incrementAndGet()
+    val element = "$it-value"
+    listOf(element)
+  }
+
+  @Test
+  fun testCache(): Unit = timeoutRunBlocking {
+    val sut = createSut(this)
+    repeat(20) {
+      Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray())
+    }
+    Assertions.assertEquals(1, producerCalled.get())
+    sut.clear("abc")
+    Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray())
+    sut.clear("foo")
+    Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray())
+    Assertions.assertEquals(2, producerCalled.get())
+  }
+
+  @Test
+  fun testCacheUpdateBackground(): Unit = timeoutRunBlocking {
+    val sut = createSut(this)
+    sut.startUpdate()
+    Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray())
+    val secsToWaitForUpdate = 30
+    repeat(secsToWaitForUpdate) {
+      delay(1.seconds)
+      if (producerCalled.get() > 5) {
+        coroutineContext.job.cancelChildren()
+        return@timeoutRunBlocking
+      }
+    }
+    Assertions.fail("Even after $secsToWaitForUpdate seconds cache hasn't been upgraded in background")
+  }
+
+
+  @ParameterizedTest
+  @ValueSource(booleans = [true, false])
+  fun testCacheAccessibleWhileUpdate(updateByTimer: Boolean): Unit = timeoutRunBlocking {
+    val lock = CompletableDeferred<Unit>()
+    val sut = Cache<String, String>(this, if (updateByTimer) 1.milliseconds else Int.MAX_VALUE.days) {
+      val producerCalledTimes = producerCalled.incrementAndGet()
+      var time = "first"
+      if (producerCalledTimes > 1) {
+        lock.await()
+        time = "not-first"
+      }
+      val element = "$it-$time"
+      listOf(element)
+    }
+    if (updateByTimer) {
+      sut.startUpdate()
+    }
+    else {
+      launch {
+        Assertions.assertArrayEquals(arrayOf("foo-not-first"), sut.updateCache("foo").toTypedArray())
+      }
+    }
+    repeat(20) {
+      Assertions.assertArrayEquals(arrayOf("foo-first"), sut.get("foo").toTypedArray())
+      delay(1.milliseconds)
+    }
+    lock.complete(Unit)
+    val secsToWaitForUpdate = 30
+    repeat(secsToWaitForUpdate) {
+      delay(1.seconds)
+      if (sut.get("foo").iterator().next() == "foo-not-first") {
+        coroutineContext.job.cancelChildren()
+        return@timeoutRunBlocking
+      }
+    }
+    Assertions.fail("Even after $secsToWaitForUpdate seconds cache hasn't been upgraded in background")
+  }
+}

python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/EnvProviderTest.kt

@@ -53,7 +53,7 @@ class EnvProviderTest {
     coEvery { provider.findSystemPythons(any()) } returns Result.success(setOf(venvPython))
     coEvery { provider.uiCustomization } returns ui
     ApplicationManager.getApplication().registerExtension(SystemPythonProvider.EP, provider, disposable)
-    val python = SystemPythonService().findSystemPythons().first { it.pythonBinary == venvPython }
+    val python = SystemPythonService().findSystemPythons(forceRefresh = true).first { it.pythonBinary == venvPython }
     assertEquals(ui, python.ui, "Wrong UI")
   }
 }

python/src/com/jetbrains/python/sdk/add/v2/CustomExistingEnvironmentSelector.kt

@@ -5,13 +5,12 @@ import com.intellij.openapi.module.Module
 import com.intellij.openapi.observable.properties.ObservableMutableProperty
 import com.intellij.openapi.observable.util.notEqualsTo
 import com.intellij.openapi.ui.validation.DialogValidationRequestor
+import com.intellij.python.community.services.shared.PythonWithLanguageLevel
 import com.intellij.ui.dsl.builder.Align
 import com.intellij.ui.dsl.builder.Panel
 import com.jetbrains.python.PyBundle.message
 import com.jetbrains.python.newProject.collector.InterpreterStatisticsInfo
 import com.jetbrains.python.sdk.ModuleOrProject
-import com.jetbrains.python.sdk.PySdkUtil
-import com.jetbrains.python.sdk.PythonSdkUtil
 import com.jetbrains.python.statistics.InterpreterCreationMode
 import com.jetbrains.python.statistics.InterpreterType
 import com.jetbrains.python.errorProcessing.ErrorSink
@@ -80,9 +79,8 @@ internal abstract class CustomExistingEnvironmentSelector(private val name: Stri
                                      InterpreterCreationMode.CUSTOM)
   }
 
-  private fun addEnvByPath(path: String) {
-    val languageLevel = PySdkUtil.getLanguageLevelForSdk(PythonSdkUtil.findSdkByKey(path))
-    val interpreter = ManuallyAddedSelectableInterpreter(path, languageLevel)
+  private fun addEnvByPath(python: PythonWithLanguageLevel) {
+    val interpreter = ManuallyAddedSelectableInterpreter(python)
     existingEnvironments.value += interpreter
   }
 

python/src/com/jetbrains/python/sdk/add/v2/models.kt

@@ -207,7 +207,13 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva
     }
   }
 
-  open fun addInterpreter(path: String): PythonSelectableInterpreter {
+  internal fun addInterpreter(python: PythonWithLanguageLevel): PythonSelectableInterpreter {
+    val interpreter = ManuallyAddedSelectableInterpreter(python)
+    manuallyAddedInterpreters.value += interpreter
+    return interpreter
+  }
+
+  internal fun addInterpreter(path: String): PythonSelectableInterpreter {
     val languageLevel = PySdkUtil.getLanguageLevelForSdk(PythonSdkUtil.findSdkByKey(path))
     val interpreter = ManuallyAddedSelectableInterpreter(path, languageLevel)
     manuallyAddedInterpreters.value += interpreter
@@ -221,7 +227,7 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva
   /**
    * Given [pathToPython] returns either cleaned path (if valid) or null and reports error to [errorSink]
    */
-  suspend fun getSystemPythonFromSelection(pathToPython: String, errorSink: ErrorSink): String? {
+  suspend fun getSystemPythonFromSelection(pathToPython: String, errorSink: ErrorSink): SystemPython? {
     val result = try {
       when (val r = systemPythonService.registerSystemPython(Path(pathToPython))) {
         is com.jetbrains.python.Result.Failure -> com.jetbrains.python.errorProcessing.failure(r.error)
@@ -233,7 +239,7 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva
     }
 
     return when (result) {
-      is com.jetbrains.python.Result.Success -> result.result.pythonBinary.pathString
+      is com.jetbrains.python.Result.Success -> result.result
       is com.jetbrains.python.Result.Failure -> {
         errorSink.emit(result.error)
         null
@@ -369,7 +375,9 @@ class DetectedSelectableInterpreter(override val homePath: String, override val
   }
 }
 
-class ManuallyAddedSelectableInterpreter(override val homePath: String, override val languageLevel: LanguageLevel) : PythonSelectableInterpreter()
+class ManuallyAddedSelectableInterpreter(override val homePath: String, override val languageLevel: LanguageLevel) : PythonSelectableInterpreter() {
+  constructor(python: PythonWithLanguageLevel) : this(python.pythonBinary.pathString, python.languageLevel)
+}
 
 class InstallableSelectableInterpreter(val sdk: PySdkToInstall) : PythonSelectableInterpreter() {
   override suspend fun isBasePython(): Boolean = true

python/src/com/jetbrains/python/sdk/add/v2/uiUtils.kt

@@ -21,6 +21,8 @@ import com.intellij.openapi.util.NlsSafe
 import com.intellij.platform.ide.progress.ModalTaskOwner
 import com.intellij.platform.ide.progress.runWithModalProgressBlocking
 import com.intellij.python.community.impl.installer.CondaInstallManager
+import com.intellij.python.community.services.shared.PythonWithLanguageLevel
+import com.intellij.python.community.services.systemPython.SystemPython
 import com.intellij.ui.AnimatedIcon
 import com.intellij.ui.ColoredListCellRenderer
 import com.intellij.ui.SimpleColoredComponent
@@ -65,6 +67,7 @@ import kotlin.coroutines.CoroutineContext
 import kotlin.io.path.Path
 import kotlin.io.path.exists
 import kotlin.io.path.isDirectory
+import kotlin.io.path.pathString
 
 
 internal fun <T> PropertyGraph.booleanProperty(dependency: ObservableProperty<T>, value: T) =
@@ -240,7 +243,7 @@ class PythonEnvironmentComboBoxRenderer : ColoredListCellRenderer<Any>() {
 internal fun Row.pythonInterpreterComboBox(
   selectedSdkProperty: ObservableMutableProperty<PythonSelectableInterpreter?>, // todo not sdk
   model: PythonAddInterpreterModel,
-  onPathSelected: (String) -> Unit, busyState: StateFlow<Boolean>? = null,
+  onPathSelected: (PythonWithLanguageLevel) -> Unit, busyState: StateFlow<Boolean>? = null,
 ): Cell<PythonInterpreterComboBox> {
 
   val comboBox = PythonInterpreterComboBox(selectedSdkProperty, model, onPathSelected, ShowingMessageErrorSync)
@@ -273,7 +276,7 @@ internal fun Row.pythonInterpreterComboBox(
 internal class PythonInterpreterComboBox(
   private val backingProperty: ObservableMutableProperty<PythonSelectableInterpreter?>,
   val controller: PythonAddInterpreterModel,
-  val onPathSelected: (String) -> Unit,
+  val onPathSelected: (PythonWithLanguageLevel) -> Unit,
   private val errorSink: ErrorSink,
 ) : ComboBox<PythonSelectableInterpreter?>() {
 
@@ -288,7 +291,7 @@ internal class PythonInterpreterComboBox(
     val newOnPathSelected: (String) -> Unit = {
       runWithModalProgressBlocking(ModalTaskOwner.guess(), message("python.sdk.validating.environment")) {
         controller.getSystemPythonFromSelection(it, errorSink)?.let { python ->
-          interpreterToSelect.set(python)
+          interpreterToSelect.set(python.pythonBinary.pathString)
           onPathSelected(python)
         }
       }