From 92e25c9022ab193e775030c3d4307f331e625a13 Mon Sep 17 00:00:00 2001 From: Ilya Kazakevich Date: Thu, 27 Mar 2025 14:31:48 +0000 Subject: [PATCH] Python: Cache system pythons to speed up a function result. 1. Preload pythons from a project start activity 2. Cache them for some time (see `cache` variable) 3. Provide API to refresh Merge-request: IJ-MR-158389 Merged-by: Ilya Kazakevich GitOrigin-RevId: 8b58ad3f35f144364d4103578d20a3cfc9b637f2 --- .../ide/impl/miscProject/impl/lib.kt | 2 +- .../pluginCore/resources/META-INF/plugin.xml | 3 +- .../community/services/systemPython/api.kt | 3 +- .../services/systemPython/impl/Cache.kt | 98 ++++++++++++ .../impl/CoreSystemPythonProvider.kt | 26 ++++ .../impl/SystemPythonInitialLoader.kt | 20 +++ .../systemPython/impl/package-info.java | 5 + .../systemPython/systemPythonServiceImpl.kt | 146 ++++++++++++------ .../SystemPythonServiceShowCaseTest.kt | 45 +++++- .../env/systemPython/impl/CacheTest.kt | 93 +++++++++++ .../env/systemPython/impl/EnvProviderTest.kt | 2 +- .../v2/CustomExistingEnvironmentSelector.kt | 8 +- .../com/jetbrains/python/sdk/add/v2/models.kt | 16 +- .../jetbrains/python/sdk/add/v2/uiUtils.kt | 9 +- 14 files changed, 410 insertions(+), 66 deletions(-) create mode 100644 python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt create mode 100644 python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/CoreSystemPythonProvider.kt create mode 100644 python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/SystemPythonInitialLoader.kt create mode 100644 python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/package-info.java create mode 100644 python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/CacheTest.kt diff --git a/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/miscProject/impl/lib.kt b/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/miscProject/impl/lib.kt index dec8e5b0769b..6d6acf4aec17 100644 --- a/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/miscProject/impl/lib.kt +++ b/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/miscProject/impl/lib.kt @@ -219,7 +219,7 @@ private suspend fun getSystemPython(confirmInstallation: suspend () -> Boolean, } is Result.Success -> { // Find the latest python again, after installation - systemPythonBinary = pythonService.findSystemPythons().firstOrNull() + systemPythonBinary = pythonService.findSystemPythons(forceRefresh = true).firstOrNull() } } } diff --git a/python/pluginCore/resources/META-INF/plugin.xml b/python/pluginCore/resources/META-INF/plugin.xml index a870b0405296..060e1dbb5f15 100644 --- a/python/pluginCore/resources/META-INF/plugin.xml +++ b/python/pluginCore/resources/META-INF/plugin.xml @@ -61,7 +61,8 @@ The Python plug-in provides smart editing for Python scripts. The feature set of - + + + suspend fun findSystemPythons(eelApi: EelApi = localEel, forceRefresh: Boolean = false): List /** * When user provides a path to the python binary, use this method to the [SystemPython]. diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt new file mode 100644 index 000000000000..51cdda6716fe --- /dev/null +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/Cache.kt @@ -0,0 +1,98 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.community.services.systemPython.impl + +import com.intellij.openapi.diagnostic.fileLogger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.util.concurrent.ConcurrentHashMap +import kotlin.time.Duration + +/** + * [K]->[[V]] cache. List of [V] is calculated by [getDataForCache]. + * [get] returns a mutable list, so you can add items there. + * If cache hasn't been calculated yet, this function suspends till [getDataForCache]. + * Cache could be cleared with [clear]. + * + * Once [startUpdate] is called (which is idempotent call) cache will be updarted on [scope] every [updateInterval]. + * + * Cache isn't blocked while updating: a previous version is used. + */ +internal class Cache(scope: CoroutineScope, updateInterval: Duration, private val getDataForCache: suspend (K) -> List) { + private companion object { + val logger = fileLogger() + } + + private val cacheUpdateMutex = Mutex() + private val cache = ConcurrentHashMap>() + + /** + * Laziness prevents postpones a job from creation till first [startUpdate] so a client might decide not to create a job at all + */ + private val cacheUpdateJob = lazy { + scope.launch(Dispatchers.Default) { + while (true) { + delay(updateInterval) + logger.debug("Updating cache") + val values = cache.keys.toList() // copy keys not to affect the current cache + for (k in values) { + updateCache(k) + } + } + } + } + + /** + * Updates cache and suspends till finished + */ + suspend fun updateCache(k: K): Set = cacheUpdateMutex.withLock { + updateCacheInternal(k) + } + + /** + * Starts cache-in-background-updating process (does nothing if started already) + */ + fun startUpdate() { + cacheUpdateJob.value + } + + /** + * Removes cache for [k]. Next [get] might suspend (if a background process wouldn't fill at in advance) + */ + fun clear(k: K) { + cache.remove(k) + logger.info("Cache flushed for $k") + } + + /** + * Returns current cache for [k], might suspend for a while if cache hasn't been filled yet. + */ + suspend fun get(k: K): MutableCollection { + cache[k]?.let { return it } // Return current cache + // No need to update the cache in several coroutines + cacheUpdateMutex.withLock { + // It could be that previous coroutine updated cache already (this is why we were blocked), so check again. + // Kinda Double-checked locking + cache[k]?.let { return it } + logger.info("No cache for $k, will fill") + return updateCacheInternal(k) + } + } + + /** + * Always call under [cacheUpdateMutex]. + * This function can't have mutex itself because double-checking locking is used in [get] but not in [cacheUpdateJob] + */ + private suspend fun updateCacheInternal(k: K): MutableSet { + logger.info("Starting update for $k") + val data = getDataForCache(k) + val newValue = ConcurrentHashMap.newKeySet(data.size) + newValue.addAll(data) + cache[k] = newValue + logger.info("End update for $k") + return newValue + } +} \ No newline at end of file diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/CoreSystemPythonProvider.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/CoreSystemPythonProvider.kt new file mode 100644 index 000000000000..7f5dea1e3c73 --- /dev/null +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/CoreSystemPythonProvider.kt @@ -0,0 +1,26 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.community.services.systemPython.impl + +import com.intellij.platform.eel.EelApi +import com.intellij.platform.eel.provider.localEel +import com.intellij.python.community.services.systemPython.SystemPythonProvider +import com.jetbrains.python.PythonBinary +import com.jetbrains.python.sdk.flavors.PythonSdkFlavor +import kotlinx.coroutines.Dispatchers + +/** + * [SystemPythonProvider] based ob [PythonSdkFlavor] (kind of a bridge) + */ +internal object CoreSystemPythonProvider : SystemPythonProvider { + override suspend fun findSystemPythons(eelApi: EelApi): Result> { + if (eelApi != localEel) return Result.success(emptySet()) + with(Dispatchers.IO) { + val paths = PythonSdkFlavor.getApplicableFlavors(false) + .flatMap { + it.dropCaches() + it.suggestLocalHomePaths(null, null) + } + return Result.success(paths.toSet()) + } + } +} \ No newline at end of file diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/SystemPythonInitialLoader.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/SystemPythonInitialLoader.kt new file mode 100644 index 000000000000..d884f2318524 --- /dev/null +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/SystemPythonInitialLoader.kt @@ -0,0 +1,20 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.community.services.systemPython.impl + +import com.intellij.openapi.diagnostic.fileLogger +import com.intellij.openapi.project.Project +import com.intellij.openapi.startup.ProjectActivity +import com.intellij.platform.eel.provider.getEelDescriptor +import com.intellij.python.community.services.systemPython.SystemPythonService +import com.intellij.python.community.services.systemPython.cacheTimeout + +private val logger = fileLogger() + +// Preload pythons as soon as a project gets loaded +internal class SystemPythonInitialLoader : ProjectActivity { + override suspend fun execute(project: Project) { + if (cacheTimeout == null) return // Cache is disabled, no need to preload it + logger.debug("Preloading pythons for $project") + SystemPythonService().findSystemPythons(project.getEelDescriptor().upgrade()) + } +} \ No newline at end of file diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/package-info.java b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/package-info.java new file mode 100644 index 000000000000..d221be74166d --- /dev/null +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@ApiStatus.Internal +package com.intellij.python.community.services.systemPython.impl; + +import org.jetbrains.annotations.ApiStatus; \ No newline at end of file diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt index 88d16a1ec33b..03855b976279 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/systemPythonServiceImpl.kt @@ -5,84 +5,134 @@ import com.intellij.openapi.application.EDT import com.intellij.openapi.components.* import com.intellij.openapi.components.Service.Level.APP import com.intellij.openapi.diagnostic.fileLogger +import com.intellij.openapi.util.registry.Registry import com.intellij.platform.eel.EelApi +import com.intellij.platform.eel.EelDescriptor import com.intellij.platform.eel.provider.getEelDescriptor import com.intellij.platform.eel.provider.localEel import com.intellij.python.community.impl.installer.PySdkToInstallManager import com.intellij.python.community.services.internal.impl.PythonWithLanguageLevelImpl import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl.MyServiceState +import com.intellij.python.community.services.systemPython.impl.Cache +import com.intellij.python.community.services.systemPython.impl.CoreSystemPythonProvider import com.jetbrains.python.PythonBinary import com.jetbrains.python.Result -import com.jetbrains.python.sdk.flavors.PythonSdkFlavor import com.jetbrains.python.sdk.installer.installBinary +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import org.jetbrains.annotations.ApiStatus.Internal import org.jetbrains.annotations.Nls +import java.nio.file.InvalidPathException +import java.nio.file.Path +import kotlin.io.path.pathString +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes -// Implementation +private val logger = fileLogger() + +// null means "disabled" +internal val cacheTimeout: Duration? + get() = Registry.get("python.system.refresh.minutes").asInteger().let { i -> + if (i > 0) i.minutes else null + } + @Service(APP) @State(name = "SystemPythonService", storages = [Storage("systemPythonService.xml", roamingType = RoamingType.LOCAL)], allowLoadInTests = true) @Internal -internal class SystemPythonServiceImpl : SystemPythonService, SimplePersistentStateComponent(MyServiceState()) { +internal class SystemPythonServiceImpl(scope: CoroutineScope) : SystemPythonService, SimplePersistentStateComponent(MyServiceState()) { + private val findPythonsMutex = Mutex() + private val cache: Cache? = cacheTimeout?.let { interval -> + Cache(scope, interval) { eelDescriptor -> + searchPythonsPhysicallyNoCache(eelDescriptor.upgrade()) + } + } + override suspend fun registerSystemPython(pythonPath: PythonBinary): Result { - val impl = PythonWithLanguageLevelImpl.createByPythonBinary(pythonPath).getOr { return it } - state.userProvidedPythons.add(pythonPath) - return Result.success(SystemPython(impl, null)) + val pythonWithLangLevel = PythonWithLanguageLevelImpl.createByPythonBinary(pythonPath).getOr { return it } + val systemPython = SystemPython(pythonWithLangLevel, null) + state.userProvidedPythons.add(pythonPath.pathString) + cache?.get(pythonPath.getEelDescriptor())?.add(systemPython) + return Result.success(systemPython) } override fun getInstaller(eelApi: EelApi): PythonInstallerService? = if (eelApi == localEel) LocalPythonInstaller else null - override suspend fun findSystemPythons(eelApi: EelApi): List = withContext(Dispatchers.IO) { - val corePythons = if (eelApi == localEel) - PythonSdkFlavor.getApplicableFlavors(false) - .flatMap { - it.dropCaches() - it.suggestLocalHomePaths(null, null) + override suspend fun findSystemPythons(eelApi: EelApi, forceRefresh: Boolean): List = + if (cache != null) { + // Cache enabled + cache.startUpdate() + if (forceRefresh) { + logger.info("pythons refresh requested") + cache.updateCache(eelApi.descriptor) // Update cache and suspend till update finished + } + else { + cache.get(eelApi.descriptor) + }.sorted() + } + else { + // Cache disabled + searchPythonsPhysicallyNoCache(eelApi) + } + + class MyServiceState : BaseState() { + // Only strings are supported by serializer + var userProvidedPythons by list() + val userProvidedPythonsAsPath: Collection + get() = userProvidedPythons.filterNotNull().mapNotNull { + try { + Path.of(it) } - else emptyList() - - val pythonsUi = mutableMapOf() - - val pythonsFromExtensions = SystemPythonProvider.EP - .extensionList - .flatMap { provider -> - val pythons = provider.findSystemPythons(eelApi).getOrNull() ?: emptyList() - val ui = provider.uiCustomization - if (ui != null) { - pythons.forEach { pythonsUi[it] = ui } + catch (_: InvalidPathException) { + logger.warn("invalid path $it") + null } - pythons - }.filter { it.getEelDescriptor().upgrade() == eelApi } - - val badPythons = mutableSetOf() - val pythons = corePythons + pythonsFromExtensions + state.userProvidedPythons.filter { it.getEelDescriptor() == eelApi.descriptor } - - val result = PythonWithLanguageLevelImpl.createByPythonBinaries(pythons.toSet()) - .mapNotNull { (python, r) -> - when (r) { - is Result.Success -> SystemPython(r.result, pythonsUi[r.result.pythonBinary]) - is Result.Failure -> { - fileLogger().info("Skipping $python : ${r.error}") - badPythons.add(python) - null - } - } - - }.toSet() - // Remove stale pythons from cache - state.userProvidedPythons.removeAll(badPythons) - return@withContext result.sorted() + } } - class MyServiceState : BaseState() { - val userProvidedPythons: MutableCollection by list() + private suspend fun searchPythonsPhysicallyNoCache(eelApi: EelApi): List = withContext(Dispatchers.IO) { + findPythonsMutex.withLock { + val pythonsUi = mutableMapOf() + + val pythonsFromExtensions = (SystemPythonProvider.EP + .extensionList + listOf(CoreSystemPythonProvider)) + .flatMap { provider -> + val pythons = provider.findSystemPythons(eelApi).getOrNull() ?: emptyList() + val ui = provider.uiCustomization + if (ui != null) { + pythons.forEach { pythonsUi[it] = ui } + } + pythons + } + + val badPythons = mutableSetOf() + val pythons = pythonsFromExtensions + state.userProvidedPythonsAsPath.filter { it.getEelDescriptor() == eelApi.descriptor } + + val result = PythonWithLanguageLevelImpl.createByPythonBinaries(pythons.toSet()) + .mapNotNull { (python, r) -> + when (r) { + is Result.Success -> SystemPython(r.result, pythonsUi[r.result.pythonBinary]) + is Result.Failure -> { + fileLogger().warn("Skipping $python : ${r.error}") + badPythons.add(python) + null + } + } + + }.toSet() + // Remove stale pythons from the cache + state.userProvidedPythons.removeAll(badPythons.map { it.pathString }) + logger.info("pythons refreshed") + return@withContext result.sorted() + } } } @@ -97,8 +147,8 @@ private object LocalPythonInstaller : PythonInstallerService { installBinary(pythonToInstall, null) { } }.getOrElse { - return Result.failure(it.message ?: it.toString()) + return Result.Companion.failure(it.message ?: it.toString()) } - return Result.success(Unit) + return Result.Companion.success(Unit) } } \ No newline at end of file diff --git a/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/SystemPythonServiceShowCaseTest.kt b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/SystemPythonServiceShowCaseTest.kt index 22b2135a3e36..103ed4d9c938 100644 --- a/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/SystemPythonServiceShowCaseTest.kt +++ b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/SystemPythonServiceShowCaseTest.kt @@ -1,6 +1,8 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.junit5Tests.env.systemPython +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.diagnostic.fileLogger import com.intellij.openapi.util.SystemInfo import com.intellij.platform.eel.executeProcess @@ -8,16 +10,24 @@ import com.intellij.platform.eel.getOrThrow import com.intellij.platform.eel.provider.getEelDescriptor import com.intellij.platform.eel.provider.utils.readWholeText import com.intellij.python.community.impl.venv.createVenv +import com.intellij.python.community.services.systemPython.SystemPythonProvider import com.intellij.python.community.services.systemPython.SystemPythonService +import com.intellij.python.community.services.systemPython.SystemPythonServiceImpl import com.intellij.python.junit5Tests.assertFail import com.intellij.python.junit5Tests.framework.env.PyEnvTestCase import com.intellij.python.junit5Tests.framework.env.PythonBinaryPath import com.intellij.python.junit5Tests.framework.winLockedFile.deleteCheckLocking import com.intellij.python.junit5Tests.randomBinary import com.intellij.testFramework.common.timeoutRunBlocking +import com.intellij.testFramework.junit5.RegistryKey +import com.intellij.testFramework.junit5.TestDisposable +import com.intellij.testFramework.registerExtension import com.jetbrains.python.getOrThrow import com.jetbrains.python.sdk.flavors.PythonSdkFlavor import com.jetbrains.python.venvReader.VirtualEnvReader +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk import kotlinx.coroutines.async import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.hasItem @@ -34,7 +44,7 @@ class SystemPythonServiceShowCaseTest { @Test fun testListPythons(): Unit = timeoutRunBlocking { - for (systemPython in SystemPythonService().findSystemPythons()) { + for (systemPython in SystemPythonService().findSystemPythons(forceRefresh = true)) { fileLogger().info("Python found: $systemPython") val eelApi = systemPython.pythonBinary.getEelDescriptor().upgrade() val process = eelApi.exec.executeProcess(systemPython.pythonBinary.pathString, "--version").getOrThrow() @@ -61,11 +71,42 @@ class SystemPythonServiceShowCaseTest { assertThat("No newly registered python returned", allPythons, hasItem(newPython)) python.deleteExisting() - allPythons = SystemPythonService().findSystemPythons() + allPythons = SystemPythonService().findSystemPythons(forceRefresh = true) assertThat("Broken python returned", allPythons, not(hasItem(newPython))) if (SystemInfo.isWindows) { deleteCheckLocking(venvPath) } } + + @Test + fun testRefresh(@TestDisposable disposable: Disposable): Unit = timeoutRunBlocking { + val mockProvider = mockk() + coEvery { mockProvider.findSystemPythons(any()) } returns Result.failure(java.lang.AssertionError("...")) + coEvery { mockProvider.uiCustomization } returns null + val sut = SystemPythonService() + sut.findSystemPythons() + ApplicationManager.getApplication().registerExtension(SystemPythonProvider.EP, mockProvider, disposable) + repeat(10) { + sut.findSystemPythons() + } + coVerify(exactly = 0) { mockProvider.findSystemPythons(any()) } + sut.findSystemPythons(forceRefresh = true) + coVerify(atLeast = 1) { mockProvider.findSystemPythons(any()) } + } + + @RegistryKey("python.system.refresh.minutes", "0") + @Test + fun testDisableCache(@TestDisposable disposable: Disposable): Unit = timeoutRunBlocking { + val timesToRepeat = 5 + val mockProvider = mockk() + coEvery { mockProvider.findSystemPythons(any()) } returns Result.success(emptySet()) + coEvery { mockProvider.uiCustomization } returns null + val sut = SystemPythonServiceImpl(this) + ApplicationManager.getApplication().registerExtension(SystemPythonProvider.EP, mockProvider, disposable) + repeat(timesToRepeat) { + sut.findSystemPythons() + } + coVerify(exactly = timesToRepeat) { mockProvider.findSystemPythons(any()) } + } } \ No newline at end of file diff --git a/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/CacheTest.kt b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/CacheTest.kt new file mode 100644 index 000000000000..1a548a08fe08 --- /dev/null +++ b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/CacheTest.kt @@ -0,0 +1,93 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.junit5Tests.env.systemPython.impl + +import com.intellij.python.community.services.systemPython.impl.Cache +import com.intellij.testFramework.common.timeoutRunBlocking +import kotlinx.coroutines.* +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import java.util.concurrent.atomic.AtomicInteger +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class CacheTest { + private val producerCalled = AtomicInteger() + + private fun createSut(scope: CoroutineScope) = Cache(scope, 1.milliseconds) { + producerCalled.incrementAndGet() + val element = "$it-value" + listOf(element) + } + + @Test + fun testCache(): Unit = timeoutRunBlocking { + val sut = createSut(this) + repeat(20) { + Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray()) + } + Assertions.assertEquals(1, producerCalled.get()) + sut.clear("abc") + Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray()) + sut.clear("foo") + Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray()) + Assertions.assertEquals(2, producerCalled.get()) + } + + @Test + fun testCacheUpdateBackground(): Unit = timeoutRunBlocking { + val sut = createSut(this) + sut.startUpdate() + Assertions.assertArrayEquals(arrayOf("foo-value"), sut.get("foo").toTypedArray()) + val secsToWaitForUpdate = 30 + repeat(secsToWaitForUpdate) { + delay(1.seconds) + if (producerCalled.get() > 5) { + coroutineContext.job.cancelChildren() + return@timeoutRunBlocking + } + } + Assertions.fail("Even after $secsToWaitForUpdate seconds cache hasn't been upgraded in background") + } + + + @ParameterizedTest + @ValueSource(booleans = [true, false]) + fun testCacheAccessibleWhileUpdate(updateByTimer: Boolean): Unit = timeoutRunBlocking { + val lock = CompletableDeferred() + val sut = Cache(this, if (updateByTimer) 1.milliseconds else Int.MAX_VALUE.days) { + val producerCalledTimes = producerCalled.incrementAndGet() + var time = "first" + if (producerCalledTimes > 1) { + lock.await() + time = "not-first" + } + val element = "$it-$time" + listOf(element) + } + if (updateByTimer) { + sut.startUpdate() + } + else { + launch { + Assertions.assertArrayEquals(arrayOf("foo-not-first"), sut.updateCache("foo").toTypedArray()) + } + } + repeat(20) { + Assertions.assertArrayEquals(arrayOf("foo-first"), sut.get("foo").toTypedArray()) + delay(1.milliseconds) + } + lock.complete(Unit) + val secsToWaitForUpdate = 30 + repeat(secsToWaitForUpdate) { + delay(1.seconds) + if (sut.get("foo").iterator().next() == "foo-not-first") { + coroutineContext.job.cancelChildren() + return@timeoutRunBlocking + } + } + Assertions.fail("Even after $secsToWaitForUpdate seconds cache hasn't been upgraded in background") + } +} \ No newline at end of file diff --git a/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/EnvProviderTest.kt b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/EnvProviderTest.kt index 9da6211a03d1..6115a02dc1ab 100644 --- a/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/EnvProviderTest.kt +++ b/python/services/system-python/tests/com/intellij/python/junit5Tests/env/systemPython/impl/EnvProviderTest.kt @@ -53,7 +53,7 @@ class EnvProviderTest { coEvery { provider.findSystemPythons(any()) } returns Result.success(setOf(venvPython)) coEvery { provider.uiCustomization } returns ui ApplicationManager.getApplication().registerExtension(SystemPythonProvider.EP, provider, disposable) - val python = SystemPythonService().findSystemPythons().first { it.pythonBinary == venvPython } + val python = SystemPythonService().findSystemPythons(forceRefresh = true).first { it.pythonBinary == venvPython } assertEquals(ui, python.ui, "Wrong UI") } } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/sdk/add/v2/CustomExistingEnvironmentSelector.kt b/python/src/com/jetbrains/python/sdk/add/v2/CustomExistingEnvironmentSelector.kt index 3cd477609901..9a4111577e56 100644 --- a/python/src/com/jetbrains/python/sdk/add/v2/CustomExistingEnvironmentSelector.kt +++ b/python/src/com/jetbrains/python/sdk/add/v2/CustomExistingEnvironmentSelector.kt @@ -5,13 +5,12 @@ import com.intellij.openapi.module.Module import com.intellij.openapi.observable.properties.ObservableMutableProperty import com.intellij.openapi.observable.util.notEqualsTo import com.intellij.openapi.ui.validation.DialogValidationRequestor +import com.intellij.python.community.services.shared.PythonWithLanguageLevel import com.intellij.ui.dsl.builder.Align import com.intellij.ui.dsl.builder.Panel import com.jetbrains.python.PyBundle.message import com.jetbrains.python.newProject.collector.InterpreterStatisticsInfo import com.jetbrains.python.sdk.ModuleOrProject -import com.jetbrains.python.sdk.PySdkUtil -import com.jetbrains.python.sdk.PythonSdkUtil import com.jetbrains.python.statistics.InterpreterCreationMode import com.jetbrains.python.statistics.InterpreterType import com.jetbrains.python.errorProcessing.ErrorSink @@ -80,9 +79,8 @@ internal abstract class CustomExistingEnvironmentSelector(private val name: Stri InterpreterCreationMode.CUSTOM) } - private fun addEnvByPath(path: String) { - val languageLevel = PySdkUtil.getLanguageLevelForSdk(PythonSdkUtil.findSdkByKey(path)) - val interpreter = ManuallyAddedSelectableInterpreter(path, languageLevel) + private fun addEnvByPath(python: PythonWithLanguageLevel) { + val interpreter = ManuallyAddedSelectableInterpreter(python) existingEnvironments.value += interpreter } diff --git a/python/src/com/jetbrains/python/sdk/add/v2/models.kt b/python/src/com/jetbrains/python/sdk/add/v2/models.kt index 254c8bae7bb4..c717ec89bb4e 100644 --- a/python/src/com/jetbrains/python/sdk/add/v2/models.kt +++ b/python/src/com/jetbrains/python/sdk/add/v2/models.kt @@ -207,7 +207,13 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva } } - open fun addInterpreter(path: String): PythonSelectableInterpreter { + internal fun addInterpreter(python: PythonWithLanguageLevel): PythonSelectableInterpreter { + val interpreter = ManuallyAddedSelectableInterpreter(python) + manuallyAddedInterpreters.value += interpreter + return interpreter + } + + internal fun addInterpreter(path: String): PythonSelectableInterpreter { val languageLevel = PySdkUtil.getLanguageLevelForSdk(PythonSdkUtil.findSdkByKey(path)) val interpreter = ManuallyAddedSelectableInterpreter(path, languageLevel) manuallyAddedInterpreters.value += interpreter @@ -221,7 +227,7 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva /** * Given [pathToPython] returns either cleaned path (if valid) or null and reports error to [errorSink] */ - suspend fun getSystemPythonFromSelection(pathToPython: String, errorSink: ErrorSink): String? { + suspend fun getSystemPythonFromSelection(pathToPython: String, errorSink: ErrorSink): SystemPython? { val result = try { when (val r = systemPythonService.registerSystemPython(Path(pathToPython))) { is com.jetbrains.python.Result.Failure -> com.jetbrains.python.errorProcessing.failure(r.error) @@ -233,7 +239,7 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva } return when (result) { - is com.jetbrains.python.Result.Success -> result.result.pythonBinary.pathString + is com.jetbrains.python.Result.Success -> result.result is com.jetbrains.python.Result.Failure -> { errorSink.emit(result.error) null @@ -369,7 +375,9 @@ class DetectedSelectableInterpreter(override val homePath: String, override val } } -class ManuallyAddedSelectableInterpreter(override val homePath: String, override val languageLevel: LanguageLevel) : PythonSelectableInterpreter() +class ManuallyAddedSelectableInterpreter(override val homePath: String, override val languageLevel: LanguageLevel) : PythonSelectableInterpreter() { + constructor(python: PythonWithLanguageLevel) : this(python.pythonBinary.pathString, python.languageLevel) +} class InstallableSelectableInterpreter(val sdk: PySdkToInstall) : PythonSelectableInterpreter() { override suspend fun isBasePython(): Boolean = true diff --git a/python/src/com/jetbrains/python/sdk/add/v2/uiUtils.kt b/python/src/com/jetbrains/python/sdk/add/v2/uiUtils.kt index f98b6838ff0a..a359ffb5d3ab 100644 --- a/python/src/com/jetbrains/python/sdk/add/v2/uiUtils.kt +++ b/python/src/com/jetbrains/python/sdk/add/v2/uiUtils.kt @@ -21,6 +21,8 @@ import com.intellij.openapi.util.NlsSafe import com.intellij.platform.ide.progress.ModalTaskOwner import com.intellij.platform.ide.progress.runWithModalProgressBlocking import com.intellij.python.community.impl.installer.CondaInstallManager +import com.intellij.python.community.services.shared.PythonWithLanguageLevel +import com.intellij.python.community.services.systemPython.SystemPython import com.intellij.ui.AnimatedIcon import com.intellij.ui.ColoredListCellRenderer import com.intellij.ui.SimpleColoredComponent @@ -65,6 +67,7 @@ import kotlin.coroutines.CoroutineContext import kotlin.io.path.Path import kotlin.io.path.exists import kotlin.io.path.isDirectory +import kotlin.io.path.pathString internal fun PropertyGraph.booleanProperty(dependency: ObservableProperty, value: T) = @@ -240,7 +243,7 @@ class PythonEnvironmentComboBoxRenderer : ColoredListCellRenderer() { internal fun Row.pythonInterpreterComboBox( selectedSdkProperty: ObservableMutableProperty, // todo not sdk model: PythonAddInterpreterModel, - onPathSelected: (String) -> Unit, busyState: StateFlow? = null, + onPathSelected: (PythonWithLanguageLevel) -> Unit, busyState: StateFlow? = null, ): Cell { val comboBox = PythonInterpreterComboBox(selectedSdkProperty, model, onPathSelected, ShowingMessageErrorSync) @@ -273,7 +276,7 @@ internal fun Row.pythonInterpreterComboBox( internal class PythonInterpreterComboBox( private val backingProperty: ObservableMutableProperty, val controller: PythonAddInterpreterModel, - val onPathSelected: (String) -> Unit, + val onPathSelected: (PythonWithLanguageLevel) -> Unit, private val errorSink: ErrorSink, ) : ComboBox() { @@ -288,7 +291,7 @@ internal class PythonInterpreterComboBox( val newOnPathSelected: (String) -> Unit = { runWithModalProgressBlocking(ModalTaskOwner.guess(), message("python.sdk.validating.environment")) { controller.getSystemPythonFromSelection(it, errorSink)?.let { python -> - interpreterToSelect.set(python) + interpreterToSelect.set(python.pythonBinary.pathString) onPathSelected(python) } }