diff --git a/python/services/internal-impl/intellij.python.community.services.internal.impl.iml b/python/services/internal-impl/intellij.python.community.services.internal.impl.iml index 0f6f44a4a7d4..9282b7000457 100644 --- a/python/services/internal-impl/intellij.python.community.services.internal.impl.iml +++ b/python/services/internal-impl/intellij.python.community.services.internal.impl.iml @@ -17,7 +17,7 @@ - + diff --git a/python/services/internal-impl/src/com/intellij/python/community/services/internal/impl/PythonWithLanguageLevelImpl.kt b/python/services/internal-impl/src/com/intellij/python/community/services/internal/impl/PythonWithLanguageLevelImpl.kt index bec9efd902b8..3c41acb9b80f 100644 --- a/python/services/internal-impl/src/com/intellij/python/community/services/internal/impl/PythonWithLanguageLevelImpl.kt +++ b/python/services/internal-impl/src/com/intellij/python/community/services/internal/impl/PythonWithLanguageLevelImpl.kt @@ -5,11 +5,18 @@ import com.intellij.platform.eel.fs.pathOs import com.intellij.platform.eel.path.EelPath import com.intellij.platform.eel.provider.asNioPath import com.intellij.platform.eel.provider.getEelDescriptor +import com.intellij.python.community.services.internal.impl.PythonWithLanguageLevelImpl.Companion.concurrentLimit +import com.intellij.python.community.services.internal.impl.PythonWithLanguageLevelImpl.Companion.createByPythonBinary import com.intellij.python.community.services.shared.PythonWithLanguageLevel import com.jetbrains.python.PythonBinary import com.jetbrains.python.Result import com.jetbrains.python.psi.LanguageLevel import com.jetbrains.python.validatePythonAndGetVersion +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import org.jetbrains.annotations.ApiStatus.Internal import org.jetbrains.annotations.Nls import kotlin.io.path.pathString @@ -21,6 +28,23 @@ class PythonWithLanguageLevelImpl internal constructor( override val languageLevel: LanguageLevel, ) : PythonWithLanguageLevel, Comparable { companion object { + + private val concurrentLimit = Semaphore(permits = 4) + + /** + * Like [createByPythonBinary] but runs in parallel up to [concurrentLimit] + */ + suspend fun createByPythonBinaries(pythonBinaries: Collection): Collection>> = + coroutineScope { + pythonBinaries.map { + async { + concurrentLimit.withPermit { + Pair(it, createByPythonBinary(it)) + } + } + }.awaitAll() + } + suspend fun createByPythonBinary(pythonBinary: PythonBinary): Result { val languageLevel = pythonBinary.validatePythonAndGetVersion().getOr { return it } return Result.success(PythonWithLanguageLevelImpl(pythonBinary, languageLevel)) diff --git a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl.kt b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl.kt index 0bf29bdda3f3..cdb6da9d99ea 100644 --- a/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl.kt +++ b/python/services/system-python/src/com/intellij/python/community/services/systemPython/impl.kt @@ -54,9 +54,9 @@ internal class SystemPythonServiceImpl : SystemPythonService, SimplePersistentSt val badPythons = mutableSetOf() val pythons = corePythons + pythonsFromExtensions + state.userProvidedPythons.filter { it.getEelDescriptor() == eelApi.descriptor } - val result = pythons.toSet() - .mapNotNull { python -> - when (val r = PythonWithLanguageLevelImpl.createByPythonBinary(python)) { + val result = PythonWithLanguageLevelImpl.createByPythonBinaries(pythons.toSet()) + .mapNotNull { (python, r) -> + when (r) { is Result.Success -> SystemPython(r.result) is Result.Failure -> { fileLogger().info("Skipping $python : ${r.error}") diff --git a/python/src/com/jetbrains/python/sdk/add/v2/models.kt b/python/src/com/jetbrains/python/sdk/add/v2/models.kt index bdad07806c81..cb797e616625 100644 --- a/python/src/com/jetbrains/python/sdk/add/v2/models.kt +++ b/python/src/com/jetbrains/python/sdk/add/v2/models.kt @@ -143,17 +143,18 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva // Venvs are not detected manually, but must migrate to VenvService or so - val venvs: List = VirtualEnvSdkFlavor.getInstance() - .suggestLocalHomePaths(null, null) - .mapNotNull { venv -> - return@mapNotNull when (val r = PythonWithLanguageLevelImpl.createByPythonBinary(venv)) { - is com.jetbrains.python.Result.Failure -> { - fileLogger().warn("Skipping $venv : ${r.error}") - null - } - is com.jetbrains.python.Result.Success -> r.result + val venvs: List = PythonWithLanguageLevelImpl.createByPythonBinaries( + VirtualEnvSdkFlavor.getInstance() + .suggestLocalHomePaths(null, null) + ).mapNotNull { (venv, r) -> + when (r) { + is com.jetbrains.python.Result.Failure -> { + fileLogger().warn("Skipping $venv : ${r.error}") + null } + is com.jetbrains.python.Result.Success -> r.result } + } // System (base) pythons val system: List = systemPythonService.findSystemPythons() diff --git a/python/src/com/jetbrains/python/sdk/conda/PyAddCondaTools.kt b/python/src/com/jetbrains/python/sdk/conda/PyAddCondaTools.kt index 98dae9f6c1e7..5bb16724d647 100644 --- a/python/src/com/jetbrains/python/sdk/conda/PyAddCondaTools.kt +++ b/python/src/com/jetbrains/python/sdk/conda/PyAddCondaTools.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.jetbrains.python.sdk.conda import com.intellij.execution.Platform @@ -20,7 +20,8 @@ import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil import com.intellij.platform.util.progress.RawProgressReporter import com.jetbrains.python.failure import com.jetbrains.python.psi.LanguageLevel -import com.jetbrains.python.sdk.* +import com.jetbrains.python.sdk.PythonSdkAdditionalData +import com.jetbrains.python.sdk.PythonSdkType import com.jetbrains.python.sdk.add.v1.PyAddCondaPanelModel import com.jetbrains.python.sdk.add.v1.loadLocalPythonCondaPath import com.jetbrains.python.sdk.add.v1.saveLocalPythonCondaPath @@ -48,9 +49,11 @@ internal val condaSupportedLanguages: List /** * See [com.jetbrains.env.conda.PyCondaSdkTest] */ -suspend fun PyCondaCommand.createCondaSdkFromExistingEnv(condaIdentity: PyCondaEnvIdentity, - existingSdks: List, - project: Project?): Sdk { +suspend fun PyCondaCommand.createCondaSdkFromExistingEnv( + condaIdentity: PyCondaEnvIdentity, + existingSdks: List, + project: Project?, +): Sdk { val condaEnv = PyCondaEnv(condaIdentity, fullCondaPathOnTarget) val flavorAndData = PyFlavorAndData(PyCondaFlavorData(condaEnv), CondaEnvSdkFlavor.getInstance()) @@ -78,19 +81,23 @@ private const val PRINT_SYS_EXECUTABLE_SCRIPT = "import sys; print(sys.executabl /** * @return path to Conda interpreter binary on target */ -private suspend fun getCondaPythonBinaryPath(project: Project?, - condaEnv: PyCondaEnv, - targetConfig: TargetEnvironmentConfiguration?): Result = +private suspend fun getCondaPythonBinaryPath( + project: Project?, + condaEnv: PyCondaEnv, + targetConfig: TargetEnvironmentConfiguration?, +): Result = getCondaInterpreterOutput(project, condaEnv, EmptyProgressIndicator(), PRINT_SYS_EXECUTABLE_SCRIPT, targetConfig).map { it.trim() } /** * Runs python [command] and returns stdout or error */ -private suspend fun getCondaInterpreterOutput(project: Project?, - condaEnv: PyCondaEnv, - indicator: ProgressIndicator, - command: String, - targetConfig: TargetEnvironmentConfiguration?): Result { +private suspend fun getCondaInterpreterOutput( + project: Project?, + condaEnv: PyCondaEnv, + indicator: ProgressIndicator, + command: String, + targetConfig: TargetEnvironmentConfiguration?, +): Result { val targetEnvRequest = targetConfig?.createEnvironmentRequest(project) ?: LocalTargetEnvironmentRequest() val cmdBuilder = TargetedCommandLineBuilder(targetEnvRequest) @@ -108,11 +115,13 @@ private suspend fun getCondaInterpreterOutput(project: Project?, /** * See [com.jetbrains.env.conda.PyCondaSdkTest] */ -suspend fun PyCondaCommand.createCondaSdkAlongWithNewEnv(newCondaEnvInfo: NewCondaEnvRequest, - uiContext: CoroutineContext, - existingSdks: List, - project: Project, - reporter: RawProgressReporter? = null): Result { +suspend fun PyCondaCommand.createCondaSdkAlongWithNewEnv( + newCondaEnvInfo: NewCondaEnvRequest, + uiContext: CoroutineContext, + existingSdks: List, + project: Project, + reporter: RawProgressReporter? = null, +): Result { val process = PyCondaEnv.createEnv(this, newCondaEnvInfo).getOrElse { return Result.failure(it) } val error = ProcessHandlerReader(process).runProcessAndGetError(uiContext, reporter) @@ -190,12 +199,12 @@ private fun String.asCommandInShell(targetPlatform: TargetPlatform): List { + for ((env, value) in System.getenv()) { + val pattern = "%${env}%" + if (pattern in result) { + result = result.replace(pattern, value) + } + } + } + false -> { // posix + result = path.replace("~", SystemProperties.getUserHome()) + } + } + return result +} \ No newline at end of file