Python: speed-up v2 interpreter widget initialization.

1. No need to use shell to expand simple vars for local execution.
2. Check pythons (sys and venv) in parallel (see `PythonWithLanguageLevelImpl`)

GitOrigin-RevId: d5432b12452a6a64213cd9da59a7d94e1ae2aa11

python/services/internal-impl/intellij.python.community.services.internal.impl.iml

@@ -17,7 +17,7 @@
     <orderEntry type="module" module-name="intellij.platform.testFramework.junit5" scope="TEST" />
     <orderEntry type="library" scope="TEST" name="JUnit5" level="project" />
     <orderEntry type="library" scope="TEST" name="hamcrest" level="project" />
-    <orderEntry type="library" scope="TEST" name="kotlinx-coroutines-core" level="project" />
+    <orderEntry type="library" name="kotlinx-coroutines-core" level="project" />
     <orderEntry type="module" module-name="intellij.python.sdk" />
     <orderEntry type="module" module-name="intellij.python.community.junit5Tests.framework" scope="TEST" />
   </component>

python/services/internal-impl/src/com/intellij/python/community/services/internal/impl/PythonWithLanguageLevelImpl.kt

@@ -5,11 +5,18 @@ import com.intellij.platform.eel.fs.pathOs
 import com.intellij.platform.eel.path.EelPath
 import com.intellij.platform.eel.provider.asNioPath
 import com.intellij.platform.eel.provider.getEelDescriptor
+import com.intellij.python.community.services.internal.impl.PythonWithLanguageLevelImpl.Companion.concurrentLimit
+import com.intellij.python.community.services.internal.impl.PythonWithLanguageLevelImpl.Companion.createByPythonBinary
 import com.intellij.python.community.services.shared.PythonWithLanguageLevel
 import com.jetbrains.python.PythonBinary
 import com.jetbrains.python.Result
 import com.jetbrains.python.psi.LanguageLevel
 import com.jetbrains.python.validatePythonAndGetVersion
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
 import org.jetbrains.annotations.ApiStatus.Internal
 import org.jetbrains.annotations.Nls
 import kotlin.io.path.pathString
@@ -21,6 +28,23 @@ class PythonWithLanguageLevelImpl internal constructor(
   override val languageLevel: LanguageLevel,
 ) : PythonWithLanguageLevel, Comparable<PythonWithLanguageLevelImpl> {
   companion object {
+
+    private val concurrentLimit = Semaphore(permits = 4)
+
+    /**
+     * Like [createByPythonBinary] but runs in parallel up to [concurrentLimit]
+     */
+    suspend fun createByPythonBinaries(pythonBinaries: Collection<PythonBinary>): Collection<Pair<PythonBinary, Result<PythonWithLanguageLevelImpl, @Nls String>>> =
+      coroutineScope {
+        pythonBinaries.map {
+          async {
+            concurrentLimit.withPermit {
+              Pair(it, createByPythonBinary(it))
+            }
+          }
+        }.awaitAll()
+      }
+
     suspend fun createByPythonBinary(pythonBinary: PythonBinary): Result<PythonWithLanguageLevelImpl, @Nls String> {
       val languageLevel = pythonBinary.validatePythonAndGetVersion().getOr { return it }
       return Result.success(PythonWithLanguageLevelImpl(pythonBinary, languageLevel))

python/services/system-python/src/com/intellij/python/community/services/systemPython/impl.kt

@@ -54,9 +54,9 @@ internal class SystemPythonServiceImpl : SystemPythonService, SimplePersistentSt
     val badPythons = mutableSetOf<PythonBinary>()
     val pythons = corePythons + pythonsFromExtensions + state.userProvidedPythons.filter { it.getEelDescriptor() == eelApi.descriptor }
 
-    val result = pythons.toSet()
-      .mapNotNull { python ->
-        when (val r = PythonWithLanguageLevelImpl.createByPythonBinary(python)) {
+    val result = PythonWithLanguageLevelImpl.createByPythonBinaries(pythons.toSet())
+      .mapNotNull { (python, r) ->
+        when (r) {
           is Result.Success -> SystemPython(r.result)
           is Result.Failure -> {
             fileLogger().info("Skipping $python : ${r.error}")

python/src/com/jetbrains/python/sdk/add/v2/models.kt

@@ -143,17 +143,18 @@ abstract class PythonAddInterpreterModel(params: PyInterpreterModelParams, priva
 
 
       // Venvs are not detected manually, but must migrate to VenvService or so
-      val venvs: List<PythonWithLanguageLevel> = VirtualEnvSdkFlavor.getInstance()
-        .suggestLocalHomePaths(null, null)
-        .mapNotNull { venv ->
-          return@mapNotNull when (val r = PythonWithLanguageLevelImpl.createByPythonBinary(venv)) {
-            is com.jetbrains.python.Result.Failure -> {
-              fileLogger().warn("Skipping $venv : ${r.error}")
-              null
-            }
-            is com.jetbrains.python.Result.Success -> r.result
+      val venvs: List<PythonWithLanguageLevel> = PythonWithLanguageLevelImpl.createByPythonBinaries(
+        VirtualEnvSdkFlavor.getInstance()
+          .suggestLocalHomePaths(null, null)
+      ).mapNotNull { (venv, r) ->
+        when (r) {
+          is com.jetbrains.python.Result.Failure -> {
+            fileLogger().warn("Skipping $venv : ${r.error}")
+            null
           }
+          is com.jetbrains.python.Result.Success -> r.result
         }
+      }
 
       // System (base) pythons
       val system: List<PythonWithLanguageLevel> = systemPythonService.findSystemPythons()

python/src/com/jetbrains/python/sdk/conda/PyAddCondaTools.kt

@@ -1,4 +1,4 @@
-// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.jetbrains.python.sdk.conda
 
 import com.intellij.execution.Platform
@@ -20,7 +20,8 @@ import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil
 import com.intellij.platform.util.progress.RawProgressReporter
 import com.jetbrains.python.failure
 import com.jetbrains.python.psi.LanguageLevel
-import com.jetbrains.python.sdk.*
+import com.jetbrains.python.sdk.PythonSdkAdditionalData
+import com.jetbrains.python.sdk.PythonSdkType
 import com.jetbrains.python.sdk.add.v1.PyAddCondaPanelModel
 import com.jetbrains.python.sdk.add.v1.loadLocalPythonCondaPath
 import com.jetbrains.python.sdk.add.v1.saveLocalPythonCondaPath
@@ -48,9 +49,11 @@ internal val condaSupportedLanguages: List<LanguageLevel>
 /**
  * See [com.jetbrains.env.conda.PyCondaSdkTest]
  */
-suspend fun PyCondaCommand.createCondaSdkFromExistingEnv(condaIdentity: PyCondaEnvIdentity,
-                                                         existingSdks: List<Sdk>,
-                                                         project: Project?): Sdk {
+suspend fun PyCondaCommand.createCondaSdkFromExistingEnv(
+  condaIdentity: PyCondaEnvIdentity,
+  existingSdks: List<Sdk>,
+  project: Project?,
+): Sdk {
   val condaEnv = PyCondaEnv(condaIdentity, fullCondaPathOnTarget)
   val flavorAndData = PyFlavorAndData(PyCondaFlavorData(condaEnv), CondaEnvSdkFlavor.getInstance())
 
@@ -78,19 +81,23 @@ private const val PRINT_SYS_EXECUTABLE_SCRIPT = "import sys; print(sys.executabl
 /**
  * @return path to Conda interpreter binary on target
  */
-private suspend fun getCondaPythonBinaryPath(project: Project?,
-                                             condaEnv: PyCondaEnv,
-                                             targetConfig: TargetEnvironmentConfiguration?): Result<FullPathOnTarget> =
+private suspend fun getCondaPythonBinaryPath(
+  project: Project?,
+  condaEnv: PyCondaEnv,
+  targetConfig: TargetEnvironmentConfiguration?,
+): Result<FullPathOnTarget> =
   getCondaInterpreterOutput(project, condaEnv, EmptyProgressIndicator(), PRINT_SYS_EXECUTABLE_SCRIPT, targetConfig).map { it.trim() }
 
 /**
  * Runs python [command] and returns stdout or error
  */
-private suspend fun getCondaInterpreterOutput(project: Project?,
-                                              condaEnv: PyCondaEnv,
-                                              indicator: ProgressIndicator,
-                                              command: String,
-                                              targetConfig: TargetEnvironmentConfiguration?): Result<String> {
+private suspend fun getCondaInterpreterOutput(
+  project: Project?,
+  condaEnv: PyCondaEnv,
+  indicator: ProgressIndicator,
+  command: String,
+  targetConfig: TargetEnvironmentConfiguration?,
+): Result<String> {
   val targetEnvRequest = targetConfig?.createEnvironmentRequest(project) ?: LocalTargetEnvironmentRequest()
 
   val cmdBuilder = TargetedCommandLineBuilder(targetEnvRequest)
@@ -108,11 +115,13 @@ private suspend fun getCondaInterpreterOutput(project: Project?,
 /**
  * See [com.jetbrains.env.conda.PyCondaSdkTest]
  */
-suspend fun PyCondaCommand.createCondaSdkAlongWithNewEnv(newCondaEnvInfo: NewCondaEnvRequest,
-                                                         uiContext: CoroutineContext,
-                                                         existingSdks: List<Sdk>,
-                                                         project: Project,
-                                                         reporter: RawProgressReporter? = null): Result<Sdk> {
+suspend fun PyCondaCommand.createCondaSdkAlongWithNewEnv(
+  newCondaEnvInfo: NewCondaEnvRequest,
+  uiContext: CoroutineContext,
+  existingSdks: List<Sdk>,
+  project: Project,
+  reporter: RawProgressReporter? = null,
+): Result<Sdk> {
   val process = PyCondaEnv.createEnv(this, newCondaEnvInfo).getOrElse { return Result.failure(it) }
   val error = ProcessHandlerReader(process).runProcessAndGetError(uiContext, reporter)
 
@@ -190,12 +199,12 @@ private fun String.asCommandInShell(targetPlatform: TargetPlatform): List<String
  */
 private suspend fun TargetCommandExecutor.getExpandedPathIfExecutable(file: FullPathOnTarget): FullPathOnTarget? = withContext(
   Dispatchers.IO) {
-  val expandedPath = executeShellCommand("echo $file").thenApply(ProcessOutput::getStdout).thenApply(String::trim).await()
   if (isLocalMachineExecutor) {
+    val expandedPath = expandPathLocally(file)
     return@withContext if (Path.of(expandedPath).isExecutable()) expandedPath else null
   }
   else {
-
+    val expandedPath = executeShellCommand("echo $file").thenApply(ProcessOutput::getStdout).thenApply(String::trim).await()
     // TODO: Should we test with browsable target as well?
 
     if (targetPlatform.await().platform == Platform.WINDOWS) {

python/src/com/jetbrains/python/sdk/conda/expandPathUtil.kt

@@ -0,0 +1,30 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.jetbrains.python.sdk.conda
+
+import com.intellij.execution.target.FullPathOnTarget
+import com.intellij.openapi.util.SystemInfoRt
+import com.intellij.util.SystemProperties
+import org.jetbrains.annotations.ApiStatus.Internal
+
+/**
+ * Expands `~` in **nix and `%ENV%` on Windows for local system only.
+ * it is a hack to be used to speed-up conda search. Do not use.
+ */
+@Internal
+fun expandPathLocally(path: FullPathOnTarget): FullPathOnTarget {
+  var result = path
+  when (SystemInfoRt.isWindows) {
+    true -> {
+      for ((env, value) in System.getenv()) {
+        val pattern = "%${env}%"
+        if (pattern in result) {
+          result = result.replace(pattern, value)
+        }
+      }
+    }
+    false -> { // posix
+      result = path.replace("~", SystemProperties.getUserHome())
+    }
+  }
+  return result
+}