[python] PY-84777 Use system pythons as a fallback for SDK configuration

There was a problem with detecting system-wide pythons, which relied on
binary not being a part of conda env or virtualenv. But it led to
unrelated Hatch and Poetry pythons automatically configured as
interpreters in new projects. Another problem is that free-threaded
python was chosen as default interpreter with highest priority because
of the newest version.

This change uses SystemPythonService to detect system pythons properly,
also free-threaded python used as a default interpreter only if it's the
only available option.


Merge-request: IJ-MR-179008
Merged-by: Alexey Katsman <alexey.katsman@jetbrains.com>

GitOrigin-RevId: f32238b063575a125642ee91984ed45ba47010ec

python/python-exec-service/execService.python/src/advancedApi/advancedApi.kt

@@ -5,9 +5,9 @@ import com.intellij.python.community.execService.*
 import com.intellij.python.community.execService.impl.transformerToHandler
 import com.intellij.python.community.execService.python.HelperName
 import com.intellij.python.community.execService.python.addHelper
-import com.intellij.python.community.execService.python.impl.validatePythonAndGetVersionImpl
+import com.intellij.python.community.execService.python.impl.validatePythonAndGetInfoImpl
+import com.jetbrains.python.PythonInfo
 import com.jetbrains.python.errorProcessing.PyResult
-import com.jetbrains.python.psi.LanguageLevel
 import org.jetbrains.annotations.ApiStatus
 
 // This in advanced API, most probably you need "api.kt"
@@ -44,11 +44,11 @@ suspend fun <T> ExecService.executeHelperAdvanced(
   options, transformerToHandler(procListener, processOutputTransformer))
 
 /**
- * Ensures that this python is executable and returns its version. Error if python is broken.
+ * Ensures that this python is executable and returns its info. Error if python is broken.
  *
  * Some pythons might be broken: they may be executable, even return a version, but still fail to execute it.
  * As we need workable pythons, we validate it by executing
  */
 @ApiStatus.Internal
-suspend fun ExecService.validatePythonAndGetVersion(python: ExecutablePython): PyResult<LanguageLevel> =
-  validatePythonAndGetVersionImpl(python)
+suspend fun ExecService.validatePythonAndGetInfo(python: ExecutablePython): PyResult<PythonInfo> =
+  validatePythonAndGetInfoImpl(python)

python/python-exec-service/execService.python/src/api.kt

@@ -4,11 +4,11 @@ package com.intellij.python.community.execService.python
 import com.intellij.python.community.execService.*
 import com.intellij.python.community.execService.python.advancedApi.ExecutablePython
 import com.intellij.python.community.execService.python.advancedApi.executeHelperAdvanced
-import com.intellij.python.community.execService.python.advancedApi.validatePythonAndGetVersion
+import com.intellij.python.community.execService.python.advancedApi.validatePythonAndGetInfo
 import com.intellij.python.community.helpersLocator.PythonHelpersLocator
 import com.jetbrains.python.PythonBinary
+import com.jetbrains.python.PythonInfo
 import com.jetbrains.python.errorProcessing.PyResult
-import com.jetbrains.python.psi.LanguageLevel
 
 /**
  * Python binary itself (i.e python.exe)
@@ -30,17 +30,17 @@ suspend fun ExecService.executeHelper(
   executeHelperAdvanced(ExecutablePython.vanillaExecutablePython(python), helper, args, options, procListener, ZeroCodeStdoutTransformer)
 
 /**
- * Ensures that this python is executable and returns its version. Error if python is broken.
+ * Ensures that this python is executable and returns its info. Error if python is broken.
  *
  * Some pythons might be broken: they may be executable, even return a version, but still fail to execute it.
  * As we need workable pythons, we validate it by executing
  */
-suspend fun ExecService.validatePythonAndGetVersion(python: PythonBinaryOnEelOrTarget): PyResult<LanguageLevel> =
-  validatePythonAndGetVersion(ExecutablePython.vanillaExecutablePython(python))
+suspend fun ExecService.validatePythonAndGetInfo(python: PythonBinaryOnEelOrTarget): PyResult<PythonInfo> =
+  validatePythonAndGetInfo(ExecutablePython.vanillaExecutablePython(python))
 
-suspend fun PythonBinaryOnEelOrTarget.validatePythonAndGetVersion(): PyResult<LanguageLevel> = ExecService().validatePythonAndGetVersion(this)
-suspend fun ExecService.validatePythonAndGetVersion(python: PythonBinary): PyResult<LanguageLevel> = validatePythonAndGetVersion(python.asBinToExec())
-suspend fun PythonBinary.validatePythonAndGetVersion(): PyResult<LanguageLevel> = asBinToExec().validatePythonAndGetVersion()
+suspend fun PythonBinaryOnEelOrTarget.validatePythonAndGetInfo(): PyResult<PythonInfo> = ExecService().validatePythonAndGetInfo(this)
+suspend fun ExecService.validatePythonAndGetInfo(python: PythonBinary): PyResult<PythonInfo> = validatePythonAndGetInfo(python.asBinToExec())
+suspend fun PythonBinary.validatePythonAndGetInfo(): PyResult<PythonInfo> = asBinToExec().validatePythonAndGetInfo()
 
 
 /**

python/python-exec-service/execService.python/src/impl/impl.kt

@@ -1,6 +1,7 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.community.execService.python.impl
 
+import com.intellij.openapi.diagnostic.fileLogger
 import com.intellij.openapi.util.NlsSafe
 import com.intellij.platform.eel.provider.utils.EelProcessExecutionResult
 import com.intellij.platform.eel.provider.utils.stderrString
@@ -11,9 +12,11 @@ import com.intellij.python.community.execService.python.advancedApi.ExecutablePy
 import com.intellij.python.community.execService.python.advancedApi.executePythonAdvanced
 import com.intellij.python.community.execService.python.impl.PyExecPythonBundle.message
 import com.jetbrains.python.PYTHON_VERSION_ARG
+import com.jetbrains.python.PythonInfo
 import com.jetbrains.python.Result
 import com.jetbrains.python.errorProcessing.PyResult
 import com.jetbrains.python.errorProcessing.getOr
+import com.jetbrains.python.onFailure
 import com.jetbrains.python.psi.LanguageLevel
 import com.jetbrains.python.sdk.flavors.PythonSdkFlavor.getLanguageLevelFromVersionStringStaticSafe
 import kotlinx.coroutines.Dispatchers
@@ -23,7 +26,7 @@ import kotlin.io.path.pathString
 import kotlin.time.Duration.Companion.minutes
 
 @ApiStatus.Internal
-internal suspend fun ExecService.validatePythonAndGetVersionImpl(python: ExecutablePython): PyResult<LanguageLevel> = withContext(Dispatchers.IO) {
+internal suspend fun ExecService.validatePythonAndGetInfoImpl(python: ExecutablePython): PyResult<PythonInfo> = withContext(Dispatchers.IO) {
   val options = ExecOptions(timeout = 1.minutes)
 
   val smokeTestOutput = executePythonAdvanced(python, Args("-c", "print(1)"), processInteractiveHandler = transformerToHandler(null, ZeroCodeStdoutTransformer), options = options).getOr(message("python.cannot.exec", python.userReadableName)) { return@withContext it }.trim()
@@ -31,16 +34,28 @@ internal suspend fun ExecService.validatePythonAndGetVersionImpl(python: Executa
     return@withContext PyResult.localizedError(message("python.get.version.error", python.userReadableName, smokeTestOutput))
   }
 
-  val versionOutput: EelProcessExecutionResult = executePythonAdvanced(python, options = options, args = Args(PYTHON_VERSION_ARG), processInteractiveHandler = transformerToHandler<EelProcessExecutionResult>(null, { r ->
+  val versionOutput: EelProcessExecutionResult = executePythonAdvanced(python, options = options, args = Args(PYTHON_VERSION_ARG), processInteractiveHandler = transformerToHandler<EelProcessExecutionResult>(null) { r ->
     if (r.exitCode == 0) Result.success(r) else Result.failure(message("python.get.version.error", python.userReadableName, r.exitCode))
-  })).getOr { return@withContext it }
+  }).getOr { return@withContext it }
   // Python 2 might return version as stderr, see https://bugs.python.org/issue18338
   val versionString = versionOutput.stdoutString.let { it.ifBlank { versionOutput.stderrString } }
   val languageLevel = getLanguageLevelFromVersionStringStaticSafe(versionString.trim())
   if (languageLevel == null) {
     return@withContext PyResult.localizedError(message("python.get.version.wrong.version", python.userReadableName, versionString))
   }
-  return@withContext Result.success(languageLevel)
+
+  val freeThreaded = if (languageLevel.isAtLeast(LanguageLevel.PYTHON313)) {
+    val gilEnabledOutput = executePythonAdvanced(
+      python,
+      Args("-c", "import sys; print(sys._is_gil_enabled())"),
+      processInteractiveHandler = transformerToHandler(null, ZeroCodeStdoutTransformer),
+      options = options
+    ).onFailure { fileLogger().warn(it.toString()) }.successOrNull?.trim()
+    gilEnabledOutput == "False"
+  }
+  else false
+
+  return@withContext Result.success(PythonInfo(languageLevel, freeThreaded))
 }
 
 private val ExecutablePython.userReadableName: @NlsSafe String

python/python-exec-service/execService.python/tests/com/intellij/python/junit5Tests/env/HelpersShowCaseTest.kt

@@ -7,7 +7,7 @@ import com.intellij.platform.testFramework.junit5.eel.params.api.*
 import com.intellij.python.community.execService.ExecService
 import com.intellij.python.community.execService.asBinToExec
 import com.intellij.python.community.execService.python.executeHelper
-import com.intellij.python.community.execService.python.validatePythonAndGetVersion
+import com.intellij.python.community.execService.python.validatePythonAndGetInfo
 import com.intellij.python.community.helpersLocator.PythonHelpersLocator
 import com.intellij.python.junit5Tests.framework.env.PyEnvTestCase
 import com.intellij.python.junit5Tests.framework.env.PythonBinaryPath
@@ -53,7 +53,7 @@ class HelpersShowCaseTest() {
       val output = ExecService().executeHelper(python.asBinToExec(), helper.name, listOf("--version")).orThrow().trim()
       Assertions.assertEquals(hello, output, "wrong helper output")
 
-      val langLevel = ExecService().validatePythonAndGetVersion(python).getOrThrow()
+      val langLevel = ExecService().validatePythonAndGetInfo(python).getOrThrow().languageLevel
       Assertions.assertTrue(langLevel.isPy3K, "Wrong lang level:$langLevel")
     }
     finally {

python/python-exec-service/execService.python/tests/com/intellij/python/junit5Tests/env/PythonBinaryValidationTest.kt

@@ -1,7 +1,7 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.junit5Tests.env
 
-import com.intellij.python.community.execService.python.validatePythonAndGetVersion
+import com.intellij.python.community.execService.python.validatePythonAndGetInfo
 import com.intellij.python.junit5Tests.framework.env.PyEnvTestCase
 import com.intellij.python.junit5Tests.framework.env.PythonBinaryPath
 import com.intellij.python.junit5Tests.randomBinary
@@ -15,13 +15,13 @@ import org.junit.jupiter.api.Test
 class PythonBinaryValidationTest {
   @Test
   fun sunnyDayTest(@PythonBinaryPath python: PythonBinary): Unit = runBlocking {
-    val level = python.validatePythonAndGetVersion().orThrow()
-    Assertions.assertNotNull(level, "Failed to get python level")
+    val info = python.validatePythonAndGetInfo().orThrow()
+    Assertions.assertNotNull(info, "Failed to get python info")
   }
 
   @Test
   fun rainyDayTest(): Unit = runBlocking {
-    when (val r = randomBinary.validatePythonAndGetVersion()) {
+    when (val r = randomBinary.validatePythonAndGetInfo()) {
       is Result.Success -> {
         Assertions.fail("${randomBinary} isn't a python, should fail, but got ${r.result}")
       }