PY-83666 Detect pypy virtual environments

We set UV_PROJECT_ENVIRONMENT from interpreter path. The problem was
that we checked whether it's a python binary or already a venv path, but
for binary the check contained only "python" substring, not a proper
regex.

These changes unify the logic for getting information about python home
in VirtualEnvReader and use it as the source of truth.

This is a port of 70a9c9ee94d8f. The 262 branch has no `PythonEnvironment`
provider and no `com.intellij.python.sdk.backend` package, so the call
sites differ. `PythonEnvironmentDetector` replaces `CondaEnvironment` and
`VenvEnvironment`, and the two `EvoSdkManager` classes replace the three
`EvoEnvironmentProvider` classes. The port leaves out PY-91826, so
`findVenvsInDir`, `findPythonInPythonRoot` and `findInterpreter` keep the
262 shape.

GitOrigin-RevId: 9904d5a29239168d20833ffef28f516f0d2fa09e
This commit is contained in:
Alexey Katsman
2026-09-04 18:01:34 +00:00
committed by intellij-monorepo-bot
parent d927c2ff4e
commit b6bf184c48
11 changed files with 112 additions and 70 deletions
@@ -142,8 +142,8 @@ class VirtualEnvReader private constructor(
*/
@RequiresBackgroundThread
fun findPythonInPythonRoot(pathOrDir: PythonHomePath): PythonBinary? {
val pythonPattern = getLayout(forcedOs ?: pathOrDir.osFamily)
if (pathOrDir.isRegularFile() && pythonPattern.pyBinaryPattern.matches(pathOrDir.name)) {
val layout = getLayout(forcedOs ?: pathOrDir.osFamily)
if (pathOrDir.isRegularFile() && layout.isPythonBinaryName(pathOrDir.name)) {
return pathOrDir
}
@@ -151,7 +151,7 @@ class VirtualEnvReader private constructor(
return null
}
val bin = pathOrDir.resolve(pythonPattern.dirWithPython)
val bin = pathOrDir.resolve(layout.dirWithPython)
if (bin.isDirectory()) {
findInterpreter(bin)?.let { return it }
}
@@ -163,28 +163,54 @@ class VirtualEnvReader private constructor(
* [binaryOrDir] is either a venv root or a python binary
*/
fun findPythonInPythonRootForTarget(binaryOrDir: FullPathOnTarget, platform: Platform): FullPathOnTarget {
val pythonPattern = getLayout(platform)
val layout = getLayout(platform)
val separator = platform.fileSeparator
val binaryOrDirWithoutSeparatorSuffix = binaryOrDir.removeSuffix(separator.toString())
if (pythonPattern.pyBinaryPattern.matches(binaryOrDirWithoutSeparatorSuffix.substringAfterLast(separator))) {
if (layout.isPythonBinaryName(binaryOrDirWithoutSeparatorSuffix.substringAfterLast(separator))) {
return binaryOrDir
}
return arrayOf(binaryOrDirWithoutSeparatorSuffix,
pythonPattern.dirWithPython,
pythonPattern.defaultPyName).joinToString(separator.toString())
layout.dirWithPython,
layout.defaultPyName).joinToString(separator.toString())
}
/**
* Returns the Python home for [binaryOrDir].
*
* [binaryOrDir] is either the Python binary of a venv or a directory. This function reads no file,
* so it can not tell a file from a directory. It returns [binaryOrDir] unchanged unless the name is
* a Python binary name and the parent is the venv [PythonOsLayout.dirWithPython] directory. A directory
* named `python` therefore stays unchanged.
*
* Use [resolvePythonHomeFromPythonBinary] when the caller knows the path is a Python binary.
*/
fun resolvePythonHomeFromBinaryOrDir(binaryOrDir: Path): PythonHomePath {
val layout = getLayout(forcedOs ?: binaryOrDir.osFamily)
val componentsToDrop = layout.componentsToPythonHome(binaryOrDir.name, binaryOrDir.parent?.name)
return generateSequence(binaryOrDir) { it.parent }.elementAtOrNull(componentsToDrop) ?: binaryOrDir
}
/**
* The [resolvePythonHomeFromBinaryOrDir] rule for a path on a target.
*/
fun resolvePythonHomeFromBinaryOrDir(binaryOrDir: FullPathOnTarget, platform: Platform): FullPathOnTarget {
val layout = getLayout(platform)
val separator = platform.fileSeparator
val binaryOrDirWithoutSeparatorSuffix = binaryOrDir.removeSuffix(separator.toString())
val binaryDirectory = binaryOrDirWithoutSeparatorSuffix.substringBeforeLast(separator)
val componentsToDrop = layout.componentsToPythonHome(binaryOrDirWithoutSeparatorSuffix.substringAfterLast(separator),
binaryDirectory.substringAfterLast(separator))
return if (componentsToDrop == 0) binaryOrDir else binaryDirectory.substringBeforeLast(separator)
}
fun getVenvRoot(path: Path): Path? {
val bin = path.parent
val bin = path.parent ?: return null
val binFolderName = getLayout(forcedOs ?: path.osFamily).dirWithPython
if (bin == null || bin.fileName.pathString != binFolderName) {
if (!getLayout(forcedOs ?: path.osFamily).isDirWithPython(bin.fileName?.pathString)) {
return null
}
val venv = bin.parent
return venv
return bin.parent
}
fun getVenvName(path: Path): String? = getVenvRoot(path)?.name
@@ -193,9 +219,7 @@ class VirtualEnvReader private constructor(
val separator = platform.fileSeparator
val bin = path.substringBeforeLast(separator)
val binFolderName = getLayout(platform).dirWithPython
if (bin.substringAfterLast(separator) != binFolderName) {
if (!getLayout(platform).isDirWithPython(bin.substringAfterLast(separator))) {
return null
}
@@ -203,6 +227,21 @@ class VirtualEnvReader private constructor(
return venv.substringAfterLast(separator).takeIf { it.isNotBlank() }
}
/**
* Returns the Python home for [pythonBinary].
*
* The caller must know that [pythonBinary] is a Python binary, because this function never reads the
* binary name. A binary that [PythonOsLayout.pyBinaryPattern] does not match, such as the macOS
* framework build `python3.13-intel64`, therefore still resolves.
*
* Use [resolvePythonHomeFromBinaryOrDir] when the caller does not know what the path holds.
*/
fun resolvePythonHomeFromPythonBinary(pythonBinary: PythonBinary): PythonHomePath {
val layout = getLayout(forcedOs ?: pythonBinary.osFamily)
val parent = pythonBinary.parent ?: return pythonBinary
return if (layout.isDirWithPython(parent.name)) parent.parent ?: parent else parent
}
/**
* Looks for python binary among directory entries.
* Prefers the shortest name (e.g., "python" over "python3.12") to ensure consistent results,
@@ -213,10 +252,10 @@ class VirtualEnvReader private constructor(
private fun findInterpreter(dir: Path): PythonBinary? =
try {
Files.newDirectoryStream(dir).use { stream ->
val pythonPattern = getLayout(forcedOs ?: dir.osFamily)
val layout = getLayout(forcedOs ?: dir.osFamily)
val candidates = stream.filter {
it.isRegularFile() && pythonPattern.pyBinaryPattern.matches(it.name)
it.isRegularFile() && layout.isPythonBinaryName(it.name)
}.toList()
candidates.minByOrNull { it.name.length }
@@ -266,8 +305,8 @@ class VirtualEnvReader private constructor(
const val PYENV_DEFAULT_DIR_NAME: String = ".pyenv"
private val WIN_LAYOUT =
PythonOsLayout(Regex("^(pypy|pythonw?)(\\d+(\\.\\d+)*)?t?(_d)?\\.exe$", RegexOption.IGNORE_CASE), "Scripts", "python.exe")
private val POSIX_LAYOUT = PythonOsLayout(Regex("^(pypy|pythonw?)(\\d+(\\.\\d+)*)?t?$"), "bin", "python")
PythonOsLayout(Regex("^(pypy|pythonw?)(\\d+(\\.\\d+)*)?t?(_d)?\\.exe$", RegexOption.IGNORE_CASE), "Scripts", "python.exe", true)
private val POSIX_LAYOUT = PythonOsLayout(Regex("^(pypy|pythonw?)(\\d+(\\.\\d+)*)?t?$"), "bin", "python", false)
private fun getLocalEelIfApp(): EelApi? = if (ApplicationManager.getApplication() != null) localEel else null
@@ -294,11 +333,39 @@ class VirtualEnvReader private constructor(
fun VirtualEnvReader(): VirtualEnvReader = Instance
/**
* [pyBinaryPattern] Returns a regex pattern that matches Python binary names.
* The layout of a Python installation on one OS family.
*
* [pyBinaryPattern] is a regex pattern that matches Python binary names.
* Matches: python, python3, python3.X, python3.X.Y, python3.X.Y.Z, etc., pypy, pypy3, pypy3.X, pypy3.X.Y, etc.
* (and .exe versions on Windows).
*
* [dirWithPython] is `Scripts` for Windows, `bin` for POSIX.
* [defaultPyName] is a python name
* [defaultPyName] is a python name.
* [ignoreCase] is true when the OS ignores the case of a file name.
*/
private data class PythonOsLayout(val pyBinaryPattern: Regex, val dirWithPython: String, val defaultPyName: String)
private class PythonOsLayout(
val pyBinaryPattern: Regex,
val dirWithPython: String,
val defaultPyName: String,
private val ignoreCase: Boolean,
) {
/**
* True if [name] is the name of a Python binary.
*/
fun isPythonBinaryName(name: String): Boolean = pyBinaryPattern.matches(name)
/**
* True if [name] is the name of the directory that holds the Python binary of a venv.
*/
fun isDirWithPython(name: String?): Boolean = name != null && name.equals(dirWithPython, ignoreCase = ignoreCase)
/**
* The number of trailing path components between a Python binary and its home.
*
* [lastName] is the name of the path itself. [parentName] is the name of the directory that holds it,
* or null at the root. This function reads no file, so the same rule holds for a local path and for a
* path on a target.
*/
fun componentsToPythonHome(lastName: String, parentName: String?): Int =
if (isPythonBinaryName(lastName) && isDirWithPython(parentName)) 2 else 0
}
@@ -17,7 +17,7 @@ import com.jetbrains.python.PythonBinary
import com.jetbrains.python.Result
import com.jetbrains.python.errorProcessing.PyError
import com.jetbrains.python.getOrNull
import com.jetbrains.python.sdk.impl.resolvePythonHome
import com.jetbrains.python.venvReader.VirtualEnvReader
import io.github.z4kn4fein.semver.Version
import org.jetbrains.annotations.ApiStatus
import java.nio.file.FileVisitOption
@@ -60,7 +60,7 @@ object VenvEvoSdkManager {
}
fun buildEvoSdk(pythonBinaryPath: Path): EvoSdk {
val sdkHome = pythonBinaryPath.resolvePythonHome()
val sdkHome = VirtualEnvReader().resolvePythonHomeFromPythonBinary(pythonBinaryPath)
val name = sdkHome.name
return EvoSdk(
@@ -6,13 +6,13 @@ import com.intellij.openapi.projectRoots.Sdk
import com.intellij.python.community.impl.poetry.common.icons.PythonCommunityImplPoetryCommonIcons
import com.intellij.python.sdk.ui.evolution.sdk.EvoSdk
import com.intellij.python.sdk.ui.evolution.sdk.EvoSdkProvider
import com.jetbrains.python.sdk.impl.resolvePythonHome
import com.jetbrains.python.venvReader.VirtualEnvReader
import java.nio.file.Path
import kotlin.io.path.name
internal object PoetryEvoSdkManager {
fun buildEvoSdk(module: Module, pythonBinaryPath: Path): EvoSdk {
val sdkHome = pythonBinaryPath.resolvePythonHome()
val sdkHome = VirtualEnvReader().resolvePythonHomeFromPythonBinary(pythonBinaryPath)
val venvName = sdkHome.name
val name = venvName.takeIf { !it.startsWith(module.name.lowercase()) }
@@ -1,22 +1,12 @@
package com.jetbrains.python.sdk.impl
import com.intellij.platform.eel.EelOsFamily
import com.intellij.platform.eel.provider.osFamily
import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
import com.jetbrains.python.PythonBinary
import com.jetbrains.python.PythonHomePath
import com.jetbrains.python.venvReader.VirtualEnvReader
import org.jetbrains.annotations.ApiStatus
import kotlin.io.path.name
@RequiresBackgroundThread
@ApiStatus.Internal
fun PythonBinary.resolvePythonHome(): PythonHomePath = when (osFamily) {
EelOsFamily.Windows -> parent.takeIf { it.name.lowercase() != "scripts" } ?: parent.parent
EelOsFamily.Posix -> parent.takeIf { it.name != "bin" } ?: parent.parent
}
@RequiresBackgroundThread
@ApiStatus.Internal
fun PythonHomePath.resolvePythonBinary(): PythonBinary? {
@@ -11,6 +11,7 @@ import com.jetbrains.python.packaging.findCondaExecutableRelativeToEnv
import com.jetbrains.python.packaging.PyCondaPackageService
import com.jetbrains.python.sdk.PythonEnvironment
import com.jetbrains.python.sdk.impl.PySdkBundle.message
import com.jetbrains.python.venvReader.VirtualEnvReader
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
@@ -33,7 +34,7 @@ import kotlin.io.path.listDirectoryEntries
@RequiresBackgroundThread
internal fun PythonBinary.detectPythonEnvironmentImpl(): PyResult<PythonEnvironment> {
if (!isExecutable()) return PyResult.localizedError(message("python.sdk.detect.binary.not.executable", this))
val home = resolvePythonHome()
val home = VirtualEnvReader().resolvePythonHomeFromPythonBinary(this)
val pyvenvCfg = home.resolve("pyvenv.cfg")
if (pyvenvCfg.exists()) {
val venvLibRoot = resolveVenvLibRoot(home)
@@ -8,7 +8,7 @@ import com.intellij.python.junit5Tests.framework.resolvePythonTool
import com.intellij.python.test.env.core.LATEST_PYTHON_VERSION
import com.intellij.python.test.env.core.PyEnvironment
import com.intellij.python.test.env.plain.pythonEnvironment
import com.jetbrains.python.sdk.impl.resolvePythonHome
import com.jetbrains.python.venvReader.VirtualEnvReader
import kotlinx.coroutines.runBlocking
import org.jetbrains.annotations.ApiStatus.Internal
import org.junit.jupiter.api.extension.BeforeAllCallback
@@ -46,7 +46,7 @@ class RequiresPoetryExtension : BeforeAllCallback, BeforeEachCallback {
}, PyEnvironment::class.java)
val pythonBinary = poetryEnv.pythonPath
val poetryPath = pythonBinary.resolvePythonHome().resolvePythonTool("poetry")
val poetryPath = VirtualEnvReader().resolvePythonHomeFromPythonBinary(pythonBinary).resolvePythonTool("poetry")
PropertiesComponent.getInstance().poetryPath = poetryPath.toString()
LOG.info("Poetry configured at: $poetryPath")
@@ -15,7 +15,7 @@ import com.intellij.python.test.env.core.PyEnvironment
import com.intellij.python.test.env.core.PyEnvironmentSpec
import com.intellij.util.containers.orNull
import com.jetbrains.python.PythonBinary
import com.jetbrains.python.sdk.impl.resolvePythonHome
import com.jetbrains.python.venvReader.VirtualEnvReader
import kotlinx.coroutines.runBlocking
import org.jetbrains.annotations.ApiStatus.Internal
import org.junit.jupiter.api.extension.ClassTemplateInvocationContext
@@ -87,14 +87,14 @@ class RunOnEnvironmentsExtension : TestTemplateInvocationContextProvider, ClassT
checkAndGetToolPath(pythonBinary, "poetry", false)?.let { PropertiesComponent.getInstance().poetryPath = it }
checkAndGetToolPath(pythonBinary, "pipenv", false)?.let { PropertiesComponent.getInstance().pipenvPath = it }
val uv = pythonBinary.resolvePythonHome().resolvePythonTool("uv")
val uv = VirtualEnvReader().resolvePythonHomeFromPythonBinary(pythonBinary).resolvePythonTool("uv")
PropertiesComponent.getInstance().setValue("PyCharm.Uv.Path", uv.toString())
return env
}
private fun checkAndGetToolPath(env: PythonBinary, toolName: String, toThrow: Boolean): String? {
val tool = env.resolvePythonHome().resolvePythonTool(toolName)
val tool = VirtualEnvReader().resolvePythonHomeFromPythonBinary(env).resolvePythonTool(toolName)
if (checkedTools[toolName]?.contains(tool) != true) {
val output = try {
CapturingProcessHandler(GeneralCommandLine(tool.toString(), "--version")).runProcess(60_000, true)
@@ -7,6 +7,7 @@ import com.intellij.remote.RemoteSdkPropertiesPaths
import com.jetbrains.python.sdk.PySdkUtil
import com.jetbrains.python.sdk.flavors.PyFlavorData
import com.jetbrains.python.sdk.legacy.PythonSdkUtil
import com.jetbrains.python.venvReader.VirtualEnvReader
import org.jetbrains.annotations.ApiStatus
import java.nio.file.Path
@@ -31,12 +32,10 @@ data class UvSdkFlavorData(
throw IllegalArgumentException("Sdk ${sdk} doesn't have interpreter path set")
}
targetCommandLineBuilder.setExePath(interpreterPath)
val separator = targetCommandLineBuilder.request.targetPlatform.platform.fileSeparator
targetCommandLineBuilder.addEnvironmentVariable("UV_PROJECT_ENVIRONMENT", interpreterPath.parentPath(separator).parentPath(separator))
val envPath = VirtualEnvReader().resolvePythonHomeFromBinaryOrDir(interpreterPath, targetCommandLineBuilder.request.targetPlatform.platform)
targetCommandLineBuilder.addEnvironmentVariable("UV_PROJECT_ENVIRONMENT", envPath)
if (!PythonSdkUtil.isRemote(sdk)) {
PySdkUtil.activateVirtualEnv(sdk)
}
}
private fun String.parentPath(separator: Char): String = removeSuffix(separator.toString()).substringBeforeLast(separator)
}
@@ -9,7 +9,7 @@ import com.intellij.python.pytools.runtime.PyToolRuntime
import com.intellij.python.uv.backend.runtime.UvConstants
import com.intellij.testFramework.common.timeoutRunBlocking
import com.jetbrains.python.PythonBinary
import com.jetbrains.python.sdk.impl.resolvePythonHome
import com.jetbrains.python.venvReader.VirtualEnvReader
import kotlinx.coroutines.Dispatchers
import org.junit.jupiter.api.TestInfo
import java.nio.file.Path
@@ -43,7 +43,7 @@ internal data class UvContext(
}
}
val uvExecutablePath = pythonPath.resolvePythonHome().resolvePythonTool("uv")
val uvExecutablePath = VirtualEnvReader().resolvePythonHomeFromPythonBinary(pythonPath).resolvePythonTool("uv")
val envVars = mapOf(
UvConstants.ConfigEnvVars.CACHE_DIR to uvCacheDirPath.toString(),
@@ -75,7 +75,6 @@ import com.jetbrains.python.sdk.createSdk
import com.jetbrains.python.sdk.getSdksToInstall
import com.jetbrains.python.sdk.impl.PySdkBundle
import com.jetbrains.python.sdk.impl.resolvePythonBinary
import com.jetbrains.python.sdk.impl.resolvePythonHome
import com.jetbrains.python.sdk.isSystemWide
import com.jetbrains.python.target.PyTargetAwareAdditionalData
import com.jetbrains.python.target.PythonLanguageRuntimeConfiguration
@@ -281,14 +280,8 @@ data class EelFileSystem(
return pythonHome.path.resolvePythonBinary()?.let { PathHolder.Eel(it) }
}
override fun resolvePythonHome(pythonHomeOrBinary: PathHolder.Eel): PathHolder.Eel {
val path = pythonHomeOrBinary.path
val fileName = path.fileName?.toString()
val parentName = path.parent?.fileName?.toString()
val isPythonBinary = fileName?.startsWith("python", ignoreCase = true) == true &&
(parentName == "bin" || parentName.equals("scripts", ignoreCase = true))
return if (isPythonBinary) PathHolder.Eel(path.resolvePythonHome()) else pythonHomeOrBinary
}
override fun resolvePythonHome(pythonHomeOrBinary: PathHolder.Eel): PathHolder.Eel =
PathHolder.Eel(VirtualEnvReader().resolvePythonHomeFromBinaryOrDir(pythonHomeOrBinary.path))
override fun getVenvName(pythonHome: PathHolder.Eel): String? {
return resolvePythonBinary(pythonHome)?.let { VirtualEnvReader().getVenvName(it.path) }
@@ -577,16 +570,8 @@ data class TargetFileSystem(
}
override fun resolvePythonHome(pythonHomeOrBinary: PathHolder.Target): PathHolder.Target {
val separator = targetEnvironmentConfiguration.getPlatformAndRoot().platform.fileSeparator
val path = pythonHomeOrBinary.pathString.removeSuffix(separator.toString())
val fileName = path.substringAfterLast(separator).lowercase()
val binaryDirectory = path.substringBeforeLast(separator)
val binaryDirectoryName = binaryDirectory.substringAfterLast(separator)
if (!fileName.startsWith("python") ||
!(binaryDirectoryName.equals("bin", ignoreCase = true) || binaryDirectoryName.equals("scripts", ignoreCase = true))) {
return pythonHomeOrBinary
}
return PathHolder.Target(binaryDirectory.substringBeforeLast(separator))
val platform = targetEnvironmentConfiguration.getPlatformAndRoot().platform
return PathHolder.Target(VirtualEnvReader().resolvePythonHomeFromBinaryOrDir(pythonHomeOrBinary.pathString, platform))
}
override fun getVenvName(pythonHome: PathHolder.Target): String? {
@@ -26,9 +26,9 @@ import com.jetbrains.python.sdk.configuration.VENV_TOOL_ID
import com.jetbrains.python.sdk.configuration.findEnvOrNull
import com.jetbrains.python.sdk.configuration.prepareSdkCreator
import com.jetbrains.python.sdk.createSdk
import com.jetbrains.python.sdk.impl.resolvePythonHome
import com.jetbrains.python.sdk.setAssociationToModule
import com.jetbrains.python.uv.sdk.configuration.isUvEnv
import com.jetbrains.python.venvReader.VirtualEnvReader
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlin.io.path.name
@@ -50,7 +50,7 @@ internal class PyVenvSdkConfiguration : PyProjectSdkConfigurationExtension {
): EnvCheckerResult = withBackgroundProgress(module.project, PyBundle.message("python.sdk.validating.environment")) {
withContext(Dispatchers.IO) {
getVirtualEnv(venvsInModule)?.let {
it.findEnvOrNull(PyBundle.message("sdk.use.existing.venv", it.resolvePythonHome().name))
it.findEnvOrNull(PyBundle.message("sdk.use.existing.venv", VirtualEnvReader().resolvePythonHomeFromPythonBinary(it).name))
} ?: EnvCheckerResult.EnvNotFound(PyBundle.message("sdk.create.venv.suggestion.no.arg"))
}
}