Python tests: separate conda from vanilla python env types for tests.

With this change we can move conda into the separate module to decouple test framework from community.impl GitOrigin-RevId: 589dc29f39d9d46252681c8f31b8fe482e63595f
2026-09-27 10:03:11 +07:00 · 2025-02-02 07:05:23 +00:00
parent 88dc070d16
commit 1be9caaac9
12 changed files with 110 additions and 97 deletions
@@ -943,7 +943,6 @@
      <module fileurl="file://$PROJECT_DIR$/python/huggingFace/intellij.python.community.impl.huggingFace.iml" filepath="$PROJECT_DIR$/python/huggingFace/intellij.python.community.impl.huggingFace.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/installer/intellij.python.community.impl.installer.iml" filepath="$PROJECT_DIR$/python/installer/intellij.python.community.impl.installer.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/poetry/intellij.python.community.impl.poetry.iml" filepath="$PROJECT_DIR$/python/poetry/intellij.python.community.impl.poetry.iml" />
-      <module fileurl="file://$PROJECT_DIR$/python/setup-test-environment/intellij.python.community.impl.setupTestEnv.iml" filepath="$PROJECT_DIR$/python/setup-test-environment/intellij.python.community.impl.setupTestEnv.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/python-venv/intellij.python.community.impl.venv.iml" filepath="$PROJECT_DIR$/python/python-venv/intellij.python.community.impl.venv.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/junit5Tests-framework/intellij.python.community.junit5Tests.framework.iml" filepath="$PROJECT_DIR$/python/junit5Tests-framework/intellij.python.community.junit5Tests.framework.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/pluginCore/intellij.python.community.plugin.iml" filepath="$PROJECT_DIR$/python/pluginCore/intellij.python.community.plugin.iml" />
@@ -957,6 +956,7 @@
      <module fileurl="file://$PROJECT_DIR$/python/services/shared/intellij.python.community.services.shared.iml" filepath="$PROJECT_DIR$/python/services/shared/intellij.python.community.services.shared.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/services/system-python/intellij.python.community.services.systemPython.iml" filepath="$PROJECT_DIR$/python/services/system-python/intellij.python.community.services.systemPython.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/testFramework/intellij.python.community.testFramework.iml" filepath="$PROJECT_DIR$/python/testFramework/intellij.python.community.testFramework.iml" />
+      <module fileurl="file://$PROJECT_DIR$/python/setup-test-environment/intellij.python.community.testFramework.testEnv.iml" filepath="$PROJECT_DIR$/python/setup-test-environment/intellij.python.community.testFramework.testEnv.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/intellij.python.community.tests.iml" filepath="$PROJECT_DIR$/python/intellij.python.community.tests.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/python-copyright/intellij.python.copyright.iml" filepath="$PROJECT_DIR$/python/python-copyright/intellij.python.copyright.iml" />
      <module fileurl="file://$PROJECT_DIR$/python/python-features-trainer/intellij.python.featuresTrainer.iml" filepath="$PROJECT_DIR$/python/python-features-trainer/intellij.python.featuresTrainer.iml" />
@@ -1,11 +1,11 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.junit5Tests.framework.env.impl

-import com.intellij.python.community.testFramework.testEnv.PythonType
+import com.intellij.python.community.testFramework.testEnv.TypeConda
 import com.intellij.python.junit5Tests.framework.env.CondaEnv
 import com.jetbrains.python.sdk.flavors.conda.PyCondaEnv

 /**
 * Mark [PyCondaEnv] param with [CondaEnv] annotation and register this extension with [com.intellij.python.junit5Tests.framework.env.PyEnvTestCaseWithConda]
 */
-internal class CondaPythonEnvExtension : PythonEnvExtensionBase<PyCondaEnv, PythonType.Conda>(CondaEnv::class, PythonType.Conda, PyCondaEnv::class)
+internal class CondaPythonEnvExtension : PythonEnvExtensionBase<PyCondaEnv, TypeConda>(CondaEnv::class, TypeConda, PyCondaEnv::class)
@@ -52,8 +52,8 @@ internal abstract class PythonEnvExtensionBase<ENV : Any, PYTHON_TYPE : PythonTy
  }

  private fun createEnv(): ResourceWrapper<ENV> {
-    val (env, autoClosable) = runBlocking {
-      pythonType.getTestEnvironment(*additionalTags).getOrElse {
+    val (_, autoClosable, env) = runBlocking {
+      pythonType.createSdkClosableEnv(*additionalTags).getOrElse {
        // Logging due to IDEA-356206
        LOG.warn(it)
        val message = "Couldn't find python to run test against ($pythonType , ${additionalTags.toList()})"
@@ -6,7 +6,7 @@ import com.intellij.execution.process.CapturingProcessHandler
 import com.intellij.ide.util.PropertiesComponent
 import com.intellij.openapi.util.SystemInfoRt
 import com.intellij.python.community.impl.poetry.poetryPath
-import com.intellij.python.community.testFramework.testEnv.PythonType
+import com.intellij.python.community.testFramework.testEnv.TypeVanillaPython3
 import com.intellij.python.junit5Tests.framework.env.PythonBinaryPath
 import com.jetbrains.python.PythonBinary
 import java.nio.file.Path
@@ -16,9 +16,9 @@ import java.nio.file.Path
 *
 * It also searches for poetry and stores it in [PropertiesComponent]
 */
-internal class VanillaPythonEnvExtension : PythonEnvExtensionBase<PythonBinary, PythonType.VanillaPython3>(
+internal class VanillaPythonEnvExtension : PythonEnvExtensionBase<PythonBinary, TypeVanillaPython3>(
  annotation = PythonBinaryPath::class,
-  pythonType = PythonType.VanillaPython3,
+  pythonType = TypeVanillaPython3,
  envType = PythonBinary::class,
  lazy = false,
  additionalTags = arrayOf("poetry")
@@ -8,6 +8,7 @@ import com.intellij.openapi.module.Module
 import com.intellij.openapi.projectRoots.ProjectJdkTable
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.python.community.testFramework.testEnv.PythonType
+import com.intellij.python.community.testFramework.testEnv.TypeVanillaPython3
 import com.intellij.testFramework.junit5.fixture.TestFixture
 import com.intellij.testFramework.junit5.fixture.testFixture
 import com.jetbrains.python.sdk.configuration.createVirtualEnvSynchronously
@@ -25,7 +26,7 @@ typealias PySdkFixture = TestFixture<Sdk>
 * Creates [Sdk] (if you only need a python path, use [PythonBinaryPath] or [CondaEnv])
 */
 fun pySdkFixture(
-  pythonType: PythonType<*> = PythonType.VanillaPython3,
+  pythonType: PythonType<*> = TypeVanillaPython3,
 ): PySdkFixture = testFixture {
  val (sdk, autoClosable) = com.jetbrains.python.tools.createSdk(LocalPython(pythonType))
  sdk.persist()
@@ -5,7 +5,7 @@ import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.util.Disposer
 import com.intellij.platform.eel.EelApi
 import com.intellij.platform.eel.provider.localEel
-import com.intellij.python.community.testFramework.testEnv.PythonType
+import com.intellij.python.community.testFramework.testEnv.TypeVanillaPython3
 import com.jetbrains.python.PythonBinary
 import com.jetbrains.python.systemPythonSpi.SystemPythonProvider
 import kotlinx.coroutines.flow.map
@@ -17,7 +17,7 @@ import kotlinx.coroutines.flow.toSet
 internal class EnvTestPythonProvider : SystemPythonProvider {
  override suspend fun findSystemPythons(eelApi: EelApi): Set<PythonBinary> {
    if (eelApi != localEel) return emptySet()
-    return PythonType.VanillaPython3
+    return TypeVanillaPython3
      .getTestEnvironments()
      .map { (python, closeable) ->
        Disposer.register(ApplicationManager.getApplication()) {
@@ -15,5 +15,6 @@
    <orderEntry type="module" module-name="intellij.python.community.impl" />
    <orderEntry type="module" module-name="intellij.platform.execution" />
    <orderEntry type="library" name="kotlin-reflect" level="project" />
+    <orderEntry type="module" module-name="intellij.platform.lang.impl" />
  </component>
 </module>
@@ -1,32 +1,22 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.python.community.testFramework.testEnv

-import com.intellij.execution.processTools.getResultStdout
-import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
-import com.intellij.openapi.diagnostic.logger
-import com.intellij.openapi.util.Disposer
-import com.intellij.openapi.vfs.newvfs.impl.VfsRootAccess
+import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.util.concurrency.annotations.RequiresBackgroundThread
 import com.jetbrains.python.PythonBinary
-import com.jetbrains.python.packaging.findCondaExecutableRelativeToEnv
-import com.jetbrains.python.sdk.conda.TargetEnvironmentRequestCommandExecutor
 import com.jetbrains.python.sdk.flavors.PythonSdkFlavor
-import com.jetbrains.python.sdk.flavors.conda.PyCondaEnv
-import com.jetbrains.python.sdk.flavors.conda.PyCondaEnvIdentity
 import com.jetbrains.python.venvReader.VirtualEnvReader
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.*
-import kotlinx.coroutines.runBlocking
 import org.jetbrains.annotations.NonNls
 import java.nio.file.Path
 import java.util.concurrent.ConcurrentHashMap
 import kotlin.Result.Companion.failure

 /**
- * Gradle script installs two types of python: conda and vanilla. Env could be obtained by [getTestEnvironment] which also provides closable
+ * Gradle script installs two types of python: conda and vanilla. Env could be obtained by [createSdkClosableEnv] which also provides closable
 * to clean up after the usage
 */
-sealed class PythonType<T : Any>(private val tag: @NonNls String? = null) {
+abstract class PythonType<T : Any>(private val tag: @NonNls String) {
  private companion object {
    val cache: MutableMap<Set<String>, List<Path>> = ConcurrentHashMap()
  }
@@ -65,72 +55,23 @@ sealed class PythonType<T : Any>(private val tag: @NonNls String? = null) {


  /**
-   * Returns first (whatever it means) test environment and closable that must be closed after the test
+   * Returns sdk, (whatever it means) test environment and closable that must be closed after the test
   */
-  suspend fun getTestEnvironment(vararg additionalTags: @NonNls String): Result<Pair<T, AutoCloseable>> =
-    getTestEnvironments(*additionalTags).firstOrNull()?.let { Result.success(it) }
+  suspend fun createSdkClosableEnv(vararg additionalTags: @NonNls String): Result<Triple<Sdk, AutoCloseable, T>> =
+    getTestEnvironments(*additionalTags).firstOrNull()?.let { (env, closable) ->
+      Result.success(Triple(createSdkFor(env), closable, env))
+    }
    ?: failure(AssertionError("No python found. See ${PyEnvTestSettings::class} class for more info"))

+  protected abstract suspend fun createSdkFor(t: T): Sdk
+

  protected abstract suspend fun pythonPathToEnvironment(pythonBinary: PythonBinary, envDir: Path): Pair<T, AutoCloseable>


-  data object VanillaPython3 : PythonType<PythonBinary>() {
-    // Python is directly executable
-    override suspend fun pythonPathToEnvironment(pythonBinary: PythonBinary, envDir: Path): Pair<PythonBinary, AutoCloseable> {
-      val disposable = Disposer.newDisposable("Python tests disposable for VfsRootAccess")
-      // We might have python installation outside the project root, but we still need to have access to it.
-      VfsRootAccess.allowRootAccess(disposable, pythonBinary.parent.toString())
-      return Pair(pythonBinary, AutoCloseable {
-        Disposer.dispose(disposable)
-      })
-    }
-  }
-
-
-  data object Conda : PythonType<PyCondaEnv>("conda") {
-
-    override suspend fun pythonPathToEnvironment(pythonBinary: PythonBinary, envDir: Path): Pair<PyCondaEnv, AutoCloseable> {
-      // First, find python binary, then calculate conda from it as env stores "conda" as a regular env
-      val condaPath = findCondaExecutableRelativeToEnv(pythonBinary) ?: error("Conda root $pythonBinary doesn't have conda binary")
-
-      // We'll remove then on close
-      val condaEnvsBeforeTest = getCondaNames(condaPath)
-
-      val cleanupCondas = AutoCloseable {
-        runBlocking(Dispatchers.IO) {
-          val condasToRemove = getCondaNames(condaPath)
-          condasToRemove.removeAll(condaEnvsBeforeTest)
-          for (envName in condasToRemove) {
-            println("Removing $envName")
-
-            for (arg in arrayOf("--name", "-p")) {
-              val args = arrayOf(condaPath.toString(), "remove", arg, envName, "--all", "-y")
-              Runtime.getRuntime().exec(args).getResultStdout().getOrElse {
-                logger<Conda>().warn(it)
-              }
-            }
-          }
-        }
-      }
-      return Pair(PyCondaEnv(PyCondaEnvIdentity.UnnamedEnv(envDir.toString(), isBase = true), condaPath.toString()), cleanupCondas)
-    }
-
-    private suspend fun getCondaNames(condaPath: Path) =
-      PyCondaEnv.getEnvs(TargetEnvironmentRequestCommandExecutor(LocalTargetEnvironmentRequest()),
-                         condaPath.toString()).getOrThrow()
-        .map { it.envIdentity.userReadableName }
-        .toMutableSet()
-  }
-
-
  @RequiresBackgroundThread
  private fun typeMatchesEnv(env: Path, vararg additionalTags: @NonNls String): Boolean {
    val envTags = loadEnvTags(env)
-
-    for (badTag in PythonType::class.sealedSubclasses.filterNot { it.isInstance(this) }.map { it.objectInstance!!.tag }) {
-      if (badTag != null && badTag in envTags) return false
-    }
-    return (tag == null || tag in envTags) && additionalTags.all { it in envTags }
+    return tag in envTags && additionalTags.all { it in envTags }
  }
 }
@@ -0,0 +1,52 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.community.testFramework.testEnv
+
+import com.intellij.execution.processTools.getResultStdout
+import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
+import com.intellij.openapi.diagnostic.logger
+import com.intellij.openapi.projectRoots.Sdk
+import com.jetbrains.python.PythonBinary
+import com.jetbrains.python.packaging.findCondaExecutableRelativeToEnv
+import com.jetbrains.python.sdk.conda.TargetEnvironmentRequestCommandExecutor
+import com.jetbrains.python.sdk.flavors.conda.PyCondaEnv
+import com.jetbrains.python.sdk.flavors.conda.PyCondaEnvIdentity
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import java.nio.file.Path
+
+data object TypeConda : PythonType<PyCondaEnv>("conda") {
+  override suspend fun createSdkFor(env: PyCondaEnv): Sdk =
+    env.createSdkFromThisEnv(null, emptyList())
+
+  override suspend fun pythonPathToEnvironment(pythonBinary: PythonBinary, envDir: Path): Pair<PyCondaEnv, AutoCloseable> {
+    // First, find python binary, then calculate conda from it as env stores "conda" as a regular env
+    val condaPath = findCondaExecutableRelativeToEnv(pythonBinary) ?: error("Conda root $pythonBinary doesn't have conda binary")
+
+    // We'll remove then on close
+    val condaEnvsBeforeTest = getCondaNames(condaPath)
+
+    val cleanupCondas = AutoCloseable {
+      runBlocking(Dispatchers.IO) {
+        val condasToRemove = getCondaNames(condaPath)
+        condasToRemove.removeAll(condaEnvsBeforeTest)
+        for (envName in condasToRemove) {
+          println("Removing $envName")
+
+          for (arg in arrayOf("--name", "-p")) {
+            val args = arrayOf(condaPath.toString(), "remove", arg, envName, "--all", "-y")
+            Runtime.getRuntime().exec(args).getResultStdout().getOrElse {
+              logger<TypeConda>().warn(it)
+            }
+          }
+        }
+      }
+    }
+    return Pair(PyCondaEnv(PyCondaEnvIdentity.UnnamedEnv(envDir.toString(), isBase = true), condaPath.toString()), cleanupCondas)
+  }
+
+  private suspend fun getCondaNames(condaPath: Path) =
+    PyCondaEnv.Companion.getEnvs(TargetEnvironmentRequestCommandExecutor(LocalTargetEnvironmentRequest()),
+                                 condaPath.toString()).getOrThrow()
+      .map { it.envIdentity.userReadableName }
+      .toMutableSet()
+}
@@ -0,0 +1,28 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.python.community.testFramework.testEnv
+
+import com.intellij.openapi.projectRoots.Sdk
+import com.intellij.openapi.projectRoots.SdkType
+import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil
+import com.intellij.openapi.util.Disposer
+import com.intellij.openapi.vfs.newvfs.impl.VfsRootAccess
+import com.intellij.testFramework.utils.vfs.refreshAndGetVirtualFile
+import com.jetbrains.python.PyNames
+import com.jetbrains.python.PythonBinary
+import java.nio.file.Path
+
+data object TypeVanillaPython3 : PythonType<PythonBinary>("python3") {
+  override suspend fun createSdkFor(python: PythonBinary): Sdk =
+    SdkConfigurationUtil.setupSdk(emptyArray(), python.refreshAndGetVirtualFile(),
+                                  SdkType.findByName(PyNames.PYTHON_SDK_ID_NAME)!!, null, null)
+
+  // Python is directly executable
+  override suspend fun pythonPathToEnvironment(pythonBinary: PythonBinary, envDir: Path): Pair<PythonBinary, AutoCloseable> {
+    val disposable = Disposer.newDisposable("Python tests disposable for VfsRootAccess")
+    // We might have python installation outside the project root, but we still need to have access to it.
+    VfsRootAccess.allowRootAccess(disposable, pythonBinary.parent.toString())
+    return Pair(pythonBinary, AutoCloseable {
+      Disposer.dispose(disposable)
+    })
+  }
+}
@@ -6,10 +6,9 @@ import com.intellij.execution.target.TargetEnvironmentConfiguration
 import com.intellij.openapi.application.writeAction
 import com.intellij.openapi.projectRoots.ProjectJdkTable
 import com.intellij.openapi.projectRoots.Sdk
-import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil
 import com.intellij.python.community.testFramework.testEnv.PythonType
+import com.intellij.python.community.testFramework.testEnv.TypeVanillaPython3
 import com.intellij.remote.RemoteSdkException
-import com.intellij.testFramework.utils.vfs.refreshAndGetVirtualFile
 import com.jetbrains.python.sdk.PythonSdkType
 import com.jetbrains.python.sdk.flavors.PyFlavorAndData
 import com.jetbrains.python.sdk.flavors.PyFlavorData
@@ -24,7 +23,7 @@ import org.junit.Assume
 * To be used with [createSdk]
 */
 sealed class SdkCreationRequest {
-  data class LocalPython(val pythonType: PythonType<*> = PythonType.VanillaPython3) : SdkCreationRequest()
+  data class LocalPython(val pythonType: PythonType<*> = TypeVanillaPython3) : SdkCreationRequest()
  data class RemotePython(val targetConfig: TargetEnvironmentConfiguration) : SdkCreationRequest()
 }

@@ -34,17 +33,8 @@ sealed class SdkCreationRequest {
 suspend fun createSdk(request: SdkCreationRequest): Pair<Sdk, AutoCloseable> = withContext(Dispatchers.IO) {
  when (request) {
    is SdkCreationRequest.LocalPython -> {
-      when (val pythonType = request.pythonType) {
-        is PythonType.Conda -> {
-          val (env, closable) = pythonType.getTestEnvironment().getOrThrow()
-          Pair(env.createSdkFromThisEnv(null, emptyList()), closable)
-        }
-        is PythonType.VanillaPython3 -> {
-          val (python, closable) = pythonType.getTestEnvironment().getOrThrow()
-          Pair(SdkConfigurationUtil.setupSdk(emptyArray(), python.refreshAndGetVirtualFile(),
-                                             PythonSdkType.getInstance(), null, null), closable)
-        }
-      }
+      val (sdk, closable, _) = request.pythonType.createSdkClosableEnv().getOrThrow()
+      Pair(sdk, closable)
    }
    is SdkCreationRequest.RemotePython -> {
      val targetData = PyTargetAwareAdditionalData(PyFlavorAndData(PyFlavorData.Empty, UnixPythonSdkFlavor.getInstance()),
@@ -3,7 +3,7 @@ package com.jetbrains.env.python.conda

 import com.intellij.execution.target.FullPathOnTarget
 import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
-import com.intellij.python.community.testFramework.testEnv.PythonType
+import com.intellij.python.community.testFramework.testEnv.TypeConda
 import com.jetbrains.python.sdk.conda.TargetCommandExecutor
 import com.jetbrains.python.sdk.conda.TargetEnvironmentRequestCommandExecutor
 import com.jetbrains.python.sdk.flavors.conda.PyCondaCommand
@@ -34,7 +34,7 @@ class LocalCondaRule : ExternalResource() {

  override fun before() {
    super.before()
-    val (condaPathEnv, autoCloseable) = runBlocking { PythonType.Conda.getTestEnvironment().getOrElse { throw AssumptionViolatedException("No conda found, run gradle script to install test env") } }
+    val (_, autoCloseable, condaPathEnv) = runBlocking { TypeConda.createSdkClosableEnv().getOrElse { throw AssumptionViolatedException("No conda found, run gradle script to install test env") } }

    condaPath = Path.of(condaPathEnv.fullCondaPathOnTarget)
    if (!condaPath.isExecutable()) {