PY-35978: Improve Conda support and refactor other parts to support it.

Each sdk has additional data with flavor and flavor-specific data. For target-based SDK there is also target information. ``PySdkExt`` has extension method that uses this data to execute code on some SDK. For Conda we store path to conda binary and env name.

GitOrigin-RevId: c63b57aac9b5a267b3a6710902670bfe7d10c722
This commit is contained in:
Ilya.Kazakevich
2022-10-12 00:13:28 +02:00
committed by intellij-monorepo-bot
parent 63e8b16ace
commit a4dcfdd16e
92 changed files with 2056 additions and 661 deletions

View File

@@ -0,0 +1,38 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains
import com.intellij.execution.processTools.getBareExecutionResult
import com.intellij.execution.target.TargetEnvironmentRequest
import com.intellij.execution.target.TargetProgressIndicator
import com.intellij.execution.target.TargetedCommandLineBuilder
import com.intellij.openapi.projectRoots.Sdk
import com.jetbrains.python.sdk.flavors.PythonSdkFlavor
import com.jetbrains.python.sdk.configureBuilderToRunPythonOnTarget
import com.jetbrains.python.sdk.getOrCreateAdditionalData
import org.junit.Assert
internal fun getPythonVersion(sdk: Sdk, request: TargetEnvironmentRequest): String? {
val commandLineBuilder = TargetedCommandLineBuilder(request)
sdk.configureBuilderToRunPythonOnTarget(commandLineBuilder)
val flavor = sdk.getOrCreateAdditionalData().flavor
return getPythonVersion(commandLineBuilder, flavor, request)
}
internal fun getPythonVersion(commandLineBuilder: TargetedCommandLineBuilder,
flavor: PythonSdkFlavor<*>,
request: TargetEnvironmentRequest): String? {
commandLineBuilder.addParameter(flavor.versionOption)
val commandLine = commandLineBuilder.build()
val result = request
.prepareEnvironment(TargetProgressIndicator.EMPTY)
.createProcess(commandLine).getBareExecutionResult().get()
// Conda python may send version to stderr, check both
val err = result.stdErr.decodeToString()
val out = result.stdOut.decodeToString()
Assert.assertEquals(err, 0, result.exitCode)
val versionString = out.ifBlank { err }.trim()
return flavor.getVersionStringFromOutput(versionString)
}

View File

@@ -0,0 +1,32 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env
import com.intellij.openapi.application.invokeAndWaitIfNeeded
import com.intellij.openapi.projectRoots.impl.ProjectJdkImpl
import com.intellij.testFramework.ProjectRule
import com.jetbrains.python.sdk.PythonSdkAdditionalData
import com.jetbrains.python.sdk.PythonSdkType
import com.jetbrains.python.sdk.getOrCreateAdditionalData
import org.jdom.Element
import org.junit.Assert
import org.junit.Rule
import org.junit.Test
class PySdkAdditionalDataSaveRestoreTest {
@JvmField
@Rule
val projectRule: ProjectRule = ProjectRule()
@Test
fun test() = invokeAndWaitIfNeeded {
val sdk = ProjectJdkImpl("mySdk", PythonSdkType.getInstance(), "path", "ver")
val data = sdk.getOrCreateAdditionalData()
val uuid = data.uuid
val elem = Element("root")
data.save(elem)
val newData = PythonSdkAdditionalData.loadFromElement(elem)
Assert.assertEquals("uuid didn't survive reloading", uuid, newData.uuid)
}
}

View File

@@ -0,0 +1,15 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env.conda
import java.io.File
import java.nio.file.Path
import kotlin.io.path.exists
/**
* See content of [yamlFile]
*/
internal const val yamlEnvName = "envFromFile"
internal val yamlFile: Path
get() = File(PyCondaTest::class.java.classLoader.getResource("com/jetbrains/env/conda/environment.yml")!!.path).toPath().also {
assert(it.exists()) { "Can't file environment file in tests" }
}

View File

@@ -0,0 +1,40 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env.conda
import com.jetbrains.env.conda.LocalCondaRule.Companion.CONDA_PATH
import com.jetbrains.python.FullPathOnTarget
import com.jetbrains.python.sdk.add.target.conda.suggestCondaPath
import com.jetbrains.python.sdk.flavors.conda.PyCondaCommand
import kotlinx.coroutines.runBlocking
import org.junit.AssumptionViolatedException
import org.junit.rules.ExternalResource
import java.nio.file.Path
import kotlin.io.path.isExecutable
/**
* Finds conda on local system using [CONDA_PATH] env var.
*
* To be fixed: support targets as well
*/
class LocalCondaRule : ExternalResource() {
private companion object {
const val CONDA_PATH = "CONDA_PATH"
}
lateinit var condaPath: Path
private set
val condaPathOnTarget: FullPathOnTarget get() = condaPath.toString()
val condaCommand: PyCondaCommand get() = PyCondaCommand(condaPathOnTarget, null)
override fun before() {
super.before()
val condaPathEnv = System.getenv()[CONDA_PATH]
?: runBlocking { suggestCondaPath(null) }
?: throw AssumptionViolatedException("No $CONDA_PATH set")
condaPath = Path.of(condaPathEnv)
if (!condaPath.isExecutable()) {
throw AssumptionViolatedException("$condaPath is not executable")
}
}
}

View File

@@ -0,0 +1,13 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env.conda
import com.jetbrains.python.sdk.flavors.conda.NewCondaEnvRequest
import org.junit.Assert
import org.junit.Test
class LocalEnvByLocalEnvironmentFileTest {
@Test
fun parseYaml() {
Assert.assertEquals("Wrong name parsed out of yaml file", yamlEnvName, NewCondaEnvRequest.LocalEnvByLocalEnvironmentFile(yamlFile).envName)
}
}

View File

@@ -0,0 +1,117 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env.conda
import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
import com.intellij.openapi.progress.ProgressSink
import com.intellij.testFramework.ProjectRule
import com.intellij.util.io.exists
import com.jetbrains.getPythonVersion
import com.jetbrains.python.psi.LanguageLevel
import com.jetbrains.python.sdk.add.target.conda.PyAddCondaPanelModel
import com.jetbrains.python.sdk.flavors.conda.PyCondaEnvIdentity
import com.jetbrains.python.sdk.flavors.conda.PyCondaFlavorData
import com.jetbrains.python.sdk.getOrCreateAdditionalData
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.test.runTest
import org.junit.Assert
import org.junit.Rule
import org.junit.Test
import java.nio.file.Path
@OptIn(ExperimentalCoroutinesApi::class)
class PyAddCondaPanelModelTest {
@JvmField
@Rule
val condaRule: LocalCondaRule = LocalCondaRule()
@JvmField
@Rule
val projectRule: ProjectRule = ProjectRule()
@Test
fun testCondaDetection(): Unit = runTest {
val model = PyAddCondaPanelModel(null, emptyList(), projectRule.project)
model.detectConda(coroutineContext)
val detectedPath = model.condaPathTextBoxRwProp.get()
if (detectedPath.isNotEmpty()) {
Assert.assertEquals("Wrong path detected", condaRule.condaPathOnTarget, detectedPath)
}
}
@Test
fun testCondaCreateNewEnv() {
val condaName = "someNewCondaEnv"
runBlocking {
val model = PyAddCondaPanelModel(null, emptyList(), projectRule.project)
model.condaPathTextBoxRwProp.set(condaRule.condaPath.toString())
model.condaActionCreateNewEnvRadioRwProp.set(true)
model.condaActionUseExistingEnvRadioRwProp.set(false)
model.newEnvLanguageLevelRwProperty.set(LanguageLevel.PYTHON38)
Assert.assertNotNull("Empty conda env name didn't lead to validation", model.getValidationError())
model.newEnvNameRwProperty.set("d f --- ")
Assert.assertNotNull("Bad conda name didn't lead to validation", model.getValidationError())
model.newEnvNameRwProperty.set(condaName)
val mockSink = MockSink()
val sdk = model.onCondaCreateSdkClicked(coroutineContext, mockSink).getOrThrow()
val newName = ((sdk.getOrCreateAdditionalData().flavorAndData.data as PyCondaFlavorData).env.envIdentity as PyCondaEnvIdentity.NamedEnv).envName
Assert.assertEquals("Wrong conda name", condaName, newName)
Assert.assertTrue("No output provided for sink", mockSink.out.toString().isNotEmpty())
}
}
@Test
fun testCondaUseExistingEnv(): Unit = runTest {
val model = PyAddCondaPanelModel(null, emptyList(), projectRule.project)
model.condaPathTextBoxRwProp.set(condaRule.condaPath.toString())
model.onCondaPathSetOkClicked(coroutineContext)
model.condaActionUseExistingEnvRadioRwProp.set(true)
model.condaActionCreateNewEnvRadioRwProp.set(false)
model.condaEnvModel.selectedItem = model.condaEnvModel.getElementAt(0)
val sdk = model.onCondaCreateSdkClicked(coroutineContext, null).getOrThrow()
Assert.assertTrue(getPythonVersion(sdk, LocalTargetEnvironmentRequest())!!.isNotBlank())
Assert.assertTrue(Path.of(sdk.homePath!!).exists())
}
@Test
fun testCondaModelValidation(): Unit = runTest {
val model = PyAddCondaPanelModel(null, emptyList(), projectRule.project)
Assert.assertNotNull("No validation error, even though path not set", model.getValidationError())
Assert.assertFalse(model.showCondaPathSetOkButtonRoProp.get())
model.condaPathTextBoxRwProp.set("Some random path that doesn't exist")
Assert.assertTrue(model.showCondaPathSetOkButtonRoProp.get())
model.onCondaPathSetOkClicked(coroutineContext)
Assert.assertNotNull("No validation error, but path is incorrect", model.getValidationError())
model.condaPathTextBoxRwProp.set(condaRule.condaPath.toString())
model.onCondaPathSetOkClicked(coroutineContext)
Assert.assertNull("Unexpected validation error", model.getValidationError())
Assert.assertTrue(model.showCondaActionsPanelRoProp.get())
model.condaActionCreateNewEnvRadioRwProp.set(true)
Assert.assertTrue("No conda envs loaded", model.condaEnvModel.size > 0)
model.condaActionCreateNewEnvRadioRwProp.set(true)
model.condaActionUseExistingEnvRadioRwProp.set(false)
Assert.assertNotNull("No validation error, but conda env name not set", model.getValidationError())
model.newEnvNameRwProperty.set("SomeEnvName")
Assert.assertNull("Unexpected error", model.getValidationError())
}
/**
* Mock doesn't work with Kotlin, hence mock manually
*/
private class MockSink : ProgressSink {
val out = StringBuilder()
override fun update(text: String?, details: String?, fraction: Double?) {
text?.let { out.append(it) }
}
}
}

View File

@@ -0,0 +1,42 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env.conda
import com.intellij.openapi.projectRoots.Sdk
import com.intellij.testFramework.ProjectRule
import com.jetbrains.python.sdk.PythonSdkAdditionalData
import com.jetbrains.python.sdk.flavors.PyFlavorAndData
import com.jetbrains.python.sdk.flavors.conda.CondaEnvSdkFlavor
import com.jetbrains.python.sdk.flavors.conda.PyCondaEnv
import com.jetbrains.python.sdk.flavors.conda.PyCondaEnvIdentity
import com.jetbrains.python.sdk.flavors.conda.PyCondaFlavorData
import org.jdom.Element
import org.junit.Assert
import org.junit.Rule
import org.junit.Test
import org.mockito.Mockito.mock
import org.mockito.Mockito.`when`
class PyCondaAdditionalDataTest {
@JvmField
@Rule
val projectRule: ProjectRule = ProjectRule()
@Test
fun testSerialize() {
val flavorData = PyCondaFlavorData(PyCondaEnv(PyCondaEnvIdentity.NamedEnv("D"), "foo"))
val data = PythonSdkAdditionalData(
PyFlavorAndData(flavorData, CondaEnvSdkFlavor.getInstance()))
val rootElement = Element("root")
data.save(rootElement)
val sdk = mock(Sdk::class.java)
`when`(sdk.homePath).thenReturn("foo")
val reloadedData = PythonSdkAdditionalData.loadFromElement(rootElement)
Assert.assertEquals(reloadedData.flavor, CondaEnvSdkFlavor.getInstance())
Assert.assertEquals(reloadedData.flavorAndData.data, flavorData)
}
}

View File

@@ -0,0 +1,62 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env.conda
import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
import com.intellij.openapi.projectRoots.Sdk
import com.intellij.testFramework.ProjectRule
import com.jetbrains.getPythonVersion
import com.jetbrains.python.sdk.add.target.conda.createCondaSdkAlongWithNewEnv
import com.jetbrains.python.sdk.add.target.conda.createCondaSdkFromExistingEnv
import com.jetbrains.python.sdk.flavors.conda.*
import com.jetbrains.python.sdk.getOrCreateAdditionalData
import com.jetbrains.python.sdk.getPythonBinaryPath
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.runTest
import org.junit.Assert
import org.junit.Rule
import org.junit.Test
import java.nio.file.Files
import java.nio.file.Path
/**
* Ensures conda SDK could be created
*/
@OptIn(ExperimentalCoroutinesApi::class)
class PyCondaSdkTest {
@JvmField
@Rule
val condaRule: LocalCondaRule = LocalCondaRule()
@JvmField
@Rule
val projectRule: ProjectRule = ProjectRule()
@Test
fun createSdkByFile() = runTest {
val newCondaInfo = NewCondaEnvRequest.LocalEnvByLocalEnvironmentFile(yamlFile)
val sdk = condaRule.condaCommand.createCondaSdkAlongWithNewEnv(newCondaInfo, coroutineContext, emptyList(),
projectRule.project).getOrThrow()
val env = (sdk.getOrCreateAdditionalData().flavorAndData.data as PyCondaFlavorData).env
val namedEnv = env.envIdentity as PyCondaEnvIdentity.NamedEnv
Assert.assertEquals("Wrong env name", yamlEnvName, namedEnv.envName)
ensureHomePathCorrect(sdk)
}
@Test
fun testCreateFromExisting() = runTest {
val env = PyCondaEnv.getEnvs(condaRule.condaCommand).getOrThrow().first()
val sdk = condaRule.condaCommand.createCondaSdkFromExistingEnv(env.envIdentity, emptyList(), projectRule.project)
Assert.assertEquals(sdk.getOrCreateAdditionalData().flavor, CondaEnvSdkFlavor.getInstance())
Assert.assertTrue(env.toString(), getPythonVersion(sdk, LocalTargetEnvironmentRequest())?.isNotBlank() == true)
Assert.assertTrue("Bad home path", Files.isExecutable(Path.of(sdk.homePath!!)))
ensureHomePathCorrect(sdk)
}
private suspend fun ensureHomePathCorrect(sdk: Sdk) {
val homePath = sdk.homePath!!
Assert.assertEquals("Wrong home path", homePath, sdk.getPythonBinaryPath(projectRule.project).getOrThrow())
}
}

View File

@@ -0,0 +1,77 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.env.conda
import com.intellij.execution.processTools.getResultStdoutStr
import com.intellij.execution.target.TargetedCommandLineBuilder
import com.intellij.execution.target.local.LocalTargetEnvironmentRequest
import com.intellij.testFramework.ProjectRule
import com.jetbrains.getPythonVersion
import com.jetbrains.python.psi.LanguageLevel
import com.jetbrains.python.sdk.flavors.conda.CondaEnvSdkFlavor
import com.jetbrains.python.sdk.flavors.conda.NewCondaEnvRequest
import com.jetbrains.python.sdk.flavors.conda.PyCondaEnv
import com.jetbrains.python.sdk.flavors.conda.PyCondaEnvIdentity
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.runTest
import org.junit.Assert
import org.junit.Rule
import org.junit.Test
@OptIn(ExperimentalCoroutinesApi::class)
class PyCondaTest {
@JvmField
@Rule
val projectRule: ProjectRule = ProjectRule()
@JvmField
@Rule
val condaRule: LocalCondaRule = LocalCondaRule()
@Test
fun testCondaCreateByYaml() = runTest {
PyCondaEnv.createEnv(condaRule.condaCommand,
NewCondaEnvRequest.LocalEnvByLocalEnvironmentFile(yamlFile)).getOrThrow().getResultStdoutStr().get().getOrThrow()
val condaEnv = PyCondaEnv.getEnvs(condaRule.condaCommand).getOrThrow()
.first { (it.envIdentity as? PyCondaEnvIdentity.NamedEnv)?.envName == yamlEnvName }
// Python version contains word "Python", LanguageLevel doesn't expect it
val pythonVersion = getPythonVersion(condaEnv).trimStart { !it.isDigit() && it != '.' }
Assert.assertEquals("Wrong python version installed", LanguageLevel.PYTHON310,
LanguageLevel.fromPythonVersion(pythonVersion))
}
@Test
fun testCondaCreateEnv(): Unit = runTest {
val envName = "myNewEnvForTests"
PyCondaEnv.createEnv(condaRule.condaCommand, NewCondaEnvRequest.EmptyNamedEnv(LanguageLevel.PYTHON39, envName))
.getOrThrow().getResultStdoutStr().get().getOrThrow()
PyCondaEnv.getEnvs(condaRule.condaCommand).getOrThrow().first { (it.envIdentity as? PyCondaEnvIdentity.NamedEnv)?.envName == envName }
}
@Test
fun testCondaListEnvs(): Unit = runTest {
val condaEnvs = PyCondaEnv.getEnvs(condaRule.condaCommand).getOrThrow()
Assert.assertTrue("No environments returned", condaEnvs.isNotEmpty())
var baseFound = false
for (condaEnv in condaEnvs) {
val version = getPythonVersion(condaEnv)
Assert.assertTrue(condaEnv.envIdentity.toString(), version.isNotBlank())
println("${condaEnv.envIdentity}: $version")
if ((condaEnv.envIdentity as? PyCondaEnvIdentity.UnnamedEnv)?.isBase == true) {
Assert.assertFalse("More than one base environment", baseFound)
baseFound = true
}
}
Assert.assertTrue("No base conda found", baseFound);
}
private fun getPythonVersion(condaEnv: PyCondaEnv): String {
val req = LocalTargetEnvironmentRequest()
val commandLine = TargetedCommandLineBuilder(req).also { condaEnv.addCondaToTargetBuilder(it) }
commandLine.addParameter("python")
return getPythonVersion(commandLine, CondaEnvSdkFlavor.getInstance(), req) ?: error("No version for $condaEnv")
}
}

View File

@@ -0,0 +1,22 @@
name: envFromFile
channels:
- defaults
dependencies:
- bzip2=1.0.8=he774522_0
- ca-certificates=2022.4.26=haa95532_0
- certifi=2022.6.15=py310haa95532_0
- libffi=3.4.2=hd77b12b_4
- openssl=1.1.1p=h2bbff1b_0
- pip=21.2.4=py310haa95532_0
- python=3.10.4=hbb2ffb3_0
- setuptools=61.2.0=py310haa95532_0
- sqlite=3.38.5=h2bbff1b_0
- tk=8.6.12=h2bbff1b_0
- tzdata=2022a=hda174b7_0
- vc=14.2=h21ff451_1
- vs2015_runtime=14.27.29016=h5e58377_2
- wheel=0.37.1=pyhd3eb1b0_0
- wincertstore=0.2=py310haa95532_2
- xz=5.2.5=h8cc25b3_1
- zlib=1.2.12=h8cc25b3_2
prefix: /some/path

View File

@@ -20,15 +20,11 @@ class PySdkAdditionalDataTest {
*/
@Test
fun saveLoadTest() {
val sut = PythonSdkAdditionalData(null)
val sut = PythonSdkAdditionalData()
val element = Element("root")
sut.save(element)
val sdk = mock(Sdk::class.java).apply {
`when`(homePath).thenReturn("some path")
}
val loadedSut = PythonSdkAdditionalData.load(sdk, element)
val loadedSut = PythonSdkAdditionalData.loadFromElement(element)
Assert.assertEquals("UUID hasn't been loaded", sut.uuid, loadedSut.uuid)
}
}

View File

@@ -199,7 +199,7 @@ class PySdkPathsTest {
val sdkModificator = editableSdk.sdkModificator
assertThat(sdkModificator.sdkAdditionalData).isNull()
mockPythonPluginDisposable()
sdkModificator.sdkAdditionalData = PythonSdkAdditionalData(null).apply {
sdkModificator.sdkAdditionalData = PythonSdkAdditionalData().apply {
setAddedPathsFromVirtualFiles(setOf(userAddedPath))
}
runWriteActionAndWait { sdkModificator.commitChanges() }

View File

@@ -46,5 +46,7 @@
<orderEntry type="library" scope="TEST" name="Velocity" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.jdom" scope="TEST" />
<orderEntry type="module" module-name="intellij.platform.ml" scope="TEST" />
<orderEntry type="library" scope="TEST" name="kotlinx-coroutines-jdk8" level="project" />
<orderEntry type="library" scope="TEST" name="jetbrains.kotlinx.coroutines.test" level="project" />
</component>
</module>