Add Conda (Miniconda/Anaconda) install manager (PY-63084)

Make project optional (PY-63084) Fix BinaryInstallerUsagesCollector(PY-63084) * align eventIds according to naming convention * fix version regex (allow only digits dots and dashes) Add Conda (Miniconda/Anaconda) install manager (PY-63084) * refactor python installers * add conda updater * create conda sdks registry Merge-request: IJ-MR-128404 Merged-by: Vitaly Legchilkin <Vitaly.Legchilkin@jetbrains.com> GitOrigin-RevId: 1e73d1bd32fced94901c4c9a2c1260fca8aca9e2
2026-04-18 20:41:22 +07:00 · 2024-03-19 00:50:15 +00:00
parent d7c00ac3da
commit a3a2a5db18
17 changed files with 9390 additions and 353 deletions
--- a/python/python-sdk/resources/messages/PySdkBundle.properties
+++ b/python/python-sdk/resources/messages/PySdkBundle.properties
@@ -36,7 +36,7 @@ python.sdk.pipenv.creating.venv.not.supported=Creating a virtual environments ba
 python.sdk.invalid.remote.sdk=Invalid remote SDK
 python.sdk.package.managing.not.supported.for.sdk=Package managing is not supported for SDK {0}
 python.sdk.preparation.progress.text=Preparing for installation\u2026
-python.sdk.downloading.progress.details=Downloading {0}
+python.sdk.downloading.progress.details=Downloading ({0}) {1} 
 python.sdk.running.progress.text=Running {0}\u2026
 python.sdk.running.one.minute.progress.details=About 1 minute left
 python.sdk.running.sudo.prompt=Enter your password to install {0}
--- a/python/python-sdk/resources/sdks.json
+++ b/python/python-sdk/resources/sdks.json
--- a/python/python-sdk/src/com/jetbrains/python/sdk/Sdks.kt
+++ b/python/python-sdk/src/com/jetbrains/python/sdk/Sdks.kt
@@ -29,7 +29,9 @@ private val LOG: Logger = logger<Sdks>()
 */
 enum class Product(val title: String) {
  CPython("Python"),
-  PyPy("PyPy");
+  PyPy("PyPy"),
+  Miniconda("Miniconda"),
+  Anaconda("Anaconda");
 }

 /**
@@ -39,6 +41,7 @@ enum class ResourceType(vararg val extensions: String) {
  MICROSOFT_WINDOWS_EXECUTABLE("exe"),
  MICROSOFT_SOFTWARE_INSTALLER("msi"),
  APPLE_SOFTWARE_PACKAGE("pkg"),
+  SHELL_SCRIPT("sh"),
  COMPRESSED("zip", "xz", "tgz", "bz2");

  companion object {
@@ -70,6 +73,7 @@ data class Binary(
  val os: OS,
  val cpuArch: CpuArch?,
  val resources: List<Resource>,
+  val tags: List<String>? = null,
 ) {
  fun isCompatible(os: OS = OS.CURRENT, cpuArch: CpuArch = CpuArch.CURRENT) = this.os == os && (this.cpuArch?.equals(cpuArch) ?: true)
 }
@@ -80,7 +84,7 @@ data class Binary(
 * Vendor + Version is a primary key.
 */
 data class Release(
-  val version: Version,
+  val version: String,
  val product: Product,
  val sources: List<Resource>?,
  val binaries: List<Binary>?,
@@ -99,30 +103,13 @@ data class Release(
 */
 data class Sdks(
  val python: List<Release> = listOf(),
+  val conda: List<Release> = listOf(),
 )


-fun Version.toLanguageLevel(): LanguageLevel? = LanguageLevel.fromPythonVersion("$major.$minor")
+fun Version?.toLanguageLevel(): LanguageLevel? = this?.let { LanguageLevel.fromPythonVersion("$major.$minor") }


-/**
- * This class replaces missed String-arg constructor in Version class for jackson deserialization.
- *
- * @see com.intellij.openapi.util.Version
- */
-class VersionDeserializer : JsonDeserializer<Version>() {
-  override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): Version {
-    return Version.parseVersion(p!!.valueAsString)!!
-  }
-}
-class VersionSerializer : JsonSerializer<Version>() {
-  override fun serialize(value: Version?, gen: JsonGenerator?, serializers: SerializerProvider?) {
-    value?.let {
-      gen?.writeString(it.toString())
-    }
-  }
-}
-
 /**
 * This class replaces missed String-arg constructor in Url class for jackson deserialization.
 *
@@ -150,7 +137,13 @@ object SdksKeeper {
  }

  fun pythonReleasesByLanguageLevel(): Map<LanguageLevel, List<Release>> {
-    return sdks.python.filter { it.version.toLanguageLevel() != null }.groupBy { it.version.toLanguageLevel()!! }
+    return sdks.python.mapNotNull { release ->
+      Version.parseVersion(release.version)?.toLanguageLevel()?.let { it to release }
+    }.groupBy({ it.first }, { it.second })
+  }
+
+  fun condaReleases(vararg products: Product = arrayOf(Product.Miniconda, Product.Anaconda)): List<Release> {
+    return sdks.conda.filter { it.product in products }
  }


@@ -158,7 +151,6 @@ object SdksKeeper {
    jacksonObjectMapper()
      .registerModule(
        SimpleModule()
-          .addDeserializer(Version::class.java, VersionDeserializer())
          .addDeserializer(Url::class.java, UrlDeserializer())
      )
      .readValue(content, Sdks::class.java)
@@ -167,11 +159,11 @@ object SdksKeeper {
    LOG.error("Json syntax error in the $configUrl", ex)
    Sdks()
  }
+
  fun serialize(sdks: Sdks): String {
    return jacksonObjectMapper()
      .registerModule(
        SimpleModule()
-          .addSerializer(Version::class.java, VersionSerializer())
          .addSerializer(Url::class.java, UrlSerializer())
      )
      .setSerializationInclusion(JsonInclude.Include.NON_NULL)
--- a/python/python-sdk/src/com/jetbrains/python/sdk/installer/ReleaseInstaller.kt
+++ b/python/python-sdk/src/com/jetbrains/python/sdk/installer/ReleaseInstaller.kt
@@ -1,137 +0,0 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.sdk.installer
-
-import com.google.common.hash.Hashing
-import com.google.common.io.Files
-import com.intellij.openapi.application.PathManager
-import com.intellij.openapi.diagnostic.logger
-import com.intellij.openapi.progress.ProcessCanceledException
-import com.intellij.openapi.progress.ProgressIndicator
-import com.intellij.openapi.util.io.FileUtil
-import com.intellij.util.io.HttpRequests
-import com.jetbrains.python.PySdkBundle
-import com.jetbrains.python.sdk.*
-import java.nio.file.Path
-import java.nio.file.Paths
-import kotlin.io.path.fileSize
-
-val LOGGER = logger<ReleaseInstaller>()
-
-data class ReleasePreview(val description: String, val size: Long) {
-  companion object {
-    fun of(resource: Resource?) = resource?.let { ReleasePreview(it.url.toString(), it.size) } ?: ReleasePreview("", 0)
-  }
-}
-
-/**
- * Software Release Installer.
- *
- * @see com.jetbrains.python.sdk.Release
- */
-interface ReleaseInstaller {
-  /**
-   * Verifies if it can install the release (have enough binary/source resources or tools installed).
-   */
-  fun canInstall(release: Release): Boolean
-
-  /**
-   * Installation process of the selected software release.
-   * Should be checked with canInstall, otherwise the behavior is not predictable.
-   * @see ReleaseInstaller.canInstall
-   * @param onPrepareComplete callback function, have to be called right before the installation process.
-   */
-  @Throws(ReleaseInstallerException::class)
-  fun install(release: Release, indicator: ProgressIndicator, onPrepareComplete: () -> Unit)
-
-  /**
-   * Preview for pre-install messages and warnings.
-   */
-  fun getPreview(release: Release): ReleasePreview
-}
-
-/**
- * Base Release Installer with additional external resource loading.
- * Responsible for loading and checking the checksum/size of additional resources before processing.
- */
-abstract class DownloadableReleaseInstaller : ReleaseInstaller {
-  /**
-   * Concrete installer should choose which resources it needs.
-   * Might be any additional resources or tools outside the release scope.
-   */
-  abstract fun getResourcesToDownload(release: Release): List<Resource>
-
-  /**
-   * Concrete installer process entry point.
-   * There is a guarantee that at this moment all requested resources were downloaded
-   * and their paths are stored in the resourcePaths.
-   *
-   * @see getResourcesToDownload
-   * @param resourcePaths - requested resources download paths (on the temp path)
-   */
-  @Throws(ProcessException::class)
-  abstract fun process(release: Release, resourcePaths: Map<Resource, Path>, indicator: ProgressIndicator)
-
-  @Throws(ReleaseInstallerException::class)
-  override fun install(release: Release, indicator: ProgressIndicator, onPrepareComplete: () -> Unit) {
-    val tempPath = PathManager.getTempPath()
-    val files = getResourcesToDownload(release).associateWith {
-      Paths.get(tempPath, "${System.nanoTime()}-${it.fileName}")
-    }
-    try {
-      indicator.text = PySdkBundle.message("python.sdk.preparation.progress.text")
-      files.forEach { (resource, path) ->
-        download(resource, path, indicator)
-      }
-
-      onPrepareComplete()
-
-      process(release, files, indicator)
-    }
-    finally {
-      files.values.forEach { runCatching { FileUtil.delete(it) } }
-    }
-  }
-
-  @Throws(PrepareException::class)
-  private fun checkConsistency(resource: Resource, target: Path) {
-    LOGGER.debug("Checking installer size")
-    val sizeDiff = target.fileSize() - resource.size
-    if (sizeDiff != 0L) {
-      throw WrongSizePrepareException(target, sizeDiff)
-    }
-
-    LOGGER.debug("Checking installer checksum")
-    val actualHashCode = Files.asByteSource(target.toFile()).hash(Hashing.sha256()).toString()
-    if (!resource.sha256.equals(actualHashCode, ignoreCase = true)) {
-      throw WrongChecksumPrepareException(target, resource.sha256, actualHashCode)
-    }
-  }
-
-  @Throws(PrepareException::class)
-  private fun download(resource: Resource, target: Path, indicator: ProgressIndicator) {
-    LOGGER.info("Downloading ${resource.url} to $target")
-    indicator.text2 = PySdkBundle.message("python.sdk.downloading.progress.details", resource.fileName)
-    try {
-      indicator.checkCanceled()
-      HttpRequests.request(resource.url)
-        .productNameAsUserAgent()
-        .saveToFile(target.toFile(), indicator)
-
-      indicator.checkCanceled()
-      checkConsistency(resource, target)
-    }
-    catch (e: ProcessCanceledException) {
-      throw CancelledPrepareException(e)
-    }
-    catch (e: PrepareException) {
-      throw e
-    }
-    catch (e: Exception) {
-      throw PrepareException(e)
-    }
-  }
-
-  override fun getPreview(release: Release): ReleasePreview {
-    return ReleasePreview.of(getResourcesToDownload(release).firstOrNull())
-  }
-}
--- a/python/python-sdk/src/com/jetbrains/python/sdk/installer/ReleaseInstallerException.kt
+++ b/python/python-sdk/src/com/jetbrains/python/sdk/installer/ReleaseInstallerException.kt
@@ -1,44 +0,0 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.sdk.installer
-
-import com.intellij.execution.configurations.GeneralCommandLine
-import com.intellij.execution.process.ProcessOutput
-import com.intellij.openapi.progress.ProcessCanceledException
-import java.io.IOException
-import java.nio.file.Path
-
-/**
- * Base Exception for the Software Release Installer.
- */
-open class ReleaseInstallerException(cause: Throwable) : Exception(cause)
-
-/**
- * Base Exception for the release prepare logic (the stage before onPrepareComplete callback).
- */
-open class PrepareException(cause: Throwable) : ReleaseInstallerException(cause)
-
-class WrongSizePrepareException(path: Path, sizeDiff: Long) : PrepareException(
-  IOException("Downloaded $path has incorrect size, difference is $sizeDiff bytes.")
-)
-
-class WrongChecksumPrepareException(path: Path, required: String, actual: String) : PrepareException(
-  IOException("Checksums for $path does not match. Actual value is $actual, expected $required.")
-)
-
-class CancelledPrepareException(cause: ProcessCanceledException) : PrepareException(cause)
-
-/**
- * Base Exception for the release processing logic (the stage after onPrepareComplete callback).
- */
-open class ProcessException(val command: GeneralCommandLine, val output: ProcessOutput?, cause: Throwable) : ReleaseInstallerException(
-  cause)
-
-class ExecutionProcessException(command: GeneralCommandLine, cause: Exception) : ProcessException(command, null, cause)
-class NonZeroExitCodeProcessException(command: GeneralCommandLine, output: ProcessOutput)
-  : ProcessException(command, output, IOException("Exit code is non-zero: ${output.exitCode}"))
-
-class TimeoutProcessException(command: GeneralCommandLine, output: ProcessOutput)
-  : ProcessException(command, output, IOException("Runtime timeout reached"))
-
-class CancelledProcessException(command: GeneralCommandLine, output: ProcessOutput)
-  : ProcessException(command, output, IOException("Installation was canceled"))
--- a/python/python-sdk/src/com/jetbrains/python/sdk/installer/ResourceTypeReleaseInstaller.kt
+++ b/python/python-sdk/src/com/jetbrains/python/sdk/installer/ResourceTypeReleaseInstaller.kt
@@ -1,96 +0,0 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.jetbrains.python.sdk.installer
-
-import com.intellij.execution.configurations.GeneralCommandLine
-import com.intellij.execution.process.CapturingProcessHandler
-import com.intellij.execution.process.ProcessOutput
-import com.intellij.execution.util.ExecUtil
-import com.intellij.openapi.progress.ProgressIndicator
-import com.jetbrains.python.PySdkBundle
-import com.jetbrains.python.sdk.Release
-import com.jetbrains.python.sdk.Resource
-import com.jetbrains.python.sdk.ResourceType
-import java.nio.file.Path
-import kotlin.io.path.absolutePathString
-
-/**
- * Software Release Installer for Apple Software Package (pkg) files
- */
-class PkgReleaseInstaller : ResourceTypeReleaseInstaller(ResourceType.APPLE_SOFTWARE_PACKAGE) {
-  override fun buildCommandLine(resource: Resource, path: Path): GeneralCommandLine {
-    return ExecUtil.sudoCommand(
-      GeneralCommandLine("installer", "-pkg", path.absolutePathString(), "-target", "/"),
-      PySdkBundle.message("python.sdk.running.sudo.prompt", resource.fileName)
-    )
-  }
-}
-
-/**
- * Software Release Installer for Microsoft Window Executable (exe) files
- */
-class ExeReleaseInstaller : ResourceTypeReleaseInstaller(ResourceType.MICROSOFT_WINDOWS_EXECUTABLE) {
-  override fun buildCommandLine(resource: Resource, path: Path): GeneralCommandLine {
-    return GeneralCommandLine(path.absolutePathString(), "/repair", "/quiet", "InstallAllUsers=0")
-  }
-}
-
-/**
- * Base Release Installer with resource type specific filtering (like exe, pkg, ...)
- */
-abstract class ResourceTypeReleaseInstaller(private val resourceType: ResourceType) : DownloadableReleaseInstaller() {
-  abstract fun buildCommandLine(resource: Resource, path: Path): GeneralCommandLine
-
-  @Throws(ProcessException::class)
-  override fun process(release: Release, resourcePaths: Map<Resource, Path>, indicator: ProgressIndicator) {
-    resourcePaths.forEach { (resource, path) ->
-      processResource(resource, path, indicator)
-    }
-  }
-
-  @Throws(ProcessException::class)
-  private fun processResource(resource: Resource, path: Path, indicator: ProgressIndicator) {
-    indicator.isIndeterminate = true
-    indicator.text = PySdkBundle.message("python.sdk.running.progress.text", resource.fileName)
-    indicator.text2 = PySdkBundle.message("python.sdk.running.one.minute.progress.details")
-
-    val commandLine = buildCommandLine(resource, path)
-    LOGGER.info("Running ${commandLine.commandLineString}")
-    val processOutput: ProcessOutput
-    try {
-      processOutput = CapturingProcessHandler(commandLine).runProcessWithProgressIndicator(indicator)
-    }
-    catch (e: Exception) {
-      throw ExecutionProcessException(commandLine, e)
-    }
-    processOutput.isCancelled.takeIf { it }?.let { throw CancelledProcessException(commandLine, processOutput) }
-    processOutput.exitCode.takeIf { it != 0 }?.let {
-      if (processOutput.stderr.contains("User cancelled", ignoreCase = true)) {
-        throw CancelledProcessException(commandLine, processOutput)
-      }
-      throw NonZeroExitCodeProcessException(commandLine, processOutput)
-    }
-    processOutput.isTimeout.takeIf { it }?.let { throw TimeoutProcessException(commandLine, processOutput) }
-  }
-
-  /**
-   * It can install a release only if the release contains at least single resource with corresponding resource type in the binaries.
-   */
-  override fun canInstall(release: Release): Boolean {
-    return release.binaries?.any {
-      it.isCompatible() &&
-      it.resources.any { r -> r.type == resourceType }
-    } ?: false
-  }
-
-  /**
-   * @returns first non-empty list with resources of the selected type in any compatible binary package or empty list if nothing is found.
-   */
-  override fun getResourcesToDownload(release: Release): List<Resource> {
-    return release.binaries
-             ?.filter { it.isCompatible() }
-             ?.firstNotNullOfOrNull {
-               it.resources.filter { r -> r.type == resourceType }.ifEmpty { null }
-             }
-           ?: listOf()
-  }
-}