PY-77696, PY-77724 Install UV fixes

1. Rewrite the installation script. Now it returns Path to executable.
2. Save an executable path to Properties.

Merge-request: IJ-MR-150390
Merged-by: Egor Eliseev <Egor.Eliseev@jetbrains.com>

(cherry picked from commit cafb40528ea0e2613248a37291e76e4d24e34674)

GitOrigin-RevId: 911afa3ee4c2e38231cc58f19337f2b7125bea44

platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt

@@ -986,7 +986,7 @@ object CommunityLibraryLicenses {
       .apache("https://github.com/JetBrains/package-search-api-models/blob/master/LICENSE")
       .suppliedByOrganizations("JetBrains Team"),
 
-    LibraryLicense("pip", version = "20.3.4", attachedTo = "intellij.python", url = "https://pip.pypa.io/")
+    LibraryLicense("pip", version = "24.3.1", attachedTo = "intellij.python", url = "https://pip.pypa.io/")
       .mit("https://github.com/pypa/pip/blob/main/LICENSE.txt"),
 
     LibraryLicense("plexus-archiver", libraryName = "plexus-archiver", url = "https://github.com/codehaus-plexus/plexus-archiver")

python/helpers/pycharm_package_installer.py

@@ -13,7 +13,6 @@ It will perform the following steps:
     * Install a script into a platform-specific path:
         - `~/.local/bin` on Unix
         - `%APPDATA%\Python\Scripts` on Windows
-    * Attempt to inform the user if they need to add this bin directory to their `$PATH`, as well as how to do so.
     * Upon failure, write an error log to `package-installer-error-<hash>.log and restore any previous environment.
 
 This script performs minimal magic, and should be relatively stable. However, it is optimized for interactive developer
@@ -40,12 +39,17 @@ from pathlib import Path
 from typing import Optional
 from urllib.request import Request
 from urllib.request import urlopen
+from enum import Enum
 
 SHELL = os.getenv("SHELL", "")
 WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
 MINGW = sysconfig.get_platform().startswith("mingw")
 MACOS = sys.platform == "darwin"
 
+class PipRunCommand(Enum):
+    PIP = ("-m", "pip")
+    WHL = (f"pip-24.3.1-py2.py3-none-any.whl{os.sep}pip", )
+
 def data_dir(package_dir_name) -> Path:
     if WINDOWS:
         base_dir = Path(_get_win_folder("CSIDL_APPDATA"))
@@ -213,9 +217,6 @@ class VirtualEnvironment:
 
         env = cls(target)
 
-        # this ensures that outdated system default pip does not trigger older bugs
-        env.pip("install", "--disable-pip-version-check", "--upgrade", "pip")
-
         return env
 
     @staticmethod
@@ -236,9 +237,15 @@ class VirtualEnvironment:
     def python(self, *args, **kwargs) -> subprocess.CompletedProcess:
         return self.run(self._python, *args, **kwargs)
 
-    def pip(self, *args, **kwargs) -> subprocess.CompletedProcess:
-        return self.python("-m", "pip", *args, **kwargs)
+    def pip(self, pip_command, *args, **kwargs) -> subprocess.CompletedProcess:
+        return self.python(*pip_command, *args, **kwargs)
 
+    def find_pip_command(self):
+        try:
+            self.pip(PipRunCommand.PIP.value, "--version")
+            return PipRunCommand.PIP
+        except:
+            return PipRunCommand.WHL
 
 class Installer:
     def __init__(
@@ -255,6 +262,8 @@ class Installer:
 
     @property
     def bin_dir(self) -> Path:
+        if self._path is not None:
+            return Path(self._path).resolve()
         if not self._bin_dir:
             self._bin_dir = bin_dir()
         return self._bin_dir
@@ -265,8 +274,8 @@ class Installer:
             self._data_dir = data_dir(self._name)
         return self._data_dir
 
-    def run(self) -> int:
-        install_version = self._path if self._path is not None else self._version
+    def run(self):
+        install_version = self._version
 
         self.display_pre_message()
         self.ensure_directories()
@@ -281,6 +290,9 @@ class Installer:
         self._write("")
         self.display_post_message_and_add_to_path(install_version)
 
+        execFile = "{package}.exe".format(package=self._name) if WINDOWS else "{package}".format(package=self._name)
+        self._write(str(self.bin_dir.joinpath(execFile)))
+
         return 0
 
     def install(self, version):
@@ -360,14 +372,12 @@ class Installer:
     def install_package(self, version: str, env: VirtualEnvironment) -> None:
         self._install_comment(version, "Installing {package}".format(package=self._name))
 
-        if self._path:
-            specification = version
-        elif version is not None:
+        if version is not None:
             specification = "{package}=={version}".format(package=self._name, version=version)
         else:
             specification = "{package}".format(package=self._name)
 
-        env.pip("install", specification)
+        env.pip(env.find_pip_command().value, "install", specification)
 
     def display_pre_message(self) -> None:
         kwargs = {
@@ -380,13 +390,16 @@ class Installer:
         if version is None:
             version = "latest"
 
-        if WINDOWS:
-            return self.display_post_message_windows_and_add_to_path(version)
+        try:
+            if WINDOWS:
+                return self.display_post_message_windows_and_add_to_path(version)
 
-        if SHELL == "fish":
-            return self.display_post_message_fish_and_add_to_path(version)
+            if SHELL == "fish":
+                return self.display_post_message_fish_and_add_to_path(version)
 
-        return self.display_post_message_unix(version)
+            return self.display_post_message_unix(version)
+        except:
+            pass
 
     def display_post_message_windows_and_add_to_path(self, version: str) -> None:
         path = self.get_windows_path_var()
@@ -472,17 +485,8 @@ def main():
         description="Installs the latest (or given) version of package"
     )
     parser.add_argument("-n", "--name", required=True)
-    parser.add_argument("--version", help="install named version", dest="version")
-    parser.add_argument(
-        "--path",
-        dest="path",
-        action="store",
-        help=(
-            "Install from a given path (file or directory) instead of "
-            "fetching the latest version of package available online."
-        ),
-    )
-
+    parser.add_argument("-v", "--version", help="install named version", dest="version")
+    parser.add_argument("-p", "--path")
     args = parser.parse_args()
 
     installer = Installer(

python/src/com/jetbrains/python/packaging/PyPackageManagerImplBase.java

@@ -36,7 +36,7 @@ import static com.intellij.webcore.packaging.PackageVersionComparator.VERSION_CO
 
 public abstract class PyPackageManagerImplBase extends PyPackageManager {
   protected static final String SETUPTOOLS_VERSION = "44.1.1";
-  protected static final String PIP_VERSION = "20.3.4";
+  protected static final String PIP_VERSION = "24.3.1";
 
   protected static final String SETUPTOOLS_WHEEL_NAME = "setuptools-" + SETUPTOOLS_VERSION + "-py2.py3-none-any.whl";
   protected static final String PIP_WHEEL_NAME = "pip-" + PIP_VERSION + "-py2.py3-none-any.whl";

python/src/com/jetbrains/python/sdk/PySdkPackageInstallationUtils.kt

@@ -1,25 +1,9 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.jetbrains.python.sdk
 
-import com.intellij.execution.RunCanceledByUserException
 import com.intellij.execution.configurations.GeneralCommandLine
-import com.intellij.execution.process.ProcessOutput
-import com.intellij.openapi.components.Service
-import com.intellij.openapi.components.service
 import com.intellij.openapi.util.SystemInfo
-import com.intellij.openapi.util.io.FileUtil
-import com.jetbrains.python.packaging.PyExecutionException
-import io.ktor.client.HttpClient
-import io.ktor.client.engine.cio.CIO
-import io.ktor.client.plugins.HttpTimeout
-import io.ktor.client.request.get
-import io.ktor.client.statement.bodyAsChannel
-import io.ktor.util.cio.writeChannel
-import io.ktor.utils.io.copyAndClose
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.withContext
 import org.jetbrains.annotations.ApiStatus.Internal
-import java.net.URL
 import java.nio.file.Path
 import kotlin.io.path.absolutePathString
 
@@ -31,95 +15,16 @@ import kotlin.io.path.absolutePathString
 @Internal
 fun getPythonExecutableString() = if (SystemInfo.isWindows) "py" else "python"
 
-
-@Service(Service.Level.APP)
-internal class PackageInstallationFilesService {
-  val ktorClient = HttpClient(CIO) {
-    install(HttpTimeout)
-  }
-  val urlToFilePathMap = mutableMapOf<URL, Path>()
-}
-
-/**
- * Installs a package with Python using the given URL and Python executable.
- *
- * @param [url] The [URL] from which to download the package.
- * @param pythonExecutable The path to the Python executable (could be "py" or "python").
- * @return A [Result] object that represents the [ProcessOutput] of the installation command.
- */
-internal suspend fun installPackageWithPython(url: URL, pythonExecutable: String): Result<String> {
-  val installationFile = downloadFile(url).getOrThrow()
-  val command = GeneralCommandLine(pythonExecutable, installationFile.absolutePathString())
-  return runCommandLine(command)
-}
-
-/**
- * Downloads a file from the specified URL.
- *
- * @param[url] The [URL] from which to download the file.
- * @return A [Result] object that represents the [Path] to the downloaded file.
- */
-internal suspend fun downloadFile(url: URL): Result<Path> {
-  val installationService = service<PackageInstallationFilesService>()
-  installationService.urlToFilePathMap[url]?.let { return Result.success(it) }
-  return withContext(Dispatchers.IO) {
-    val installationFile = FileUtil.createTempFile("_installation_file.py", null)
-    installationService.ktorClient.get(url).bodyAsChannel().copyAndClose(installationFile.writeChannel())
-    installationService.urlToFilePathMap[url] = installationFile.toPath()
-
-    Result.success(installationFile.toPath())
-  }
-}
-
-/**
- * Checks if a package is installed by running the specified command(s) with the "--version"
- * argument and checking the success of the command.
- *
- * @param [commands] The commands to execute. These commands should include the package and any required arguments.
- * @return true if the package is installed, false otherwise
- */
-@Internal
-suspend fun isPackageInstalled(vararg commands: String): Boolean {
-  val command = GeneralCommandLine(*commands, "--version")
-  return runCommandLine(command).isSuccess
-}
-
-/**
- * Installs an executable via pip.
- *
- * @param [executableName] The name of the executable to install.
- * @param [pythonExecutable] The path to the Python executable (could be "py" or "python").
- * @param [isUserSitePackages] Whether to install the executable in the user's site packages directory. Defaults to true.
- */
-@Internal
-suspend fun installExecutableViaPip(
-  executableName: String,
-  pythonExecutable: String,
-  isUserSitePackages: Boolean = true,
-) {
-  val commandList = mutableListOf(pythonExecutable, "-m", "pip", "install", executableName)
-  if (isUserSitePackages) {
-    commandList.add("--user")
-  }
-
-  runCommandLine(GeneralCommandLine(commandList)).getOrThrow()
-}
-
-internal suspend fun installPipIfNeeded(pythonExecutable: String) {
-  if (!isPackageInstalled(pythonExecutable, "-m", "pip") && !isPackageInstalled("pip")) {
-    installPackageWithPython(URL("https://bootstrap.pypa.io/get-pip.py"), pythonExecutable).getOrThrow()
-  }
-}
-
 /**
  * Installs an executable via a Python script.
  *
  * @param [scriptPath] The [Path] to the Python script used for installation.
  * @param [pythonExecutable] The path to the Python executable (could be "py" or "python").
  *
- * @throws [RunCanceledByUserException] if the user cancels the command execution.
- * @throws [PyExecutionException] if the command execution fails.
+ * @return executable [Path]
  */
 @Internal
-suspend fun installExecutableViaPythonScript(scriptPath: Path, pythonExecutable: String, vararg args: String) =
-  runCommandLine(GeneralCommandLine(pythonExecutable, scriptPath.absolutePathString(), *args)).getOrThrow()
+suspend fun installExecutableViaPythonScript(scriptPath: Path, pythonExecutable: String, vararg args: String): Result<Path> {
+  val result = runCommandLine(GeneralCommandLine(pythonExecutable, scriptPath.absolutePathString(), *args)).getOrElse { return Result.failure(it) }
+  return Result.success(Path.of(result.split("\n").last()))
+}

python/src/com/jetbrains/python/sdk/add/v2/CustomNewEnvironmentCreator.kt

@@ -52,7 +52,7 @@ abstract class CustomNewEnvironmentCreator(private val name: String, model: Pyth
   }
 
   override suspend fun getOrCreateSdk(moduleOrProject: ModuleOrProject): Result<Sdk> {
-    savePathToExecutableToProperties()
+    savePathToExecutableToProperties(null)
 
     // todo think about better error handling
     val selectedBasePython = model.state.baseInterpreter.get()!!
@@ -117,10 +117,11 @@ abstract class CustomNewEnvironmentCreator(private val name: String, model: Pyth
   private fun installExecutable() {
     val pythonExecutable = model.state.baseInterpreter.get()?.homePath ?: getPythonExecutableString()
     runWithModalProgressBlocking(ModalTaskOwner.guess(), message("sdk.create.custom.venv.install.fix.title", name, "via pip")) {
-      installPipIfNeeded(pythonExecutable)
-
       if (installationScript != null) {
-        installExecutableViaPythonScript(installationScript, pythonExecutable, "-n", name)
+        val executablePath = installExecutableViaPythonScript(installationScript, pythonExecutable, "-n", name).getOrNull()
+        if (executablePath != null) {
+          savePathToExecutableToProperties(executablePath)
+        }
       }
     }
   }
@@ -136,7 +137,13 @@ abstract class CustomNewEnvironmentCreator(private val name: String, model: Pyth
    */
   private val installationScript: Path? = PythonHelpersLocator.findPathInHelpers("pycharm_package_installer.py")
 
-  internal abstract fun savePathToExecutableToProperties()
+
+  /**
+   * Saves the provided path to an executable in the properties of the environment
+   *
+   * @param [path] The path to the executable that needs to be saved. This may be null when tries to find automatically.
+   */
+  internal abstract fun savePathToExecutableToProperties(path: Path?)
 
   protected abstract suspend fun setupEnvSdk(project: Project?, module: Module?, baseSdks: List<Sdk>, projectPath: String, homePath: String?, installPackages: Boolean): Result<Sdk>
 

python/src/com/jetbrains/python/sdk/add/v2/EnvironmentCreatorPip.kt

@@ -10,14 +10,17 @@ import com.intellij.util.text.nullize
 import com.jetbrains.python.sdk.pipenv.pipEnvPath
 import com.jetbrains.python.sdk.pipenv.setupPipEnvSdkUnderProgress
 import com.jetbrains.python.statistics.InterpreterType
+import java.nio.file.Path
+import kotlin.io.path.pathString
 
 
 class EnvironmentCreatorPip(model: PythonMutableTargetAddInterpreterModel) : CustomNewEnvironmentCreator("pipenv", model) {
   override val interpreterType: InterpreterType = InterpreterType.PIPENV
   override val executable: ObservableMutableProperty<String> = model.state.pipenvExecutable
 
-  override fun savePathToExecutableToProperties() {
-    PropertiesComponent.getInstance().pipEnvPath = executable.get().nullize()
+  override fun savePathToExecutableToProperties(path: Path?) {
+    val savingPath = path?.pathString ?: executable.get().nullize() ?: return
+    PropertiesComponent.getInstance().pipEnvPath = savingPath
   }
 
   override suspend fun setupEnvSdk(project: Project?, module: Module?, baseSdks: List<Sdk>, projectPath: String, homePath: String?, installPackages: Boolean): Result<Sdk> =

python/src/com/jetbrains/python/sdk/add/v2/EnvironmentCreatorPoetry.kt

@@ -27,11 +27,11 @@ import com.jetbrains.python.sdk.poetry.poetryToml
 import com.jetbrains.python.sdk.poetry.pyProjectToml
 import com.jetbrains.python.sdk.poetry.setupPoetrySdkUnderProgress
 import com.jetbrains.python.statistics.InterpreterType
-import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.withContext
 import java.nio.file.Path
+import kotlin.io.path.pathString
 
 class EnvironmentCreatorPoetry(model: PythonMutableTargetAddInterpreterModel, private val moduleOrProject: ModuleOrProject?) : CustomNewEnvironmentCreator("poetry", model) {
   override val interpreterType: InterpreterType = InterpreterType.POETRY
@@ -61,8 +61,9 @@ class EnvironmentCreatorPoetry(model: PythonMutableTargetAddInterpreterModel, pr
     basePythonComboBox.setItems(validatedInterpreters)
   }
 
-  override fun savePathToExecutableToProperties() {
-    PropertiesComponent.getInstance().poetryPath = executable.get().nullize()
+  override fun savePathToExecutableToProperties(path: Path?) {
+    val savingPath = path?.pathString ?: executable.get().nullize() ?: return
+    PropertiesComponent.getInstance().poetryPath = savingPath
   }
 
   override suspend fun setupEnvSdk(project: Project?, module: Module?, baseSdks: List<Sdk>, projectPath: String, homePath: String?, installPackages: Boolean): Result<Sdk> {

python/src/com/jetbrains/python/sdk/add/v2/EnvironmentCreatorUv.kt

@@ -5,6 +5,7 @@ import com.intellij.openapi.module.Module
 import com.intellij.openapi.observable.properties.ObservableMutableProperty
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.projectRoots.Sdk
+import com.intellij.util.text.nullize
 import com.jetbrains.python.sdk.ModuleOrProject
 import com.jetbrains.python.sdk.uv.impl.setUvExecutable
 import com.jetbrains.python.sdk.uv.setupUvSdkUnderProgress
@@ -20,8 +21,9 @@ class EnvironmentCreatorUv(model: PythonMutableTargetAddInterpreterModel, privat
     basePythonComboBox.setItems(model.baseInterpreters)
   }
 
-  override fun savePathToExecutableToProperties() {
-    setUvExecutable(Path.of(executable.get()))
+  override fun savePathToExecutableToProperties(path: Path?) {
+    val savingPath = path ?: executable.get().nullize()?.let { Path.of(it) } ?: return
+    setUvExecutable(savingPath)
   }
 
   override suspend fun setupEnvSdk(project: Project?, module: Module?, baseSdks: List<Sdk>, projectPath: String, homePath: String?, installPackages: Boolean): Result<Sdk> {