From 1a7a0eefd98405a4c5704693d7054f6851ab51db Mon Sep 17 00:00:00 2001 From: "Nikita.Ashihmin" Date: Mon, 11 Aug 2025 03:11:35 +0400 Subject: [PATCH] PY-78749 Packages: Remove PyTargetEnvironmentPackageManager GitOrigin-RevId: 1a3eb25f4ef3c897fb3abba61be00acf366dde66 --- ...equirementsTxtOrSetupPySdkConfiguration.kt | 31 +- .../packaging/PyPackageManagerBridge.kt | 33 +- ...r.java => PyTargetEnvCreationManager.java} | 476 +++++------------- .../PyProjectVirtualEnvConfiguration.kt | 11 +- .../jetbrains/env/python/PyPackagingTest.java | 42 +- 5 files changed, 169 insertions(+), 424 deletions(-) rename python/src/com/jetbrains/python/packaging/{PyTargetEnvironmentPackageManager.java => PyTargetEnvCreationManager.java} (65%) diff --git a/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/configuration/PyRequirementsTxtOrSetupPySdkConfiguration.kt b/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/configuration/PyRequirementsTxtOrSetupPySdkConfiguration.kt index 44afd638afa2..3e7c8fb42738 100644 --- a/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/configuration/PyRequirementsTxtOrSetupPySdkConfiguration.kt +++ b/python/ide/impl/src/com/intellij/pycharm/community/ide/impl/configuration/PyRequirementsTxtOrSetupPySdkConfiguration.kt @@ -18,7 +18,6 @@ import com.intellij.openapi.util.NlsSafe import com.intellij.openapi.util.use import com.intellij.openapi.vfs.VfsUtil import com.intellij.openapi.vfs.VirtualFile -import com.intellij.platform.util.progress.reportRawProgress import com.intellij.pycharm.community.ide.impl.PyCharmCommunityCustomizationBundle import com.intellij.pycharm.community.ide.impl.configuration.PySdkConfigurationCollector.InputData import com.intellij.pycharm.community.ide.impl.configuration.PySdkConfigurationCollector.Source @@ -27,10 +26,8 @@ import com.intellij.pycharm.community.ide.impl.configuration.ui.PyAddNewVirtualE import com.intellij.ui.IdeBorderFactory import com.intellij.ui.components.JBLabel import com.intellij.util.ui.JBUI -import com.jetbrains.python.PyBundle import com.jetbrains.python.PySdkBundle import com.jetbrains.python.errorProcessing.PyResult -import com.jetbrains.python.packaging.PyPackageManager import com.jetbrains.python.packaging.PyPackageUtil import com.jetbrains.python.packaging.management.PythonPackageManager import com.jetbrains.python.packaging.requirementsTxt.PythonRequirementTxtSdkUtils @@ -40,7 +37,6 @@ import com.jetbrains.python.sdk.PythonSdkUtil import com.jetbrains.python.sdk.basePath import com.jetbrains.python.sdk.configuration.PyProjectSdkConfigurationExtension import com.jetbrains.python.sdk.configuration.createVirtualEnvAndSdkSynchronously -import com.jetbrains.python.sdk.isTargetBased import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.awt.BorderLayout @@ -94,20 +90,7 @@ class PyRequirementsTxtOrSetupPySdkConfiguration : PyProjectSdkConfigurationExte PythonRequirementTxtSdkUtils.saveRequirementsTxtPath(module.project, sdk, requirementsTxtOrSetupPyFile.toNioPath()) } - if (!sdk.isTargetBased()) { - val pythonPackageManager = PythonPackageManager.forSdk(module.project, sdk) - pythonPackageManager.sync().getOr { - PySdkConfigurationCollector.logVirtualEnv(module.project, VirtualEnvResult.INSTALLATION_FAILURE) - return it - } - } - else { - withContext(Dispatchers.Default) { - createTargetBased(sdk, requirementsTxtOrSetupPyFile) - } - } - - return PyResult.success(sdk) + return PythonPackageManager.forSdk(module.project, sdk).sync().mapSuccess { sdk } } catch (e: ExecutionException) { PySdkConfigurationCollector.logVirtualEnv(module.project, VirtualEnvResult.INSTALLATION_FAILURE) @@ -116,18 +99,6 @@ class PyRequirementsTxtOrSetupPySdkConfiguration : PyProjectSdkConfigurationExte } } - private suspend fun createTargetBased( - sdk: Sdk, - requirementsTxtOrSetupPyFile: VirtualFile, - ) { - reportRawProgress { - it.text(PyBundle.message("python.packaging.installing.packages")) - val packageManager = PyPackageManager.getInstance(sdk) - val command = getCommandForPipInstall(requirementsTxtOrSetupPyFile) - packageManager.install(emptyList(), command) - } - } - private fun getRequirementsTxtOrSetupPy(module: Module) = PyPackageUtil.findRequirementsTxt(module) ?: PyPackageUtil.findSetupPy(module)?.virtualFile diff --git a/python/src/com/jetbrains/python/packaging/PyPackageManagerBridge.kt b/python/src/com/jetbrains/python/packaging/PyPackageManagerBridge.kt index 5f1935a32d93..9636c63cb082 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackageManagerBridge.kt +++ b/python/src/com/jetbrains/python/packaging/PyPackageManagerBridge.kt @@ -2,26 +2,49 @@ package com.jetbrains.python.packaging import com.intellij.execution.ExecutionException +import com.intellij.openapi.application.writeAction import com.intellij.openapi.diagnostic.thisLogger import com.intellij.openapi.progress.runBlockingMaybeCancellable import com.intellij.openapi.project.ProjectManager import com.intellij.openapi.project.getOpenedProjects import com.intellij.openapi.projectRoots.Sdk +import com.intellij.openapi.roots.OrderRootType +import com.intellij.openapi.vfs.VfsUtil +import com.intellij.util.cancelOnDispose import com.jetbrains.python.getOrNull import com.jetbrains.python.onFailure import com.jetbrains.python.packaging.management.PythonPackageManager import com.jetbrains.python.packaging.management.ui.PythonPackageManagerUI import com.jetbrains.python.packaging.management.ui.installPyRequirementsBackground +import com.jetbrains.python.packaging.utils.PyPackageCoroutine +import com.jetbrains.python.sdk.PythonSdkType import org.jetbrains.annotations.ApiStatus @ApiStatus.Internal -internal open class PyPackageManagerBridge(sdk: Sdk) : PyTargetEnvironmentPackageManager(sdk) { +internal open class PyPackageManagerBridge(private val sdk: Sdk) : PyPackageManager(sdk) { protected val packageManager: PythonPackageManager = PythonPackageManager.forSdk(guessProject(), sdk = sdk) protected val packageManagerUI: PythonPackageManagerUI = PythonPackageManagerUI.forPackageManager(packageManager) override fun installManagement() {} override fun hasManagement(): Boolean = true + override fun refresh() { + PyPackageCoroutine.launch(null) { + thisLogger().debug("Refreshing SDK roots and packages cache") + writeAction { + val files = sdk.getRootProvider().getFiles(OrderRootType.CLASSES) + VfsUtil.markDirtyAndRefresh(true, true, true, *files) + } + PythonSdkType.getInstance().setupSdkPaths(sdk) + }.cancelOnDispose(this) + } + + override fun createVirtualEnv(destinationDir: String, useGlobalSite: Boolean): String { + val manager = PyTargetEnvCreationManager(getSdk()) + return manager.createVirtualEnv(destinationDir, useGlobalSite) + + } + @Throws(ExecutionException::class) override fun install(requirements: MutableList?, extraArgs: MutableList) { runBlockingMaybeCancellable { @@ -48,14 +71,6 @@ internal open class PyPackageManagerBridge(sdk: Sdk) : PyTargetEnvironmentPackag } - @Throws(ExecutionException::class) - override fun collectPackages(): List { - return runBlockingMaybeCancellable { - packageManagerUI.manager.listInstalledPackages() - }.map { PyPackage(it.name, it.version) } - } - - override fun getPackages(): List { return packageManagerUI.manager.listInstalledPackagesSnapshot().map { PyPackage(it.name, it.version) } } diff --git a/python/src/com/jetbrains/python/packaging/PyTargetEnvironmentPackageManager.java b/python/src/com/jetbrains/python/packaging/PyTargetEnvCreationManager.java similarity index 65% rename from python/src/com/jetbrains/python/packaging/PyTargetEnvironmentPackageManager.java rename to python/src/com/jetbrains/python/packaging/PyTargetEnvCreationManager.java index c7325e4c7259..1c4d03140e78 100644 --- a/python/src/com/jetbrains/python/packaging/PyTargetEnvironmentPackageManager.java +++ b/python/src/com/jetbrains/python/packaging/PyTargetEnvCreationManager.java @@ -5,7 +5,6 @@ import com.intellij.execution.ExecutionException; import com.intellij.execution.RunCanceledByUserException; import com.intellij.execution.configurations.GeneralCommandLine; import com.intellij.execution.process.CapturingProcessHandler; -import com.intellij.execution.process.ProcessNotCreatedException; import com.intellij.execution.process.ProcessOutput; import com.intellij.execution.target.TargetEnvironment; import com.intellij.execution.target.TargetEnvironmentRequest; @@ -14,28 +13,20 @@ import com.intellij.execution.target.TargetedCommandLine; import com.intellij.execution.target.local.LocalTargetEnvironment; import com.intellij.execution.target.value.TargetEnvironmentFunctions; import com.intellij.execution.util.ExecUtil; -import com.intellij.openapi.application.Application; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.progress.EmptyProgressIndicator; import com.intellij.openapi.progress.ProgressIndicator; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.project.ProjectManager; import com.intellij.openapi.projectRoots.Sdk; -import com.intellij.openapi.roots.OrderRootType; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.openapi.vfs.VfsUtil; -import com.intellij.openapi.vfs.VirtualFile; import com.intellij.python.community.helpersLocator.PythonHelpersLocator; import com.intellij.util.concurrency.annotations.RequiresReadLock; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.net.HttpConfigurable; import com.jetbrains.python.HelperPackage; -import com.jetbrains.python.PyPsiPackageUtil; import com.jetbrains.python.PySdkBundle; import com.jetbrains.python.PythonHelper; -import com.jetbrains.python.errorProcessing.ExecErrorImpl; -import com.jetbrains.python.errorProcessing.ExecErrorReason; import com.jetbrains.python.packaging.common.PythonPackage; import com.jetbrains.python.packaging.pip.PipParseUtils; import com.jetbrains.python.packaging.repository.PyPackageRepositoryUtil; @@ -63,16 +54,15 @@ import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; -import static com.intellij.webcore.packaging.PackageVersionComparator.VERSION_COMPARATOR; - /** * @deprecated TODO: explain */ @SuppressWarnings("ALL") @Deprecated(forRemoval = true) @ApiStatus.Internal -public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager { - private static final Logger LOG = Logger.getInstance(PyTargetEnvironmentPackageManager.class); +public class PyTargetEnvCreationManager { + private static final Logger LOG = Logger.getInstance(PyTargetEnvCreationManager.class); + private final @NotNull Sdk mySdk; protected static final String SETUPTOOLS_VERSION = "44.1.1"; protected static final String PIP_VERSION = "24.3.1"; @@ -92,119 +82,169 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager protected String mySeparator = File.separator; protected volatile @Nullable List myPackagesCache = null; - @Override - public void refresh() { - LOG.debug("Refreshing SDK roots and packages cache"); - final Application application = ApplicationManager.getApplication(); - application.invokeLater(() -> { - final Sdk sdk = getSdk(); - application.runWriteAction(() -> { - final VirtualFile[] files = sdk.getRootProvider().getFiles(OrderRootType.CLASSES); - VfsUtil.markDirtyAndRefresh(true, true, true, files); - }); - PythonSdkType.getInstance().setupSdkPaths(sdk); - }); + public PyTargetEnvCreationManager(final @NotNull Sdk sdk) { + mySdk = sdk; } - @Override - public void installManagement() throws ExecutionException { - final LanguageLevel languageLevel = PythonSdkType.getLanguageLevelForSdk(getSdk()); + public @NotNull String createVirtualEnv(@NotNull String destinationDir, boolean useGlobalSite) throws ExecutionException { + final Sdk sdk = getSdk(); + final LanguageLevel languageLevel = getOrRequestLanguageLevelForSdk(sdk); + if (languageLevel.isOlderThan(LanguageLevel.PYTHON27)) { - throw new ExecutionException(PySdkBundle.message("python.sdk.packaging.package.management.for.python.not.supported", + throw new ExecutionException(PySdkBundle.message("python.sdk.packaging.creating.virtual.environment.for.python.not.supported", languageLevel, LanguageLevel.PYTHON27)); } - boolean success = updatePackagingTools(); - if (success) { - return; - } + HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter(); + TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareTargetRequest.getTargetEnvironmentRequest(); - if (languageLevel.isOlderThan(LanguageLevel.PYTHON312)) { // Python 3.12 doesn't require setuptools to list packages anymore - final PyPackage installedSetuptools = refreshAndCheckForSetuptools(); - if (installedSetuptools == null || VERSION_COMPARATOR.compare(installedSetuptools.getVersion(), SETUPTOOLS_VERSION) < 0) { - installManagement(Objects.requireNonNull(getHelperPath(SETUPTOOLS_WHEEL_NAME))); - } + PythonScriptExecution pythonExecution = PythonScripts.prepareHelperScriptExecution( + isLegacyPython(languageLevel) ? PythonHelper.LEGACY_VIRTUALENV_ZIPAPP : PythonHelper.VIRTUALENV_ZIPAPP, + helpersAwareTargetRequest); + if (useGlobalSite) { + pythonExecution.addParameter("--system-site-packages"); } + pythonExecution.addParameter(destinationDir); + // TODO [targets] Pass `parentDir = null` + getPythonProcessResult(pythonExecution, false, true, targetEnvironmentRequest); - final PyPackage installedPip = PyPsiPackageUtil.findPackage(refreshAndGetPackages(false), PyPackageUtil.PIP); - if (installedPip == null || VERSION_COMPARATOR.compare(installedPip.getVersion(), PIP_VERSION) < 0) { - installManagement(Objects.requireNonNull(getHelperPath(PIP_WHEEL_NAME))); - } + final Path binary = VirtualEnvReader.getInstance().findPythonInPythonRoot(Path.of(destinationDir)); + final char separator = targetEnvironmentRequest.getTargetPlatform().getPlatform().fileSeparator; + final String binaryFallback = destinationDir + separator + "bin" + separator + "python"; + + return (binary != null) ? binary.toString() : binaryFallback; } - protected final boolean updatePackagingTools() { - try { - installUsingPipWheel("--upgrade", "--force-reinstall", PyPackageUtil.SETUPTOOLS, PyPackageUtil.PIP); - return true; - } - catch (ExecutionException e) { - LOG.info(e); - return false; - } - finally { - refreshPackagesSynchronously(); + private @NotNull HelpersAwareTargetEnvironmentRequest getPythonTargetInterpreter() throws ExecutionException { + HelpersAwareTargetEnvironmentRequest request = PythonInterpreterTargetEnvironmentFactory.findPythonTargetInterpreter(getSdk(), + ProjectManager.getInstance() + .getDefaultProject()); + if (request == null) { + throw new ExecutionException(PySdkBundle.message("python.sdk.package.managing.not.supported.for.sdk", getSdk().getName())); } + return request; } - @Override - public boolean hasManagement() throws ExecutionException { - final LanguageLevel languageLevel = PythonSdkType.getLanguageLevelForSdk(getSdk()); - final Boolean hasSetuptools = languageLevel.isAtLeast(LanguageLevel.PYTHON312) || refreshAndCheckForSetuptools() != null; - final Boolean hasPip = PyPsiPackageUtil.findPackage(refreshAndGetPackages(false), PyPackageUtil.PIP) != null; - return hasSetuptools && hasPip; + /** + * Is it a legacy python version that we still support + */ + private static @NotNull Boolean isLegacyPython(@NotNull LanguageLevel languageLevel) { + return languageLevel.isPython2() || languageLevel.isOlderThan(LanguageLevel.PYTHON37); } - protected final @Nullable PyPackage refreshAndCheckForSetuptools() throws ExecutionException { - try { - final List packages = refreshAndGetPackages(false); - final PyPackage setuptoolsPackage = PyPsiPackageUtil.findPackage(packages, PyPackageUtil.SETUPTOOLS); - return setuptoolsPackage != null ? setuptoolsPackage : PyPsiPackageUtil.findPackage(packages, PyPackageUtil.DISTRIBUTE); + private @NotNull String getPythonProcessResult(@NotNull PythonExecution pythonExecution, + boolean askForSudo, + boolean showProgress, + @NotNull TargetEnvironmentRequest targetEnvironmentRequest) throws ExecutionException { + ProcessOutputWithCommandLine result = getPythonProcessOutput(pythonExecution, askForSudo, showProgress, targetEnvironmentRequest); + String path = result.getExePath(); + List args = result.getArgs(); + ProcessOutput processOutput = result.getProcessOutput(); + int exitCode = processOutput.getExitCode(); + if (processOutput.isTimeout()) { + // TODO [targets] Make cancellable right away? + throw PyExecutionException.createForTimeout(PySdkBundle.message("python.sdk.packaging.timed.out"), path, args); } - catch (PyExecutionException e) { - var pyError = e.getPyError(); - if (pyError instanceof ExecErrorImpl error) { - var errorReason = error.getErrorReason(); - if (errorReason instanceof ExecErrorReason.UnexpectedProcessTermination unexpectedProcessTermination) { - int exitCode = unexpectedProcessTermination.getExitCode(); - if (exitCode == ERROR_NO_SETUPTOOLS) { - return null; - } - } - } - throw e; + else if (exitCode != 0) { + throw new PyExecutionException(PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode), path, args, processOutput); } + return processOutput.getStdout(); } - protected void installManagement(@NotNull String name) throws ExecutionException { - installUsingPipWheel("--no-index", name); + protected final @NotNull Sdk getSdk() { + return mySdk; } - @Override - public @NotNull List refreshAndGetPackages(boolean alwaysRefresh) throws ExecutionException { - final List currentPackages = myPackagesCache; - if (alwaysRefresh || currentPackages == null) { - myPackagesCache = null; + + private @NotNull PyTargetEnvCreationManager.ProcessOutputWithCommandLine getPythonProcessOutput(@NotNull PythonExecution pythonExecution, + boolean askForSudo, + boolean showProgress, + @NotNull TargetEnvironmentRequest targetEnvironmentRequest) + throws ExecutionException { + // TODO [targets] Use `showProgress = true` + // TODO [targets] Use `workingDir` + // TODO [targets] Use `useUserSite` (handle use sudo) + TargetProgressIndicator targetProgressIndicator = TargetProgressIndicator.EMPTY; + TargetEnvironment targetEnvironment = targetEnvironmentRequest.prepareEnvironment(targetProgressIndicator); + for (Map.Entry entry : targetEnvironment.getUploadVolumes() + .entrySet()) { try { - final List packages = collectPackages(); - LOG.debug("Packages installed in " + getSdk().getName() + ": " + packages); - myPackagesCache = packages; - ApplicationManager.getApplication().getMessageBus().syncPublisher(PACKAGE_MANAGER_TOPIC).packagesRefreshed(getSdk()); - return Collections.unmodifiableList(packages); + entry.getValue().upload(".", TargetProgressIndicator.EMPTY); } - catch (ExecutionException e) { - myPackagesCache = Collections.emptyList(); - throw e; + catch (IOException e) { + throw new ExecutionException(e); } } - return Collections.unmodifiableList(currentPackages); + // TODO [targets] Should `interpreterParameters` be here? + TargetedCommandLine targetedCommandLine = PythonScripts.buildTargetedCommandLine(pythonExecution, + targetEnvironment, + getSdk(), + Collections.emptyList() + ); + // TODO [targets] Set parent directory of interpreter as the working directory + + LOG.info("Running packaging tool"); + + // TODO [targets] Apply environment variables: setPythonUnbuffered(...), setPythonDontWriteBytecode(...), resetHomePathChanges(...) + // TODO [targets] Apply flavor from PythonSdkFlavor.getFlavor(mySdk) + ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator(); + Process process = createProcess(targetEnvironment, targetedCommandLine, askForSudo, indicator); + List commandLine = targetedCommandLine.collectCommandsSynchronously(); + String commandLineString = StringUtil.join(commandLine, " "); + final CapturingProcessHandler handler = + new CapturingProcessHandler(process, targetedCommandLine.getCharset(), commandLineString); + final ProcessOutput result; + if (showProgress && indicator != null) { + handler.addProcessListener(new IndicatedProcessOutputListener(indicator)); + result = handler.runProcessWithProgressIndicator(indicator); + } + else { + // TODO [targets] Check if timeout is ok for all targets + result = handler.runProcess(TIMEOUT); + } + if (result.isCancelled()) { + throw new RunCanceledByUserException(); + } + result.checkSuccess(LOG); + final int exitCode = result.getExitCode(); + String helperPath = ContainerUtil.getFirstItem(commandLine, ""); + List args = commandLine.subList(Math.min(1, commandLine.size()), commandLine.size()); + if (exitCode != 0) { + final String message = StringUtil.isEmptyOrSpaces(result.getStdout()) && StringUtil.isEmptyOrSpaces(result.getStderr()) + ? PySdkBundle.message("python.conda.permission.denied") + : PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode); + throw new PyExecutionException(message, helperPath, args, result); + } + return new ProcessOutputWithCommandLine(helperPath, args, result); } - protected final void refreshPackagesSynchronously() { - PyPackageUtil.updatePackagesSynchronouslyWithGuard(this, myUpdatingCache); + private @NotNull Process createProcess(@NotNull TargetEnvironment targetEnvironment, + @NotNull TargetedCommandLine targetedCommandLine, + boolean askForSudo, + @Nullable ProgressIndicator indicator) throws ExecutionException { + if (askForSudo) { + if (!(targetEnvironment instanceof LocalTargetEnvironment)) { + // TODO [targets] Execute process on non-local target using sudo + LOG.warn("Sudo flag is ignored"); + } + else if (PySdkExtKt.adminPermissionsNeeded(getSdk())) { + // This is hack to process sudo flag in the local environment + GeneralCommandLine localCommandLine = ((LocalTargetEnvironment)targetEnvironment).createGeneralCommandLine(targetedCommandLine); + return executeOnLocalMachineWithSudo(localCommandLine); + } + } + // TODO [targets] Pass meaningful progress indicator + return targetEnvironment.createProcess(targetedCommandLine, Objects.requireNonNullElseGet(indicator, EmptyProgressIndicator::new)); } - protected @Nullable String getHelperPath(final @NotNull String helper) throws ExecutionException { - return PythonHelpersLocator.findPathStringInHelpers(helper); + protected void installUsingPipWheel(String @NotNull ... pipArgs) throws ExecutionException { + HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter(); + PythonScriptExecution pythonExecution = + PythonScripts.prepareHelperScriptExecution(getPipHelperPackage(), helpersAwareTargetRequest); + pythonExecution.addParameter(INSTALL); + pythonExecution.addParameters(pipArgs); + + getPythonProcessResult(pythonExecution, true, true, helpersAwareTargetRequest.getTargetEnvironmentRequest()); } @RequiresReadLock(generateAssertion = false) @@ -284,73 +324,6 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager return packages; } - protected void installUsingPipWheel(String @NotNull ... pipArgs) throws ExecutionException { - HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter(); - PythonScriptExecution pythonExecution = - PythonScripts.prepareHelperScriptExecution(getPipHelperPackage(), helpersAwareTargetRequest); - pythonExecution.addParameter(INSTALL); - pythonExecution.addParameters(pipArgs); - - getPythonProcessResult(pythonExecution, true, true, helpersAwareTargetRequest.getTargetEnvironmentRequest()); - } - - public PyTargetEnvironmentPackageManager(final @NotNull Sdk sdk) { - super(sdk); - } - - @Override - public void install(@Nullable List requirements, @NotNull List extraArgs) throws ExecutionException { - install(requirements, extraArgs, null); - } - - public void install(@Nullable List requirements, @NotNull List extraArgs, @Nullable String workingDir) - throws ExecutionException { - if (requirements == null) return; - if (!hasManagement()) { - installManagement(); - } - HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter(); - TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareTargetRequest.getTargetEnvironmentRequest(); - PythonScriptExecution pythonExecution = - PythonScripts.prepareHelperScriptExecution(PythonHelper.PACKAGING_TOOL, helpersAwareTargetRequest); - - applyWorkingDir(pythonExecution, workingDir); - - pythonExecution.addParameter(INSTALL); - - final boolean useUserSite = extraArgs.contains(USE_USER_SITE); - - final String proxyString = getProxyString(); - if (proxyString != null) { - pythonExecution.addParameter("--proxy"); - pythonExecution.addParameter(proxyString); - } - pythonExecution.addParameters(extraArgs); - for (PyRequirement req : requirements) { - pythonExecution.addParameters(req.getInstallOptions()); - } - try { - getPythonProcessResult(pythonExecution, !useUserSite, true, targetEnvironmentRequest); - } - catch (PyExecutionException e) { - final List simplifiedArgs = new ArrayList<>(); - simplifiedArgs.add("install"); - if (proxyString != null) { - simplifiedArgs.add("--proxy"); - simplifiedArgs.add(proxyString); - } - simplifiedArgs.addAll(extraArgs); - for (PyRequirement req : requirements) { - simplifiedArgs.addAll(req.getInstallOptions()); - } - throw PyExecutionExceptionExtKt.copyWith(e, "pip", makeSafeToDisplayCommand(simplifiedArgs)); - } - finally { - LOG.debug("Packages cache is about to be refreshed because these requirements were installed: " + requirements); - refreshPackagesSynchronously(); - } - } - private static void applyWorkingDir(@NotNull PythonScriptExecution execution, @Nullable String workingDir) { if (workingDir == null) { // TODO [targets] Set the parent of home path as the working directory @@ -361,174 +334,6 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager } - @Override - public @Nullable List getPackages() { - final List packages = myPackagesCache; - return packages != null ? Collections.unmodifiableList(packages) : null; - } - - protected @NotNull List collectPackages() throws ExecutionException { - HelpersAwareTargetEnvironmentRequest helpersAwareRequest = getPythonTargetInterpreter(); - TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareRequest.getTargetEnvironmentRequest(); - final String output; - try { - LOG.debug("Collecting installed packages for the SDK " + getSdk().getName(), new Throwable()); - PythonScriptExecution pythonExecution = - PythonScripts.prepareHelperScriptExecution(PythonHelper.PACKAGING_TOOL, helpersAwareRequest); - pythonExecution.addParameter("list"); - output = getPythonProcessResult(pythonExecution, false, false, targetEnvironmentRequest); - } - catch (ProcessNotCreatedException ex) { - if (ApplicationManager.getApplication().isUnitTestMode()) { - LOG.info("Not-env unit test mode, will return mock packages"); - return List.of(new PyPackage(PyPackageUtil.PIP, PIP_VERSION), - new PyPackage(PyPackageUtil.SETUPTOOLS, SETUPTOOLS_VERSION)); - } - else { - throw ex; - } - } - - return parsePackagingToolOutput(output); - } - - @Override - public @NotNull String createVirtualEnv(@NotNull String destinationDir, boolean useGlobalSite) throws ExecutionException { - final Sdk sdk = getSdk(); - final LanguageLevel languageLevel = getOrRequestLanguageLevelForSdk(sdk); - - if (languageLevel.isOlderThan(LanguageLevel.PYTHON27)) { - throw new ExecutionException(PySdkBundle.message("python.sdk.packaging.creating.virtual.environment.for.python.not.supported", - languageLevel, LanguageLevel.PYTHON27)); - } - - HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter(); - TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareTargetRequest.getTargetEnvironmentRequest(); - - PythonScriptExecution pythonExecution = PythonScripts.prepareHelperScriptExecution( - isLegacyPython(languageLevel) ? PythonHelper.LEGACY_VIRTUALENV_ZIPAPP : PythonHelper.VIRTUALENV_ZIPAPP, - helpersAwareTargetRequest); - if (useGlobalSite) { - pythonExecution.addParameter("--system-site-packages"); - } - pythonExecution.addParameter(destinationDir); - // TODO [targets] Pass `parentDir = null` - getPythonProcessResult(pythonExecution, false, true, targetEnvironmentRequest); - - final Path binary = VirtualEnvReader.getInstance().findPythonInPythonRoot(Path.of(destinationDir)); - final char separator = targetEnvironmentRequest.getTargetPlatform().getPlatform().fileSeparator; - final String binaryFallback = destinationDir + separator + "bin" + separator + "python"; - - return (binary != null) ? binary.toString() : binaryFallback; - } - - /** - * Is it a legacy python version that we still support - */ - private static @NotNull Boolean isLegacyPython(@NotNull LanguageLevel languageLevel) { - return languageLevel.isPython2() || languageLevel.isOlderThan(LanguageLevel.PYTHON37); - } - - private @NotNull String getPythonProcessResult(@NotNull PythonExecution pythonExecution, - boolean askForSudo, - boolean showProgress, - @NotNull TargetEnvironmentRequest targetEnvironmentRequest) throws ExecutionException { - ProcessOutputWithCommandLine result = getPythonProcessOutput(pythonExecution, askForSudo, showProgress, targetEnvironmentRequest); - String path = result.getExePath(); - List args = result.getArgs(); - ProcessOutput processOutput = result.getProcessOutput(); - int exitCode = processOutput.getExitCode(); - if (processOutput.isTimeout()) { - // TODO [targets] Make cancellable right away? - throw PyExecutionException.createForTimeout(PySdkBundle.message("python.sdk.packaging.timed.out"), path, args); - } - else if (exitCode != 0) { - throw new PyExecutionException(PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode), path, args, processOutput); - } - return processOutput.getStdout(); - } - - private @NotNull PyTargetEnvironmentPackageManager.ProcessOutputWithCommandLine getPythonProcessOutput(@NotNull PythonExecution pythonExecution, - boolean askForSudo, - boolean showProgress, - @NotNull TargetEnvironmentRequest targetEnvironmentRequest) - throws ExecutionException { - // TODO [targets] Use `showProgress = true` - // TODO [targets] Use `workingDir` - // TODO [targets] Use `useUserSite` (handle use sudo) - TargetProgressIndicator targetProgressIndicator = TargetProgressIndicator.EMPTY; - TargetEnvironment targetEnvironment = targetEnvironmentRequest.prepareEnvironment(targetProgressIndicator); - for (Map.Entry entry : targetEnvironment.getUploadVolumes() - .entrySet()) { - try { - entry.getValue().upload(".", TargetProgressIndicator.EMPTY); - } - catch (IOException e) { - throw new ExecutionException(e); - } - } - // TODO [targets] Should `interpreterParameters` be here? - TargetedCommandLine targetedCommandLine = PythonScripts.buildTargetedCommandLine(pythonExecution, - targetEnvironment, - getSdk(), - Collections.emptyList() - ); - // TODO [targets] Set parent directory of interpreter as the working directory - - LOG.info("Running packaging tool"); - - // TODO [targets] Apply environment variables: setPythonUnbuffered(...), setPythonDontWriteBytecode(...), resetHomePathChanges(...) - // TODO [targets] Apply flavor from PythonSdkFlavor.getFlavor(mySdk) - ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator(); - Process process = createProcess(targetEnvironment, targetedCommandLine, askForSudo, indicator); - List commandLine = targetedCommandLine.collectCommandsSynchronously(); - String commandLineString = StringUtil.join(commandLine, " "); - final CapturingProcessHandler handler = - new CapturingProcessHandler(process, targetedCommandLine.getCharset(), commandLineString); - final ProcessOutput result; - if (showProgress && indicator != null) { - handler.addProcessListener(new IndicatedProcessOutputListener(indicator)); - result = handler.runProcessWithProgressIndicator(indicator); - } - else { - // TODO [targets] Check if timeout is ok for all targets - result = handler.runProcess(TIMEOUT); - } - if (result.isCancelled()) { - throw new RunCanceledByUserException(); - } - result.checkSuccess(LOG); - final int exitCode = result.getExitCode(); - String helperPath = ContainerUtil.getFirstItem(commandLine, ""); - List args = commandLine.subList(Math.min(1, commandLine.size()), commandLine.size()); - if (exitCode != 0) { - final String message = StringUtil.isEmptyOrSpaces(result.getStdout()) && StringUtil.isEmptyOrSpaces(result.getStderr()) - ? PySdkBundle.message("python.conda.permission.denied") - : PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode); - throw new PyExecutionException(message, helperPath, args, result); - } - return new ProcessOutputWithCommandLine(helperPath, args, result); - } - - private @NotNull Process createProcess(@NotNull TargetEnvironment targetEnvironment, - @NotNull TargetedCommandLine targetedCommandLine, - boolean askForSudo, - @Nullable ProgressIndicator indicator) throws ExecutionException { - if (askForSudo) { - if (!(targetEnvironment instanceof LocalTargetEnvironment)) { - // TODO [targets] Execute process on non-local target using sudo - LOG.warn("Sudo flag is ignored"); - } - else if (PySdkExtKt.adminPermissionsNeeded(getSdk())) { - // This is hack to process sudo flag in the local environment - GeneralCommandLine localCommandLine = ((LocalTargetEnvironment)targetEnvironment).createGeneralCommandLine(targetedCommandLine); - return executeOnLocalMachineWithSudo(localCommandLine); - } - } - // TODO [targets] Pass meaningful progress indicator - return targetEnvironment.createProcess(targetedCommandLine, Objects.requireNonNullElseGet(indicator, EmptyProgressIndicator::new)); - } - private static @NotNull Process executeOnLocalMachineWithSudo(@NotNull GeneralCommandLine localCommandLine) throws ExecutionException { try { return ExecUtil.sudo(localCommandLine, PySdkBundle.message("python.sdk.packaging.enter.your.password.to.make.changes")); @@ -540,15 +345,6 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager } } - private @NotNull HelpersAwareTargetEnvironmentRequest getPythonTargetInterpreter() throws ExecutionException { - HelpersAwareTargetEnvironmentRequest request = PythonInterpreterTargetEnvironmentFactory.findPythonTargetInterpreter(getSdk(), - ProjectManager.getInstance() - .getDefaultProject()); - if (request == null) { - throw new ExecutionException(PySdkBundle.message("python.sdk.package.managing.not.supported.for.sdk", getSdk().getName())); - } - return request; - } private static @NotNull HelperPackage getPipHelperPackage() { return new PythonHelper.ScriptPythonHelper(PIP_WHEEL_NAME + "/" + PyPackageUtil.PIP, diff --git a/python/src/com/jetbrains/python/sdk/configuration/PyProjectVirtualEnvConfiguration.kt b/python/src/com/jetbrains/python/sdk/configuration/PyProjectVirtualEnvConfiguration.kt index 86addab0c539..c1ef6f246d75 100644 --- a/python/src/com/jetbrains/python/sdk/configuration/PyProjectVirtualEnvConfiguration.kt +++ b/python/src/com/jetbrains/python/sdk/configuration/PyProjectVirtualEnvConfiguration.kt @@ -22,17 +22,17 @@ import com.intellij.openapi.util.UserDataHolderBase import com.intellij.openapi.util.io.FileUtil import com.intellij.util.concurrency.annotations.RequiresEdt import com.jetbrains.python.PySdkBundle -import com.jetbrains.python.packaging.PyPackageManager import com.jetbrains.python.packaging.PyPackageManagers +import com.jetbrains.python.packaging.PyTargetEnvCreationManager import com.jetbrains.python.run.PythonInterpreterTargetEnvironmentFactory import com.jetbrains.python.sdk.* -import com.jetbrains.python.target.ui.PyAddSdkPanelBase -import com.jetbrains.python.target.ui.PyAddSdkPanelBase.Companion.isLocal -import com.jetbrains.python.target.ui.TargetPanelExtension import com.jetbrains.python.sdk.flavors.PyFlavorAndData import com.jetbrains.python.sdk.flavors.PyFlavorData import com.jetbrains.python.target.PyTargetAwareAdditionalData import com.jetbrains.python.target.getInterpreterVersion +import com.jetbrains.python.target.ui.PyAddSdkPanelBase +import com.jetbrains.python.target.ui.PyAddSdkPanelBase.Companion.isLocal +import com.jetbrains.python.target.ui.TargetPanelExtension import com.jetbrains.python.ui.pyModalBlocking import org.jetbrains.annotations.ApiStatus @@ -75,8 +75,7 @@ fun createVirtualEnvAndSdkSynchronously( } try { - val packageManager = PyPackageManager.getInstance(sdk) - return packageManager.createVirtualEnv(venvRoot, inheritSitePackages) + return PyTargetEnvCreationManager(sdk).createVirtualEnv(venvRoot, inheritSitePackages) } finally { PyPackageManagers.getInstance().clearCache(sdk) diff --git a/python/testSrc/com/jetbrains/env/python/PyPackagingTest.java b/python/testSrc/com/jetbrains/env/python/PyPackagingTest.java index c57bb4a482c7..75002860df32 100644 --- a/python/testSrc/com/jetbrains/env/python/PyPackagingTest.java +++ b/python/testSrc/com/jetbrains/env/python/PyPackagingTest.java @@ -12,7 +12,7 @@ import com.jetbrains.env.PyExecutionFixtureTestTask; import com.jetbrains.env.PyTestTask; import com.jetbrains.python.packaging.PyPackage; import com.jetbrains.python.packaging.PyPackageManager; -import com.jetbrains.python.packaging.requirement.PyRequirementRelation; +import com.jetbrains.python.packaging.PyTargetEnvCreationManager; import com.jetbrains.python.sdk.flavors.PythonSdkFlavor; import com.jetbrains.python.sdk.flavors.VirtualEnvSdkFlavor; import com.jetbrains.python.tools.sdkTools.SdkCreationType; @@ -23,14 +23,11 @@ import org.junit.Test; import java.io.File; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; import static com.intellij.testFramework.UsefulTestCase.assertInstanceOf; import static com.jetbrains.python.SdkUiUtilKt.isVirtualEnv; -import static com.jetbrains.python.packaging.PyRequirementsKt.pyRequirement; import static org.junit.Assert.*; public class PyPackagingTest extends PyEnvTestCase { @@ -72,7 +69,8 @@ public class PyPackagingTest extends PyEnvTestCase { try { final File tempDir = FileUtil.createTempDirectory(getTestName(false), null); final File venvDir = new File(tempDir, "venv"); - final String venvSdkHome = PyPackageManager.getInstance(sdk).createVirtualEnv(venvDir.toString(), + PyTargetEnvCreationManager manager = new PyTargetEnvCreationManager(sdk); + final String venvSdkHome = manager.createVirtualEnv(venvDir.toString(), false); final Sdk venvSdk = createTempSdk(venvSdkHome, SdkCreationType.EMPTY_SDK); assertNotNull(venvSdk); @@ -96,40 +94,6 @@ public class PyPackagingTest extends PyEnvTestCase { }); } - @Test - public void testInstallPackage() { - runPythonTest(new PyPackagingTestTask() { - - @Override - public void runTestOn(@NotNull String sdkHome, @Nullable Sdk existingSdk) throws Exception { - final Sdk sdk = createTempSdk(sdkHome, SdkCreationType.EMPTY_SDK); - try { - final File tempDir = FileUtil.createTempDirectory(getTestName(false), null); - final File venvDir = new File(tempDir, "venv"); - final String venvSdkHome = PyPackageManager.getInstance(sdk).createVirtualEnv(venvDir.getPath(), false); - final Sdk venvSdk = createTempSdk(venvSdkHome, SdkCreationType.EMPTY_SDK); - assertNotNull(venvSdk); - final PyPackageManager manager = PyPackageManager.getInstance(venvSdk); - final List packages1 = manager.refreshAndGetPackages(false); - // TODO: Install Markdown from a local file - manager.install(Arrays.asList(pyRequirement("Markdown", PyRequirementRelation.LTE, "3.3.3"), pyRequirement("httplib2",null)), - Collections.emptyList()); - final List packages2 = manager.refreshAndGetPackages(false); - final PyPackage markdown2 = findPackage("markdown", packages2); - assertNotNull(markdown2); - final PyPackage pip1 = findPackage("pip", packages1); - assertNotNull(pip1); - assertEquals("pip", pip1.getName()); - } - catch (ExecutionException e) { - throw new RuntimeException(String.format("Error for interpreter '%s': %s", sdk.getHomePath(), e.getMessage()), e); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - } @Nullable private static PyPackage findPackage(String name, List packages) {