PY-78749 Packages: Remove PyTargetEnvironmentPackageManager

GitOrigin-RevId: 1a3eb25f4ef3c897fb3abba61be00acf366dde66
This commit is contained in:
Nikita.Ashihmin
2025-08-11 01:30:33 +00:00
committed by intellij-monorepo-bot
parent 8beb05e4ad
commit 1a7a0eefd9
5 changed files with 169 additions and 424 deletions
@@ -18,7 +18,6 @@ import com.intellij.openapi.util.NlsSafe
import com.intellij.openapi.util.use
import com.intellij.openapi.vfs.VfsUtil
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.platform.util.progress.reportRawProgress
import com.intellij.pycharm.community.ide.impl.PyCharmCommunityCustomizationBundle
import com.intellij.pycharm.community.ide.impl.configuration.PySdkConfigurationCollector.InputData
import com.intellij.pycharm.community.ide.impl.configuration.PySdkConfigurationCollector.Source
@@ -27,10 +26,8 @@ import com.intellij.pycharm.community.ide.impl.configuration.ui.PyAddNewVirtualE
import com.intellij.ui.IdeBorderFactory
import com.intellij.ui.components.JBLabel
import com.intellij.util.ui.JBUI
import com.jetbrains.python.PyBundle
import com.jetbrains.python.PySdkBundle
import com.jetbrains.python.errorProcessing.PyResult
import com.jetbrains.python.packaging.PyPackageManager
import com.jetbrains.python.packaging.PyPackageUtil
import com.jetbrains.python.packaging.management.PythonPackageManager
import com.jetbrains.python.packaging.requirementsTxt.PythonRequirementTxtSdkUtils
@@ -40,7 +37,6 @@ import com.jetbrains.python.sdk.PythonSdkUtil
import com.jetbrains.python.sdk.basePath
import com.jetbrains.python.sdk.configuration.PyProjectSdkConfigurationExtension
import com.jetbrains.python.sdk.configuration.createVirtualEnvAndSdkSynchronously
import com.jetbrains.python.sdk.isTargetBased
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import java.awt.BorderLayout
@@ -94,20 +90,7 @@ class PyRequirementsTxtOrSetupPySdkConfiguration : PyProjectSdkConfigurationExte
PythonRequirementTxtSdkUtils.saveRequirementsTxtPath(module.project, sdk, requirementsTxtOrSetupPyFile.toNioPath())
}
if (!sdk.isTargetBased()) {
val pythonPackageManager = PythonPackageManager.forSdk(module.project, sdk)
pythonPackageManager.sync().getOr {
PySdkConfigurationCollector.logVirtualEnv(module.project, VirtualEnvResult.INSTALLATION_FAILURE)
return it
}
}
else {
withContext(Dispatchers.Default) {
createTargetBased(sdk, requirementsTxtOrSetupPyFile)
}
}
return PyResult.success(sdk)
return PythonPackageManager.forSdk(module.project, sdk).sync().mapSuccess { sdk }
}
catch (e: ExecutionException) {
PySdkConfigurationCollector.logVirtualEnv(module.project, VirtualEnvResult.INSTALLATION_FAILURE)
@@ -116,18 +99,6 @@ class PyRequirementsTxtOrSetupPySdkConfiguration : PyProjectSdkConfigurationExte
}
}
private suspend fun createTargetBased(
sdk: Sdk,
requirementsTxtOrSetupPyFile: VirtualFile,
) {
reportRawProgress {
it.text(PyBundle.message("python.packaging.installing.packages"))
val packageManager = PyPackageManager.getInstance(sdk)
val command = getCommandForPipInstall(requirementsTxtOrSetupPyFile)
packageManager.install(emptyList(), command)
}
}
private fun getRequirementsTxtOrSetupPy(module: Module) =
PyPackageUtil.findRequirementsTxt(module) ?: PyPackageUtil.findSetupPy(module)?.virtualFile
@@ -2,26 +2,49 @@
package com.jetbrains.python.packaging
import com.intellij.execution.ExecutionException
import com.intellij.openapi.application.writeAction
import com.intellij.openapi.diagnostic.thisLogger
import com.intellij.openapi.progress.runBlockingMaybeCancellable
import com.intellij.openapi.project.ProjectManager
import com.intellij.openapi.project.getOpenedProjects
import com.intellij.openapi.projectRoots.Sdk
import com.intellij.openapi.roots.OrderRootType
import com.intellij.openapi.vfs.VfsUtil
import com.intellij.util.cancelOnDispose
import com.jetbrains.python.getOrNull
import com.jetbrains.python.onFailure
import com.jetbrains.python.packaging.management.PythonPackageManager
import com.jetbrains.python.packaging.management.ui.PythonPackageManagerUI
import com.jetbrains.python.packaging.management.ui.installPyRequirementsBackground
import com.jetbrains.python.packaging.utils.PyPackageCoroutine
import com.jetbrains.python.sdk.PythonSdkType
import org.jetbrains.annotations.ApiStatus
@ApiStatus.Internal
internal open class PyPackageManagerBridge(sdk: Sdk) : PyTargetEnvironmentPackageManager(sdk) {
internal open class PyPackageManagerBridge(private val sdk: Sdk) : PyPackageManager(sdk) {
protected val packageManager: PythonPackageManager = PythonPackageManager.forSdk(guessProject(), sdk = sdk)
protected val packageManagerUI: PythonPackageManagerUI = PythonPackageManagerUI.forPackageManager(packageManager)
override fun installManagement() {}
override fun hasManagement(): Boolean = true
override fun refresh() {
PyPackageCoroutine.launch(null) {
thisLogger().debug("Refreshing SDK roots and packages cache")
writeAction {
val files = sdk.getRootProvider().getFiles(OrderRootType.CLASSES)
VfsUtil.markDirtyAndRefresh(true, true, true, *files)
}
PythonSdkType.getInstance().setupSdkPaths(sdk)
}.cancelOnDispose(this)
}
override fun createVirtualEnv(destinationDir: String, useGlobalSite: Boolean): String {
val manager = PyTargetEnvCreationManager(getSdk())
return manager.createVirtualEnv(destinationDir, useGlobalSite)
}
@Throws(ExecutionException::class)
override fun install(requirements: MutableList<PyRequirement>?, extraArgs: MutableList<String>) {
runBlockingMaybeCancellable {
@@ -48,14 +71,6 @@ internal open class PyPackageManagerBridge(sdk: Sdk) : PyTargetEnvironmentPackag
}
@Throws(ExecutionException::class)
override fun collectPackages(): List<PyPackage> {
return runBlockingMaybeCancellable {
packageManagerUI.manager.listInstalledPackages()
}.map { PyPackage(it.name, it.version) }
}
override fun getPackages(): List<PyPackage> {
return packageManagerUI.manager.listInstalledPackagesSnapshot().map { PyPackage(it.name, it.version) }
}
@@ -5,7 +5,6 @@ import com.intellij.execution.ExecutionException;
import com.intellij.execution.RunCanceledByUserException;
import com.intellij.execution.configurations.GeneralCommandLine;
import com.intellij.execution.process.CapturingProcessHandler;
import com.intellij.execution.process.ProcessNotCreatedException;
import com.intellij.execution.process.ProcessOutput;
import com.intellij.execution.target.TargetEnvironment;
import com.intellij.execution.target.TargetEnvironmentRequest;
@@ -14,28 +13,20 @@ import com.intellij.execution.target.TargetedCommandLine;
import com.intellij.execution.target.local.LocalTargetEnvironment;
import com.intellij.execution.target.value.TargetEnvironmentFunctions;
import com.intellij.execution.util.ExecUtil;
import com.intellij.openapi.application.Application;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.EmptyProgressIndicator;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.project.ProjectManager;
import com.intellij.openapi.projectRoots.Sdk;
import com.intellij.openapi.roots.OrderRootType;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.VfsUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.python.community.helpersLocator.PythonHelpersLocator;
import com.intellij.util.concurrency.annotations.RequiresReadLock;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.net.HttpConfigurable;
import com.jetbrains.python.HelperPackage;
import com.jetbrains.python.PyPsiPackageUtil;
import com.jetbrains.python.PySdkBundle;
import com.jetbrains.python.PythonHelper;
import com.jetbrains.python.errorProcessing.ExecErrorImpl;
import com.jetbrains.python.errorProcessing.ExecErrorReason;
import com.jetbrains.python.packaging.common.PythonPackage;
import com.jetbrains.python.packaging.pip.PipParseUtils;
import com.jetbrains.python.packaging.repository.PyPackageRepositoryUtil;
@@ -63,16 +54,15 @@ import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
import static com.intellij.webcore.packaging.PackageVersionComparator.VERSION_COMPARATOR;
/**
* @deprecated TODO: explain
*/
@SuppressWarnings("ALL")
@Deprecated(forRemoval = true)
@ApiStatus.Internal
public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager {
private static final Logger LOG = Logger.getInstance(PyTargetEnvironmentPackageManager.class);
public class PyTargetEnvCreationManager {
private static final Logger LOG = Logger.getInstance(PyTargetEnvCreationManager.class);
private final @NotNull Sdk mySdk;
protected static final String SETUPTOOLS_VERSION = "44.1.1";
protected static final String PIP_VERSION = "24.3.1";
@@ -92,119 +82,169 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager
protected String mySeparator = File.separator;
protected volatile @Nullable List<PyPackage> myPackagesCache = null;
@Override
public void refresh() {
LOG.debug("Refreshing SDK roots and packages cache");
final Application application = ApplicationManager.getApplication();
application.invokeLater(() -> {
final Sdk sdk = getSdk();
application.runWriteAction(() -> {
final VirtualFile[] files = sdk.getRootProvider().getFiles(OrderRootType.CLASSES);
VfsUtil.markDirtyAndRefresh(true, true, true, files);
});
PythonSdkType.getInstance().setupSdkPaths(sdk);
});
public PyTargetEnvCreationManager(final @NotNull Sdk sdk) {
mySdk = sdk;
}
@Override
public void installManagement() throws ExecutionException {
final LanguageLevel languageLevel = PythonSdkType.getLanguageLevelForSdk(getSdk());
public @NotNull String createVirtualEnv(@NotNull String destinationDir, boolean useGlobalSite) throws ExecutionException {
final Sdk sdk = getSdk();
final LanguageLevel languageLevel = getOrRequestLanguageLevelForSdk(sdk);
if (languageLevel.isOlderThan(LanguageLevel.PYTHON27)) {
throw new ExecutionException(PySdkBundle.message("python.sdk.packaging.package.management.for.python.not.supported",
throw new ExecutionException(PySdkBundle.message("python.sdk.packaging.creating.virtual.environment.for.python.not.supported",
languageLevel, LanguageLevel.PYTHON27));
}
boolean success = updatePackagingTools();
if (success) {
return;
}
HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter();
TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareTargetRequest.getTargetEnvironmentRequest();
if (languageLevel.isOlderThan(LanguageLevel.PYTHON312)) { // Python 3.12 doesn't require setuptools to list packages anymore
final PyPackage installedSetuptools = refreshAndCheckForSetuptools();
if (installedSetuptools == null || VERSION_COMPARATOR.compare(installedSetuptools.getVersion(), SETUPTOOLS_VERSION) < 0) {
installManagement(Objects.requireNonNull(getHelperPath(SETUPTOOLS_WHEEL_NAME)));
}
PythonScriptExecution pythonExecution = PythonScripts.prepareHelperScriptExecution(
isLegacyPython(languageLevel) ? PythonHelper.LEGACY_VIRTUALENV_ZIPAPP : PythonHelper.VIRTUALENV_ZIPAPP,
helpersAwareTargetRequest);
if (useGlobalSite) {
pythonExecution.addParameter("--system-site-packages");
}
pythonExecution.addParameter(destinationDir);
// TODO [targets] Pass `parentDir = null`
getPythonProcessResult(pythonExecution, false, true, targetEnvironmentRequest);
final PyPackage installedPip = PyPsiPackageUtil.findPackage(refreshAndGetPackages(false), PyPackageUtil.PIP);
if (installedPip == null || VERSION_COMPARATOR.compare(installedPip.getVersion(), PIP_VERSION) < 0) {
installManagement(Objects.requireNonNull(getHelperPath(PIP_WHEEL_NAME)));
}
final Path binary = VirtualEnvReader.getInstance().findPythonInPythonRoot(Path.of(destinationDir));
final char separator = targetEnvironmentRequest.getTargetPlatform().getPlatform().fileSeparator;
final String binaryFallback = destinationDir + separator + "bin" + separator + "python";
return (binary != null) ? binary.toString() : binaryFallback;
}
protected final boolean updatePackagingTools() {
try {
installUsingPipWheel("--upgrade", "--force-reinstall", PyPackageUtil.SETUPTOOLS, PyPackageUtil.PIP);
return true;
}
catch (ExecutionException e) {
LOG.info(e);
return false;
}
finally {
refreshPackagesSynchronously();
private @NotNull HelpersAwareTargetEnvironmentRequest getPythonTargetInterpreter() throws ExecutionException {
HelpersAwareTargetEnvironmentRequest request = PythonInterpreterTargetEnvironmentFactory.findPythonTargetInterpreter(getSdk(),
ProjectManager.getInstance()
.getDefaultProject());
if (request == null) {
throw new ExecutionException(PySdkBundle.message("python.sdk.package.managing.not.supported.for.sdk", getSdk().getName()));
}
return request;
}
@Override
public boolean hasManagement() throws ExecutionException {
final LanguageLevel languageLevel = PythonSdkType.getLanguageLevelForSdk(getSdk());
final Boolean hasSetuptools = languageLevel.isAtLeast(LanguageLevel.PYTHON312) || refreshAndCheckForSetuptools() != null;
final Boolean hasPip = PyPsiPackageUtil.findPackage(refreshAndGetPackages(false), PyPackageUtil.PIP) != null;
return hasSetuptools && hasPip;
/**
* Is it a legacy python version that we still support
*/
private static @NotNull Boolean isLegacyPython(@NotNull LanguageLevel languageLevel) {
return languageLevel.isPython2() || languageLevel.isOlderThan(LanguageLevel.PYTHON37);
}
protected final @Nullable PyPackage refreshAndCheckForSetuptools() throws ExecutionException {
try {
final List<PyPackage> packages = refreshAndGetPackages(false);
final PyPackage setuptoolsPackage = PyPsiPackageUtil.findPackage(packages, PyPackageUtil.SETUPTOOLS);
return setuptoolsPackage != null ? setuptoolsPackage : PyPsiPackageUtil.findPackage(packages, PyPackageUtil.DISTRIBUTE);
private @NotNull String getPythonProcessResult(@NotNull PythonExecution pythonExecution,
boolean askForSudo,
boolean showProgress,
@NotNull TargetEnvironmentRequest targetEnvironmentRequest) throws ExecutionException {
ProcessOutputWithCommandLine result = getPythonProcessOutput(pythonExecution, askForSudo, showProgress, targetEnvironmentRequest);
String path = result.getExePath();
List<String> args = result.getArgs();
ProcessOutput processOutput = result.getProcessOutput();
int exitCode = processOutput.getExitCode();
if (processOutput.isTimeout()) {
// TODO [targets] Make cancellable right away?
throw PyExecutionException.createForTimeout(PySdkBundle.message("python.sdk.packaging.timed.out"), path, args);
}
catch (PyExecutionException e) {
var pyError = e.getPyError();
if (pyError instanceof ExecErrorImpl<?> error) {
var errorReason = error.getErrorReason();
if (errorReason instanceof ExecErrorReason.UnexpectedProcessTermination unexpectedProcessTermination) {
int exitCode = unexpectedProcessTermination.getExitCode();
if (exitCode == ERROR_NO_SETUPTOOLS) {
return null;
}
}
}
throw e;
else if (exitCode != 0) {
throw new PyExecutionException(PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode), path, args, processOutput);
}
return processOutput.getStdout();
}
protected void installManagement(@NotNull String name) throws ExecutionException {
installUsingPipWheel("--no-index", name);
protected final @NotNull Sdk getSdk() {
return mySdk;
}
@Override
public @NotNull List<PyPackage> refreshAndGetPackages(boolean alwaysRefresh) throws ExecutionException {
final List<PyPackage> currentPackages = myPackagesCache;
if (alwaysRefresh || currentPackages == null) {
myPackagesCache = null;
private @NotNull PyTargetEnvCreationManager.ProcessOutputWithCommandLine getPythonProcessOutput(@NotNull PythonExecution pythonExecution,
boolean askForSudo,
boolean showProgress,
@NotNull TargetEnvironmentRequest targetEnvironmentRequest)
throws ExecutionException {
// TODO [targets] Use `showProgress = true`
// TODO [targets] Use `workingDir`
// TODO [targets] Use `useUserSite` (handle use sudo)
TargetProgressIndicator targetProgressIndicator = TargetProgressIndicator.EMPTY;
TargetEnvironment targetEnvironment = targetEnvironmentRequest.prepareEnvironment(targetProgressIndicator);
for (Map.Entry<TargetEnvironment.UploadRoot, TargetEnvironment.UploadableVolume> entry : targetEnvironment.getUploadVolumes()
.entrySet()) {
try {
final List<PyPackage> packages = collectPackages();
LOG.debug("Packages installed in " + getSdk().getName() + ": " + packages);
myPackagesCache = packages;
ApplicationManager.getApplication().getMessageBus().syncPublisher(PACKAGE_MANAGER_TOPIC).packagesRefreshed(getSdk());
return Collections.unmodifiableList(packages);
entry.getValue().upload(".", TargetProgressIndicator.EMPTY);
}
catch (ExecutionException e) {
myPackagesCache = Collections.emptyList();
throw e;
catch (IOException e) {
throw new ExecutionException(e);
}
}
return Collections.unmodifiableList(currentPackages);
// TODO [targets] Should `interpreterParameters` be here?
TargetedCommandLine targetedCommandLine = PythonScripts.buildTargetedCommandLine(pythonExecution,
targetEnvironment,
getSdk(),
Collections.emptyList()
);
// TODO [targets] Set parent directory of interpreter as the working directory
LOG.info("Running packaging tool");
// TODO [targets] Apply environment variables: setPythonUnbuffered(...), setPythonDontWriteBytecode(...), resetHomePathChanges(...)
// TODO [targets] Apply flavor from PythonSdkFlavor.getFlavor(mySdk)
ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
Process process = createProcess(targetEnvironment, targetedCommandLine, askForSudo, indicator);
List<String> commandLine = targetedCommandLine.collectCommandsSynchronously();
String commandLineString = StringUtil.join(commandLine, " ");
final CapturingProcessHandler handler =
new CapturingProcessHandler(process, targetedCommandLine.getCharset(), commandLineString);
final ProcessOutput result;
if (showProgress && indicator != null) {
handler.addProcessListener(new IndicatedProcessOutputListener(indicator));
result = handler.runProcessWithProgressIndicator(indicator);
}
else {
// TODO [targets] Check if timeout is ok for all targets
result = handler.runProcess(TIMEOUT);
}
if (result.isCancelled()) {
throw new RunCanceledByUserException();
}
result.checkSuccess(LOG);
final int exitCode = result.getExitCode();
String helperPath = ContainerUtil.getFirstItem(commandLine, "");
List<String> args = commandLine.subList(Math.min(1, commandLine.size()), commandLine.size());
if (exitCode != 0) {
final String message = StringUtil.isEmptyOrSpaces(result.getStdout()) && StringUtil.isEmptyOrSpaces(result.getStderr())
? PySdkBundle.message("python.conda.permission.denied")
: PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode);
throw new PyExecutionException(message, helperPath, args, result);
}
return new ProcessOutputWithCommandLine(helperPath, args, result);
}
protected final void refreshPackagesSynchronously() {
PyPackageUtil.updatePackagesSynchronouslyWithGuard(this, myUpdatingCache);
private @NotNull Process createProcess(@NotNull TargetEnvironment targetEnvironment,
@NotNull TargetedCommandLine targetedCommandLine,
boolean askForSudo,
@Nullable ProgressIndicator indicator) throws ExecutionException {
if (askForSudo) {
if (!(targetEnvironment instanceof LocalTargetEnvironment)) {
// TODO [targets] Execute process on non-local target using sudo
LOG.warn("Sudo flag is ignored");
}
else if (PySdkExtKt.adminPermissionsNeeded(getSdk())) {
// This is hack to process sudo flag in the local environment
GeneralCommandLine localCommandLine = ((LocalTargetEnvironment)targetEnvironment).createGeneralCommandLine(targetedCommandLine);
return executeOnLocalMachineWithSudo(localCommandLine);
}
}
// TODO [targets] Pass meaningful progress indicator
return targetEnvironment.createProcess(targetedCommandLine, Objects.requireNonNullElseGet(indicator, EmptyProgressIndicator::new));
}
protected @Nullable String getHelperPath(final @NotNull String helper) throws ExecutionException {
return PythonHelpersLocator.findPathStringInHelpers(helper);
protected void installUsingPipWheel(String @NotNull ... pipArgs) throws ExecutionException {
HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter();
PythonScriptExecution pythonExecution =
PythonScripts.prepareHelperScriptExecution(getPipHelperPackage(), helpersAwareTargetRequest);
pythonExecution.addParameter(INSTALL);
pythonExecution.addParameters(pipArgs);
getPythonProcessResult(pythonExecution, true, true, helpersAwareTargetRequest.getTargetEnvironmentRequest());
}
@RequiresReadLock(generateAssertion = false)
@@ -284,73 +324,6 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager
return packages;
}
protected void installUsingPipWheel(String @NotNull ... pipArgs) throws ExecutionException {
HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter();
PythonScriptExecution pythonExecution =
PythonScripts.prepareHelperScriptExecution(getPipHelperPackage(), helpersAwareTargetRequest);
pythonExecution.addParameter(INSTALL);
pythonExecution.addParameters(pipArgs);
getPythonProcessResult(pythonExecution, true, true, helpersAwareTargetRequest.getTargetEnvironmentRequest());
}
public PyTargetEnvironmentPackageManager(final @NotNull Sdk sdk) {
super(sdk);
}
@Override
public void install(@Nullable List<PyRequirement> requirements, @NotNull List<String> extraArgs) throws ExecutionException {
install(requirements, extraArgs, null);
}
public void install(@Nullable List<PyRequirement> requirements, @NotNull List<String> extraArgs, @Nullable String workingDir)
throws ExecutionException {
if (requirements == null) return;
if (!hasManagement()) {
installManagement();
}
HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter();
TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareTargetRequest.getTargetEnvironmentRequest();
PythonScriptExecution pythonExecution =
PythonScripts.prepareHelperScriptExecution(PythonHelper.PACKAGING_TOOL, helpersAwareTargetRequest);
applyWorkingDir(pythonExecution, workingDir);
pythonExecution.addParameter(INSTALL);
final boolean useUserSite = extraArgs.contains(USE_USER_SITE);
final String proxyString = getProxyString();
if (proxyString != null) {
pythonExecution.addParameter("--proxy");
pythonExecution.addParameter(proxyString);
}
pythonExecution.addParameters(extraArgs);
for (PyRequirement req : requirements) {
pythonExecution.addParameters(req.getInstallOptions());
}
try {
getPythonProcessResult(pythonExecution, !useUserSite, true, targetEnvironmentRequest);
}
catch (PyExecutionException e) {
final List<String> simplifiedArgs = new ArrayList<>();
simplifiedArgs.add("install");
if (proxyString != null) {
simplifiedArgs.add("--proxy");
simplifiedArgs.add(proxyString);
}
simplifiedArgs.addAll(extraArgs);
for (PyRequirement req : requirements) {
simplifiedArgs.addAll(req.getInstallOptions());
}
throw PyExecutionExceptionExtKt.copyWith(e, "pip", makeSafeToDisplayCommand(simplifiedArgs));
}
finally {
LOG.debug("Packages cache is about to be refreshed because these requirements were installed: " + requirements);
refreshPackagesSynchronously();
}
}
private static void applyWorkingDir(@NotNull PythonScriptExecution execution, @Nullable String workingDir) {
if (workingDir == null) {
// TODO [targets] Set the parent of home path as the working directory
@@ -361,174 +334,6 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager
}
@Override
public @Nullable List<PyPackage> getPackages() {
final List<PyPackage> packages = myPackagesCache;
return packages != null ? Collections.unmodifiableList(packages) : null;
}
protected @NotNull List<PyPackage> collectPackages() throws ExecutionException {
HelpersAwareTargetEnvironmentRequest helpersAwareRequest = getPythonTargetInterpreter();
TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareRequest.getTargetEnvironmentRequest();
final String output;
try {
LOG.debug("Collecting installed packages for the SDK " + getSdk().getName(), new Throwable());
PythonScriptExecution pythonExecution =
PythonScripts.prepareHelperScriptExecution(PythonHelper.PACKAGING_TOOL, helpersAwareRequest);
pythonExecution.addParameter("list");
output = getPythonProcessResult(pythonExecution, false, false, targetEnvironmentRequest);
}
catch (ProcessNotCreatedException ex) {
if (ApplicationManager.getApplication().isUnitTestMode()) {
LOG.info("Not-env unit test mode, will return mock packages");
return List.of(new PyPackage(PyPackageUtil.PIP, PIP_VERSION),
new PyPackage(PyPackageUtil.SETUPTOOLS, SETUPTOOLS_VERSION));
}
else {
throw ex;
}
}
return parsePackagingToolOutput(output);
}
@Override
public @NotNull String createVirtualEnv(@NotNull String destinationDir, boolean useGlobalSite) throws ExecutionException {
final Sdk sdk = getSdk();
final LanguageLevel languageLevel = getOrRequestLanguageLevelForSdk(sdk);
if (languageLevel.isOlderThan(LanguageLevel.PYTHON27)) {
throw new ExecutionException(PySdkBundle.message("python.sdk.packaging.creating.virtual.environment.for.python.not.supported",
languageLevel, LanguageLevel.PYTHON27));
}
HelpersAwareTargetEnvironmentRequest helpersAwareTargetRequest = getPythonTargetInterpreter();
TargetEnvironmentRequest targetEnvironmentRequest = helpersAwareTargetRequest.getTargetEnvironmentRequest();
PythonScriptExecution pythonExecution = PythonScripts.prepareHelperScriptExecution(
isLegacyPython(languageLevel) ? PythonHelper.LEGACY_VIRTUALENV_ZIPAPP : PythonHelper.VIRTUALENV_ZIPAPP,
helpersAwareTargetRequest);
if (useGlobalSite) {
pythonExecution.addParameter("--system-site-packages");
}
pythonExecution.addParameter(destinationDir);
// TODO [targets] Pass `parentDir = null`
getPythonProcessResult(pythonExecution, false, true, targetEnvironmentRequest);
final Path binary = VirtualEnvReader.getInstance().findPythonInPythonRoot(Path.of(destinationDir));
final char separator = targetEnvironmentRequest.getTargetPlatform().getPlatform().fileSeparator;
final String binaryFallback = destinationDir + separator + "bin" + separator + "python";
return (binary != null) ? binary.toString() : binaryFallback;
}
/**
* Is it a legacy python version that we still support
*/
private static @NotNull Boolean isLegacyPython(@NotNull LanguageLevel languageLevel) {
return languageLevel.isPython2() || languageLevel.isOlderThan(LanguageLevel.PYTHON37);
}
private @NotNull String getPythonProcessResult(@NotNull PythonExecution pythonExecution,
boolean askForSudo,
boolean showProgress,
@NotNull TargetEnvironmentRequest targetEnvironmentRequest) throws ExecutionException {
ProcessOutputWithCommandLine result = getPythonProcessOutput(pythonExecution, askForSudo, showProgress, targetEnvironmentRequest);
String path = result.getExePath();
List<String> args = result.getArgs();
ProcessOutput processOutput = result.getProcessOutput();
int exitCode = processOutput.getExitCode();
if (processOutput.isTimeout()) {
// TODO [targets] Make cancellable right away?
throw PyExecutionException.createForTimeout(PySdkBundle.message("python.sdk.packaging.timed.out"), path, args);
}
else if (exitCode != 0) {
throw new PyExecutionException(PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode), path, args, processOutput);
}
return processOutput.getStdout();
}
private @NotNull PyTargetEnvironmentPackageManager.ProcessOutputWithCommandLine getPythonProcessOutput(@NotNull PythonExecution pythonExecution,
boolean askForSudo,
boolean showProgress,
@NotNull TargetEnvironmentRequest targetEnvironmentRequest)
throws ExecutionException {
// TODO [targets] Use `showProgress = true`
// TODO [targets] Use `workingDir`
// TODO [targets] Use `useUserSite` (handle use sudo)
TargetProgressIndicator targetProgressIndicator = TargetProgressIndicator.EMPTY;
TargetEnvironment targetEnvironment = targetEnvironmentRequest.prepareEnvironment(targetProgressIndicator);
for (Map.Entry<TargetEnvironment.UploadRoot, TargetEnvironment.UploadableVolume> entry : targetEnvironment.getUploadVolumes()
.entrySet()) {
try {
entry.getValue().upload(".", TargetProgressIndicator.EMPTY);
}
catch (IOException e) {
throw new ExecutionException(e);
}
}
// TODO [targets] Should `interpreterParameters` be here?
TargetedCommandLine targetedCommandLine = PythonScripts.buildTargetedCommandLine(pythonExecution,
targetEnvironment,
getSdk(),
Collections.emptyList()
);
// TODO [targets] Set parent directory of interpreter as the working directory
LOG.info("Running packaging tool");
// TODO [targets] Apply environment variables: setPythonUnbuffered(...), setPythonDontWriteBytecode(...), resetHomePathChanges(...)
// TODO [targets] Apply flavor from PythonSdkFlavor.getFlavor(mySdk)
ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
Process process = createProcess(targetEnvironment, targetedCommandLine, askForSudo, indicator);
List<String> commandLine = targetedCommandLine.collectCommandsSynchronously();
String commandLineString = StringUtil.join(commandLine, " ");
final CapturingProcessHandler handler =
new CapturingProcessHandler(process, targetedCommandLine.getCharset(), commandLineString);
final ProcessOutput result;
if (showProgress && indicator != null) {
handler.addProcessListener(new IndicatedProcessOutputListener(indicator));
result = handler.runProcessWithProgressIndicator(indicator);
}
else {
// TODO [targets] Check if timeout is ok for all targets
result = handler.runProcess(TIMEOUT);
}
if (result.isCancelled()) {
throw new RunCanceledByUserException();
}
result.checkSuccess(LOG);
final int exitCode = result.getExitCode();
String helperPath = ContainerUtil.getFirstItem(commandLine, "");
List<String> args = commandLine.subList(Math.min(1, commandLine.size()), commandLine.size());
if (exitCode != 0) {
final String message = StringUtil.isEmptyOrSpaces(result.getStdout()) && StringUtil.isEmptyOrSpaces(result.getStderr())
? PySdkBundle.message("python.conda.permission.denied")
: PySdkBundle.message("python.sdk.packaging.non.zero.exit.code", exitCode);
throw new PyExecutionException(message, helperPath, args, result);
}
return new ProcessOutputWithCommandLine(helperPath, args, result);
}
private @NotNull Process createProcess(@NotNull TargetEnvironment targetEnvironment,
@NotNull TargetedCommandLine targetedCommandLine,
boolean askForSudo,
@Nullable ProgressIndicator indicator) throws ExecutionException {
if (askForSudo) {
if (!(targetEnvironment instanceof LocalTargetEnvironment)) {
// TODO [targets] Execute process on non-local target using sudo
LOG.warn("Sudo flag is ignored");
}
else if (PySdkExtKt.adminPermissionsNeeded(getSdk())) {
// This is hack to process sudo flag in the local environment
GeneralCommandLine localCommandLine = ((LocalTargetEnvironment)targetEnvironment).createGeneralCommandLine(targetedCommandLine);
return executeOnLocalMachineWithSudo(localCommandLine);
}
}
// TODO [targets] Pass meaningful progress indicator
return targetEnvironment.createProcess(targetedCommandLine, Objects.requireNonNullElseGet(indicator, EmptyProgressIndicator::new));
}
private static @NotNull Process executeOnLocalMachineWithSudo(@NotNull GeneralCommandLine localCommandLine) throws ExecutionException {
try {
return ExecUtil.sudo(localCommandLine, PySdkBundle.message("python.sdk.packaging.enter.your.password.to.make.changes"));
@@ -540,15 +345,6 @@ public abstract class PyTargetEnvironmentPackageManager extends PyPackageManager
}
}
private @NotNull HelpersAwareTargetEnvironmentRequest getPythonTargetInterpreter() throws ExecutionException {
HelpersAwareTargetEnvironmentRequest request = PythonInterpreterTargetEnvironmentFactory.findPythonTargetInterpreter(getSdk(),
ProjectManager.getInstance()
.getDefaultProject());
if (request == null) {
throw new ExecutionException(PySdkBundle.message("python.sdk.package.managing.not.supported.for.sdk", getSdk().getName()));
}
return request;
}
private static @NotNull HelperPackage getPipHelperPackage() {
return new PythonHelper.ScriptPythonHelper(PIP_WHEEL_NAME + "/" + PyPackageUtil.PIP,
@@ -22,17 +22,17 @@ import com.intellij.openapi.util.UserDataHolderBase
import com.intellij.openapi.util.io.FileUtil
import com.intellij.util.concurrency.annotations.RequiresEdt
import com.jetbrains.python.PySdkBundle
import com.jetbrains.python.packaging.PyPackageManager
import com.jetbrains.python.packaging.PyPackageManagers
import com.jetbrains.python.packaging.PyTargetEnvCreationManager
import com.jetbrains.python.run.PythonInterpreterTargetEnvironmentFactory
import com.jetbrains.python.sdk.*
import com.jetbrains.python.target.ui.PyAddSdkPanelBase
import com.jetbrains.python.target.ui.PyAddSdkPanelBase.Companion.isLocal
import com.jetbrains.python.target.ui.TargetPanelExtension
import com.jetbrains.python.sdk.flavors.PyFlavorAndData
import com.jetbrains.python.sdk.flavors.PyFlavorData
import com.jetbrains.python.target.PyTargetAwareAdditionalData
import com.jetbrains.python.target.getInterpreterVersion
import com.jetbrains.python.target.ui.PyAddSdkPanelBase
import com.jetbrains.python.target.ui.PyAddSdkPanelBase.Companion.isLocal
import com.jetbrains.python.target.ui.TargetPanelExtension
import com.jetbrains.python.ui.pyModalBlocking
import org.jetbrains.annotations.ApiStatus
@@ -75,8 +75,7 @@ fun createVirtualEnvAndSdkSynchronously(
}
try {
val packageManager = PyPackageManager.getInstance(sdk)
return packageManager.createVirtualEnv(venvRoot, inheritSitePackages)
return PyTargetEnvCreationManager(sdk).createVirtualEnv(venvRoot, inheritSitePackages)
}
finally {
PyPackageManagers.getInstance().clearCache(sdk)
@@ -12,7 +12,7 @@ import com.jetbrains.env.PyExecutionFixtureTestTask;
import com.jetbrains.env.PyTestTask;
import com.jetbrains.python.packaging.PyPackage;
import com.jetbrains.python.packaging.PyPackageManager;
import com.jetbrains.python.packaging.requirement.PyRequirementRelation;
import com.jetbrains.python.packaging.PyTargetEnvCreationManager;
import com.jetbrains.python.sdk.flavors.PythonSdkFlavor;
import com.jetbrains.python.sdk.flavors.VirtualEnvSdkFlavor;
import com.jetbrains.python.tools.sdkTools.SdkCreationType;
@@ -23,14 +23,11 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import static com.intellij.testFramework.UsefulTestCase.assertInstanceOf;
import static com.jetbrains.python.SdkUiUtilKt.isVirtualEnv;
import static com.jetbrains.python.packaging.PyRequirementsKt.pyRequirement;
import static org.junit.Assert.*;
public class PyPackagingTest extends PyEnvTestCase {
@@ -72,7 +69,8 @@ public class PyPackagingTest extends PyEnvTestCase {
try {
final File tempDir = FileUtil.createTempDirectory(getTestName(false), null);
final File venvDir = new File(tempDir, "venv");
final String venvSdkHome = PyPackageManager.getInstance(sdk).createVirtualEnv(venvDir.toString(),
PyTargetEnvCreationManager manager = new PyTargetEnvCreationManager(sdk);
final String venvSdkHome = manager.createVirtualEnv(venvDir.toString(),
false);
final Sdk venvSdk = createTempSdk(venvSdkHome, SdkCreationType.EMPTY_SDK);
assertNotNull(venvSdk);
@@ -96,40 +94,6 @@ public class PyPackagingTest extends PyEnvTestCase {
});
}
@Test
public void testInstallPackage() {
runPythonTest(new PyPackagingTestTask() {
@Override
public void runTestOn(@NotNull String sdkHome, @Nullable Sdk existingSdk) throws Exception {
final Sdk sdk = createTempSdk(sdkHome, SdkCreationType.EMPTY_SDK);
try {
final File tempDir = FileUtil.createTempDirectory(getTestName(false), null);
final File venvDir = new File(tempDir, "venv");
final String venvSdkHome = PyPackageManager.getInstance(sdk).createVirtualEnv(venvDir.getPath(), false);
final Sdk venvSdk = createTempSdk(venvSdkHome, SdkCreationType.EMPTY_SDK);
assertNotNull(venvSdk);
final PyPackageManager manager = PyPackageManager.getInstance(venvSdk);
final List<PyPackage> packages1 = manager.refreshAndGetPackages(false);
// TODO: Install Markdown from a local file
manager.install(Arrays.asList(pyRequirement("Markdown", PyRequirementRelation.LTE, "3.3.3"), pyRequirement("httplib2",null)),
Collections.emptyList());
final List<PyPackage> packages2 = manager.refreshAndGetPackages(false);
final PyPackage markdown2 = findPackage("markdown", packages2);
assertNotNull(markdown2);
final PyPackage pip1 = findPackage("pip", packages1);
assertNotNull(pip1);
assertEquals("pip", pip1.getName());
}
catch (ExecutionException e) {
throw new RuntimeException(String.format("Error for interpreter '%s': %s", sdk.getHomePath(), e.getMessage()), e);
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
});
}
@Nullable
private static PyPackage findPackage(String name, List<PyPackage> packages) {