diff --git a/platform/lang-impl/src/com/intellij/webcore/packaging/InstalledPackagesPanel.java b/platform/lang-impl/src/com/intellij/webcore/packaging/InstalledPackagesPanel.java index 7a1cd6ce9a13..3ae1c810c0ae 100644 --- a/platform/lang-impl/src/com/intellij/webcore/packaging/InstalledPackagesPanel.java +++ b/platform/lang-impl/src/com/intellij/webcore/packaging/InstalledPackagesPanel.java @@ -278,7 +278,7 @@ public class InstalledPackagesPanel extends JPanel { return ObjectUtils.tryCast(myPackageManagementService, PackageManagementServiceEx.class); } - private void updateUninstallUpgrade() { + protected void updateUninstallUpgrade() { final int[] selected = myPackagesTable.getSelectedRows(); boolean upgradeAvailable = false; boolean canUninstall = selected.length != 0; diff --git a/platform/lang-impl/src/com/intellij/webcore/packaging/ManagePackagesDialog.java b/platform/lang-impl/src/com/intellij/webcore/packaging/ManagePackagesDialog.java index 5aef94eba8e1..0bb6051cee36 100644 --- a/platform/lang-impl/src/com/intellij/webcore/packaging/ManagePackagesDialog.java +++ b/platform/lang-impl/src/com/intellij/webcore/packaging/ManagePackagesDialog.java @@ -171,7 +171,7 @@ public class ManagePackagesDialog extends DialogWrapper { } private void addManageAction() { - if (myController.getAllRepositories() != null) { + if (myController.canManageRepositories()) { myManageButton.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent event) { diff --git a/platform/lang-impl/src/com/intellij/webcore/packaging/ManageRepoDialog.java b/platform/lang-impl/src/com/intellij/webcore/packaging/ManageRepoDialog.java index 441b6e2e3b3a..f31f9eb4cdfe 100644 --- a/platform/lang-impl/src/com/intellij/webcore/packaging/ManageRepoDialog.java +++ b/platform/lang-impl/src/com/intellij/webcore/packaging/ManageRepoDialog.java @@ -15,6 +15,9 @@ */ package com.intellij.webcore.packaging; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.application.ModalityState; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.project.Project; import com.intellij.openapi.ui.DialogWrapper; import com.intellij.openapi.ui.InputValidator; @@ -22,30 +25,52 @@ import com.intellij.openapi.ui.Messages; import com.intellij.openapi.util.text.StringUtil; import com.intellij.ui.ToolbarDecorator; import com.intellij.ui.components.JBList; +import com.intellij.util.CatchingConsumer; import com.intellij.util.ui.JBUI; import javax.swing.*; +import java.util.List; public class ManageRepoDialog extends DialogWrapper { private JPanel myMainPanel; - private final JBList myList; + private final JBList myList; private boolean myEnabled; + private static final Logger LOG = Logger.getInstance(ManageRepoDialog.class); public ManageRepoDialog(Project project, final PackageManagementService controller) { super(project, false); init(); setTitle("Manage Repositories"); - final DefaultListModel repoModel = new DefaultListModel(); - for(String repoUrl: controller.getAllRepositories()) { - repoModel.addElement(repoUrl); - } - myList = new JBList(); + myList = new JBList<>(); + myList.setPaintBusy(true); + final DefaultListModel repoModel = new DefaultListModel<>(); + controller.fetchAllRepositories(new CatchingConsumer, Exception>() { + @Override + public void consume(List repoUrls) { + ApplicationManager.getApplication().invokeLater(() -> { + if (isDisposed()) return; + myList.setPaintBusy(false); + for (String repoUrl: repoUrls) { + repoModel.addElement(repoUrl); + } + }, ModalityState.any()); + } + + @Override + public void consume(Exception e) { + ApplicationManager.getApplication().invokeLater(() -> { + if (isDisposed()) return; + myList.setPaintBusy(false); + LOG.warn(e); + }); + } + }); myList.setModel(repoModel); myList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION); myList.addListSelectionListener(event -> { - final Object selected = myList.getSelectedValue(); - myEnabled = controller.canModifyRepository((String) selected); + final String selected = myList.getSelectedValue(); + myEnabled = controller.canModifyRepository(selected); }); final ToolbarDecorator decorator = ToolbarDecorator.createDecorator(myList).disableUpDownActions(); @@ -61,7 +86,7 @@ public class ManageRepoDialog extends DialogWrapper { } }); decorator.setEditAction(button -> { - final String oldValue = (String)myList.getSelectedValue(); + final String oldValue = myList.getSelectedValue(); String url = Messages.showInputDialog("Please edit repository URL", "Repository URL", null, oldValue, new InputValidator() { @Override @@ -82,7 +107,7 @@ public class ManageRepoDialog extends DialogWrapper { } }); decorator.setRemoveAction(button -> { - String selected = (String)myList.getSelectedValue(); + String selected = myList.getSelectedValue(); controller.removeRepository(selected); repoModel.removeElement(selected); button.setEnabled(false); diff --git a/platform/lang-impl/src/com/intellij/webcore/packaging/PackageManagementService.java b/platform/lang-impl/src/com/intellij/webcore/packaging/PackageManagementService.java index a17edb4d5d56..aa78ebd9fedb 100644 --- a/platform/lang-impl/src/com/intellij/webcore/packaging/PackageManagementService.java +++ b/platform/lang-impl/src/com/intellij/webcore/packaging/PackageManagementService.java @@ -23,6 +23,20 @@ public abstract class PackageManagementService { return null; } + /** + * An async version of {@link #getAllRepositories()}. + */ + public void fetchAllRepositories(@NotNull CatchingConsumer, ? super Exception> consumer) { + consumer.consume(getAllRepositories()); + } + + /** + * Returns true if the service supports managing repositories. + */ + public boolean canManageRepositories() { + return getAllRepositories() != null; + } + /** * Checks if the user can change the URL of the specified repository or remove it from the list. * diff --git a/python/helpers/conda_packaging_tool.py b/python/helpers/conda_packaging_tool.py index b225b6bcd726..5dcfc386ac58 100644 --- a/python/helpers/conda_packaging_tool.py +++ b/python/helpers/conda_packaging_tool.py @@ -4,11 +4,13 @@ import traceback ERROR_WRONG_USAGE = 1 ERROR_EXCEPTION = 4 + def usage(): - sys.stderr.write('Usage: conda_packaging_tool.py \n') + sys.stderr.write('Usage: conda_packaging_tool.py listall | channels | versions PACKAGE\n') sys.stderr.flush() exit(ERROR_WRONG_USAGE) + def do_list_available_packages(): import conda version = conda.__version__ @@ -23,6 +25,7 @@ def do_list_available_packages(): minor_version = int(version_splitted[1]) if major_version >= 4 and minor_version >= 4: + init_context() from conda.core.index import get_index index = get_index() elif major_version == 4 and minor_version >= 2: @@ -36,20 +39,52 @@ def do_list_available_packages(): index = common.get_index_trap() for pkg in index.values(): - sys.stdout.write("\t".join([pkg["name"], pkg["version"], ":".join(pkg["depends"])])+chr(10)) + sys.stdout.write("\t".join([pkg["name"], pkg["version"], ":".join(pkg["depends"])]) + chr(10)) sys.stdout.flush() def do_list_channels(): - import conda.config as config - if hasattr(config, "get_channel_urls"): - channels = config.get_channel_urls() + context = init_context() + if context: + channels = context.channels else: - channels = config.context.channels - for channel in channels: - if channel != 'defaults': - sys.stdout.write(channel+chr(10)) - sys.stdout.flush() + import conda.config as config + if hasattr(config, "get_channel_urls"): + channels = config.get_channel_urls() + else: + channels = config.context.channels + sys.stdout.write('\n'.join(channels)) + sys.stdout.write('\n') + sys.stdout.flush() + + +def fetch_versions(package): + import json + from distutils.version import LooseVersion + from conda.cli.python_api import run_command, Commands + + stdout, stderr, ret_code = run_command(Commands.SEARCH, package, '--json') + if ret_code != 0: + raise Exception(stderr) + results = json.loads(stdout) + results = results.get(package, []) + all_versions = (r.get('version') for r in results) + return sorted(set(v for v in all_versions if v), key=LooseVersion, reverse=True) + + +def do_list_versions(package): + sys.stdout.write('\n'.join(fetch_versions(package))) + sys.stderr.write('\n') + sys.stdout.flush() + + +def init_context(): + try: + from conda.base.context import context + except ImportError: + return None + context.__init__() + return context def main(): @@ -68,6 +103,11 @@ def main(): usage() return do_list_channels() + elif cmd == 'versions': + if len(sys.argv) != 3: + usage() + return + do_list_versions(sys.argv[2]) else: usage() except Exception: @@ -75,5 +115,6 @@ def main(): exit(ERROR_EXCEPTION) exit(retcode) + if __name__ == '__main__': main() diff --git a/python/ide/src/com/jetbrains/python/PythonSdkConfigurator.kt b/python/ide/src/com/jetbrains/python/PythonSdkConfigurator.kt index ad90d15b60e3..675f8fd0cd19 100644 --- a/python/ide/src/com/jetbrains/python/PythonSdkConfigurator.kt +++ b/python/ide/src/com/jetbrains/python/PythonSdkConfigurator.kt @@ -1,8 +1,12 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.jetbrains.python +import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.module.Module import com.intellij.openapi.module.ModuleManager +import com.intellij.openapi.progress.ProgressIndicator +import com.intellij.openapi.progress.ProgressManager +import com.intellij.openapi.progress.Task import com.intellij.openapi.project.Project import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil @@ -27,13 +31,22 @@ class PythonSdkConfigurator : DirectoryProjectConfigurator { } fun findDetectedAssociatedEnvironment(module: Module, existingSdks: List): PyDetectedSdk? { - detectVirtualEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let { - return it - } - detectCondaEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let { - return it - } - return null + // TODO: Move all interpreter detection away from EDT & use proper synchronization for that + val progress = ProgressManager.getInstance() + return progress.run(object : Task.WithResult(module.project, + "Looking for Virtual Environments", + false) { + override fun compute(indicator: ProgressIndicator): PyDetectedSdk? { + indicator.isIndeterminate = true + detectVirtualEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let { + return it + } + detectCondaEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let { + return it + } + return null + } + }) } private fun findExistingSystemWideSdk(existingSdks: List) = diff --git a/python/src/com/jetbrains/python/packaging/PyCondaPackageCache.java b/python/src/com/jetbrains/python/packaging/PyCondaPackageCache.java deleted file mode 100644 index 0dfad04da68b..000000000000 --- a/python/src/com/jetbrains/python/packaging/PyCondaPackageCache.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -package com.jetbrains.python.packaging; - -import com.google.common.collect.Multimap; -import org.jetbrains.annotations.NotNull; - -import java.util.ArrayList; - -/** - * @author Mikhail Golubev - */ -public class PyCondaPackageCache extends PyAbstractPackageCache { - private static final String CACHE_FILE_NAME = "conda-cache.json"; - - private static PyCondaPackageCache ourInstance; - - @NotNull - public static synchronized PyCondaPackageCache getInstance() { - if (ourInstance == null) { - ourInstance = load(PyCondaPackageCache.class, new PyCondaPackageCache(), getDefaultCachePath(CACHE_FILE_NAME)); - } - return ourInstance; - } - - @NotNull - public static synchronized PyCondaPackageCache reload(@NotNull Multimap nameToVersion) { - ourInstance = new PyCondaPackageCache(nameToVersion); - store(ourInstance, CACHE_FILE_NAME); - return ourInstance; - } - - private PyCondaPackageCache() { - } - - private PyCondaPackageCache(@NotNull Multimap nameToVersion) { - for (String name : nameToVersion.keySet()) { - myPackages.put(name, new PyAbstractPackageCache.PackageInfo(new ArrayList<>(nameToVersion.get(name)))); - } - } -} diff --git a/python/src/com/jetbrains/python/packaging/PyCondaPackageManagerImpl.java b/python/src/com/jetbrains/python/packaging/PyCondaPackageManagerImpl.java index 2aaa4ddf06ff..1e0add2940e5 100644 --- a/python/src/com/jetbrains/python/packaging/PyCondaPackageManagerImpl.java +++ b/python/src/com/jetbrains/python/packaging/PyCondaPackageManagerImpl.java @@ -145,6 +145,11 @@ public class PyCondaPackageManagerImpl extends PyPackageManagerImpl { } } + @Override + public boolean hasManagement() throws ExecutionException { + return useConda || super.hasManagement(); + } + @NotNull private List parseCondaToolOutput(@NotNull String s) throws ExecutionException { final String[] lines = StringUtil.splitByLines(s); diff --git a/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java b/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java index d0951e7b4372..4ae88777279b 100644 --- a/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java +++ b/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java @@ -3,6 +3,7 @@ package com.jetbrains.python.packaging; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; +import com.intellij.execution.ExecutionException; import com.intellij.execution.configurations.PathEnvironmentVariableUtil; import com.intellij.execution.process.ProcessOutput; import com.intellij.openapi.components.*; @@ -13,7 +14,6 @@ import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.StandardFileSystems; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.SystemProperties; -import com.intellij.util.containers.ContainerUtil; import com.intellij.util.text.VersionComparatorUtil; import com.intellij.util.xmlb.XmlSerializerUtil; import com.jetbrains.python.PythonHelpersLocator; @@ -30,8 +30,6 @@ import java.util.*; @State(name = "PyCondaPackageService", storages = @Storage(value="conda_packages.xml", roamingType = RoamingType.DISABLED)) public class PyCondaPackageService implements PersistentStateComponent { private static final Logger LOG = Logger.getInstance(PyCondaPackageService.class); - public Set CONDA_CHANNELS = ContainerUtil.newConcurrentSet(); - public long LAST_TIME_CHECKED = 0; @Nullable @SystemDependent public String PREFERRED_CONDA_PATH = null; @Override @@ -48,27 +46,6 @@ public class PyCondaPackageService implements PersistentStateComponent loadAndGetChannels() { - if (CONDA_CHANNELS.isEmpty()) { - updateChannels(); - } - return CONDA_CHANNELS; - } - - public void addChannel(@NotNull final String url) { - CONDA_CHANNELS.add(url); - } - - public void removeChannel(@NotNull final String url) { - CONDA_CHANNELS.remove(url); - } - @Nullable public static String getCondaPython() { final String conda = getSystemCondaExecutable(); @@ -107,14 +84,11 @@ public class PyCondaPackageService implements PersistentStateComponent listAllPackagesAndVersions() { try { - output = PyCondaRunKt.runCondaPython(condaPython, Arrays.asList(path, "listall")); + final String output = runCondaPackagingHelper("listall"); + final Multimap nameToVersions = + Multimaps.newSortedSetMultimap(new HashMap<>(), () -> new TreeSet<>(VersionComparatorUtil.COMPARATOR.reversed())); + for (String line : StringUtil.split(output, "\n")) { + final List split = StringUtil.split(line, "\t"); + if (split.size() < 2) continue; + nameToVersions.put(split.get(0), split.get(1)); + } + return nameToVersions; } - catch (PyExecutionException e) { + catch (ExecutionException e) { LOG.warn("Failed to get list of conda packages. " + e); - return; + return null; } - - final Multimap nameToVersions = - Multimaps.newSortedSetMultimap(new HashMap<>(), () -> new TreeSet<>(VersionComparatorUtil.COMPARATOR.reversed())); - for (String line : output.getStdoutLines()) { - final List split = StringUtil.split(line, "\t"); - if (split.size() < 2) continue; - nameToVersions.put(split.get(0), split.get(1)); - } - PyCondaPackageCache.reload(nameToVersions); - LAST_TIME_CHECKED = System.currentTimeMillis(); } @NotNull - public List getPackageVersions(@NotNull final String packageName) { - return ContainerUtil.notNullize(PyCondaPackageCache.getInstance().getVersions(packageName)); + public List listPackageVersions(@NotNull String packageName) throws ExecutionException { + final String output = runCondaPackagingHelper("versions", packageName); + return StringUtil.split(output, "\n"); } - public void updateChannels() { + @Nullable + public List listChannels() throws ExecutionException { + final String output = runCondaPackagingHelper("channels"); + return StringUtil.split(output, "\n"); + } + + @NotNull + private static String runCondaPackagingHelper(@NotNull String... args) throws ExecutionException { + final List commandArgs = new ArrayList<>(); + commandArgs.add(PythonHelpersLocator.getHelperPath("conda_packaging_tool.py")); + commandArgs.addAll(Arrays.asList(args)); final String condaPython = getCondaPython(); - if (condaPython == null) return; - final String path = PythonHelpersLocator.getHelperPath("conda_packaging_tool.py"); - final ProcessOutput output; - try { - output = PyCondaRunKt.runCondaPython(condaPython, Arrays.asList(path, "channels")); + if (condaPython == null) { + throw new PyExecutionException("Cannot find Python executable for conda", + "python", commandArgs, new ProcessOutput()); } - catch (PyExecutionException e) { - LOG.warn("Failed to update conda channels. " + e); - return; - } - final List lines = output.getStdoutLines(); - CONDA_CHANNELS.addAll(lines); - LAST_TIME_CHECKED = System.currentTimeMillis(); + final ProcessOutput output = PyCondaRunKt.runCondaPython(condaPython, commandArgs); + return output.getStdout(); } } diff --git a/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java b/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java index 57dce76cd81b..013ed02794f8 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java +++ b/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java @@ -53,17 +53,6 @@ public class PyPackagesUpdater implements StartupActivity { } }); } - if (checkCondaUpdateNeeded(project)) { - application.executeOnPooledThread(() -> PyCondaPackageService.getInstance().updatePackagesCache()); - } - } - - private static boolean checkCondaUpdateNeeded(Project project) { - if (!hasPython(project)) return false; - final long timeDelta = System.currentTimeMillis() - PyCondaPackageService.getInstance().LAST_TIME_CHECKED; - if (Math.abs(timeDelta) < EXPIRATION_TIMEOUT) return false; - LOG.debug("Updating outdated Conda package cache"); - return true; } private static boolean hasPython(Project project) { diff --git a/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java b/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java index c0a0b76b86cd..1e922f1b8a4d 100644 --- a/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java +++ b/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java @@ -16,15 +16,18 @@ package com.jetbrains.python.packaging.ui; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import com.intellij.execution.ExecutionException; import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.progress.ProgressIndicator; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.progress.Task; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.util.CatchingConsumer; import com.intellij.util.containers.ContainerUtil; import com.intellij.webcore.packaging.InstalledPackage; import com.intellij.webcore.packaging.RepoPackage; -import com.jetbrains.python.packaging.PyCondaPackageCache; import com.jetbrains.python.packaging.PyCondaPackageManagerImpl; import com.jetbrains.python.packaging.PyCondaPackageService; import com.jetbrains.python.packaging.PyPackageManager; @@ -32,6 +35,8 @@ import com.jetbrains.python.sdk.flavors.PyCondaRunKt; import org.jetbrains.annotations.NotNull; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class PyCondaManagementService extends PyPackageManagementService { @@ -49,45 +54,77 @@ public class PyCondaManagementService extends PyPackageManagementService { @NotNull public List getAllPackagesCached() { if (useConda()) { - return getCachedCondaPackages(); + return Collections.emptyList(); + } + else { + return super.getAllPackagesCached(); } - return super.getAllPackagesCached(); } @Override @NotNull public List getAllPackages() throws IOException { if (useConda()) { - PyCondaPackageService.getInstance().loadAndGetPackages(false); - return getAllPackagesCached(); + return reloadAllPackages(); + } + else { + return super.getAllPackages(); } - return super.getAllPackages(); } @Override @NotNull public List reloadAllPackages() throws IOException { if (useConda()) { - PyCondaPackageService.getInstance().loadAndGetPackages(true); - return getAllPackagesCached(); + final Multimap packages = PyCondaPackageService.getInstance().listAllPackagesAndVersions(); + if (packages == null) return Collections.emptyList(); + final List results = new ArrayList<>(); + for (String pkg : packages.keySet()) { + results.add(new RepoPackage(pkg, null, ContainerUtil.getFirstItem(packages.get(pkg)))); + } + return results; } return super.reloadAllPackages(); } @Override - public List getAllRepositories() { - return useConda() ? Lists.newArrayList(PyCondaPackageService.getInstance().loadAndGetChannels()) : super.getAllRepositories(); + public boolean canManageRepositories() { + return true; + } + + @Override + public void fetchAllRepositories(@NotNull CatchingConsumer, ? super Exception> consumer) { + if (useConda()) { + myExecutorService.submit(() -> { + try { + final List channels = ContainerUtil.notNullize(PyCondaPackageService.getInstance().listChannels()); + consumer.consume(channels); + } + catch (ExecutionException e) { + consumer.consume(e); + } + }); + } + else { + super.fetchAllRepositories(consumer); + } } @Override public void addRepository(String repositoryUrl) { if (useConda()) { - try { - PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--add", "channels", repositoryUrl, "--force")); - } - catch (ExecutionException e) { - LOG.warn("Failed to add repository. " + e); - } + ProgressManager.getInstance().run(new Task.Modal(getProject(), "Adding Conda Channel", true) { + @Override + public void run(@NotNull ProgressIndicator indicator) { + indicator.setIndeterminate(true); + try { + PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--add", "channels", repositoryUrl, "--force")); + } + catch (ExecutionException e) { + LOG.warn("Failed to add repository. " + e); + } + } + }); } else { super.addRepository(repositoryUrl); @@ -97,12 +134,18 @@ public class PyCondaManagementService extends PyPackageManagementService { @Override public void removeRepository(String repositoryUrl) { if (useConda()) { - try { - PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--remove", "channels", repositoryUrl, "--force")); - } - catch (ExecutionException e) { - LOG.warn("Failed to remove repository. " + e); - } + ProgressManager.getInstance().run(new Task.Modal(getProject(), "Removing Conda Channel", true) { + @Override + public void run(@NotNull ProgressIndicator indicator) { + indicator.setIndeterminate(true); + try { + PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--remove", "channels", repositoryUrl, "--force")); + } + catch (ExecutionException e) { + LOG.warn("Failed to remove repository. " + e); + } + } + }); } else { super.removeRepository(repositoryUrl); @@ -117,7 +160,14 @@ public class PyCondaManagementService extends PyPackageManagementService { @Override public void fetchPackageVersions(String packageName, CatchingConsumer, Exception> consumer) { if (useConda()) { - consumer.consume(PyCondaPackageService.getInstance().getPackageVersions(packageName)); + myExecutorService.submit(() -> { + try { + consumer.consume(PyCondaPackageService.getInstance().listPackageVersions(packageName)); + } + catch (ExecutionException e) { + LOG.warn("Failed to fetch versions for '" + packageName + "'. " + e); + } + }); } else { super.fetchPackageVersions(packageName, consumer); @@ -128,20 +178,23 @@ public class PyCondaManagementService extends PyPackageManagementService { public void fetchLatestVersion(@NotNull InstalledPackage pkg, @NotNull CatchingConsumer consumer) { final String packageName = pkg.getName(); if (useConda()) { - final String latestVersion = ContainerUtil.getFirstItem(PyCondaPackageCache.getInstance().getVersions(packageName)); - consumer.consume(latestVersion); + myExecutorService.submit(() -> { + try { + final String latestVersion = ContainerUtil.getFirstItem(PyCondaPackageService.getInstance().listPackageVersions(packageName)); + consumer.consume(latestVersion); + } + catch (ExecutionException e) { + LOG.warn("Failed to fetch versions for '" + packageName + "'. " + e); + } + }); } else { super.fetchLatestVersion(pkg, consumer); } } - @NotNull - private static List getCachedCondaPackages() { - final PyCondaPackageCache instance = PyCondaPackageCache.getInstance(); - return ContainerUtil.map(instance.getPackageNames(), name -> { - final String latestVersion = ContainerUtil.getFirstItem(instance.getVersions(name)); - return new RepoPackage(name, null, latestVersion); - }); + @Override + public boolean shouldFetchLatestVersionsForOnlyInstalledPackages() { + return false; } } diff --git a/python/src/com/jetbrains/python/packaging/ui/PyInstalledPackagesPanel.java b/python/src/com/jetbrains/python/packaging/ui/PyInstalledPackagesPanel.java index a40b3866da73..5800538d89e9 100644 --- a/python/src/com/jetbrains/python/packaging/ui/PyInstalledPackagesPanel.java +++ b/python/src/com/jetbrains/python/packaging/ui/PyInstalledPackagesPanel.java @@ -45,7 +45,7 @@ import java.util.Set; * @author yole */ public class PyInstalledPackagesPanel extends InstalledPackagesPanel { - private boolean myHasManagement = false; + private volatile boolean myHasManagement = false; public PyInstalledPackagesPanel(@NotNull Project project, @NotNull PackagesNotificationPanel area) { super(project, area); @@ -98,6 +98,7 @@ public class PyInstalledPackagesPanel extends InstalledPackagesPanel { PyExecutionException exception = null; try { myHasManagement = PyPackageManager.getInstance(selectedSdk).hasManagement(); + application.invokeLater(() -> updateUninstallUpgrade(), ModalityState.any()); if (!myHasManagement) { throw new PyExecutionException("Python packaging tools not found", "pip", Collections.emptyList(), "", "", 0, ImmutableList.of(new PyInstallPackageManagementFix())); diff --git a/python/src/com/jetbrains/python/sdk/add/PyAddExistingCondaEnvPanel.kt b/python/src/com/jetbrains/python/sdk/add/PyAddExistingCondaEnvPanel.kt index aae1e0a1f934..edb478ba26aa 100644 --- a/python/src/com/jetbrains/python/sdk/add/PyAddExistingCondaEnvPanel.kt +++ b/python/src/com/jetbrains/python/sdk/add/PyAddExistingCondaEnvPanel.kt @@ -15,16 +15,26 @@ */ package com.jetbrains.python.sdk.add +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.ModalityState import com.intellij.openapi.module.Module import com.intellij.openapi.project.Project import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.ui.ValidationInfo import com.intellij.ui.components.JBCheckBox +import com.intellij.util.ui.AsyncProcessIcon import com.intellij.util.ui.FormBuilder import com.jetbrains.python.sdk.* +import com.sun.glass.ui.Application import icons.PythonIcons +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking import java.awt.BorderLayout import javax.swing.Icon +import javax.swing.SwingUtilities +import javax.swing.SwingWorker /** * @author vlan @@ -34,10 +44,8 @@ class PyAddExistingCondaEnvPanel(private val project: Project?, private val existingSdks: List, override var newProjectPath: String?) : PyAddSdkPanel() { override val panelName: String = "Existing environment" - override val icon: Icon = PythonIcons.Python.Condaenv - private val sdkComboBox = PySdkPathChoosingComboBox(detectCondaEnvs(module, existingSdks) - .filterNot { it.isAssociatedWithAnotherModule(module) }, - null) + override val icon: Icon = PythonIcons.Python.Anaconda + private val sdkComboBox = PySdkPathChoosingComboBox(listOf(), null) private val makeSharedField = JBCheckBox("Make available to all projects") init { @@ -47,6 +55,17 @@ class PyAddExistingCondaEnvPanel(private val project: Project?, .addComponent(makeSharedField) .panel add(formPanel, BorderLayout.NORTH) + ApplicationManager.getApplication().executeOnPooledThread(object: Runnable { + override fun run() { + if (module != null && module.isDisposed) return + val sdks = detectCondaEnvs(module, existingSdks) + ApplicationManager.getApplication().invokeLater({ + sdks.forEach { + sdkComboBox.childComponent.addItem(it) + } + }, ModalityState.any()) + } + }) } override fun validateAll(): List = listOfNotNull(validateSdkComboBox(sdkComboBox)) diff --git a/python/src/com/jetbrains/python/sdk/flavors/CondaEnvSdkFlavor.java b/python/src/com/jetbrains/python/sdk/flavors/CondaEnvSdkFlavor.java index c27361be8c24..e949db080387 100644 --- a/python/src/com/jetbrains/python/sdk/flavors/CondaEnvSdkFlavor.java +++ b/python/src/com/jetbrains/python/sdk/flavors/CondaEnvSdkFlavor.java @@ -15,26 +15,26 @@ */ package com.jetbrains.python.sdk.flavors; +import com.intellij.execution.ExecutionException; +import com.intellij.openapi.application.ReadAction; import com.intellij.openapi.module.Module; -import com.intellij.openapi.util.SystemInfo; -import com.intellij.openapi.vfs.LocalFileSystem; +import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.vfs.StandardFileSystems; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.util.SystemProperties; -import com.jetbrains.python.packaging.PyCondaPackageService; import com.jetbrains.python.sdk.PythonSdkType; import icons.PythonIcons; +import one.util.streamex.StreamEx; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jetbrains.annotations.SystemDependent; import javax.swing.*; import java.io.File; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; -import static com.jetbrains.python.sdk.flavors.VirtualEnvSdkFlavor.findInDirectory; +import static com.jetbrains.python.sdk.flavors.VirtualEnvSdkFlavor.findInRootDirectory; public class CondaEnvSdkFlavor extends CPythonSdkFlavor { private CondaEnvSdkFlavor() { @@ -47,80 +47,29 @@ public class CondaEnvSdkFlavor extends CPythonSdkFlavor { @Override public Collection suggestHomePaths(@Nullable Module module) { - List candidates = new ArrayList<>(); - - for (VirtualFile file : getCondaDefaultLocations()) { - candidates.addAll(findInDirectory(file)); - } - - return candidates; - } - - public static List getCondaDefaultLocations() { - List roots = new ArrayList<>(); - final VirtualFile userHome = LocalFileSystem.getInstance().findFileByPath(SystemProperties.getUserHome().replace('\\','/')); - if (userHome != null) { - final VirtualFile condaHidden = userHome.findChild(".conda"); - if (condaHidden != null) { - addEnvsFolder(roots, condaHidden); - } - for (String root : CONDA_DEFAULT_ROOTS) { - VirtualFile condaFolder = userHome.findChild(root); - addEnvsFolder(roots, condaFolder); - if (SystemInfo.isWindows) { - final VirtualFile appData = userHome.findFileByRelativePath("AppData\\Local\\Continuum\\" + root); - addEnvsFolder(roots, appData); - condaFolder = LocalFileSystem.getInstance().findFileByPath("C:\\" + root); - } - else { - final String systemWidePath = "/opt/anaconda"; - condaFolder = LocalFileSystem.getInstance().findFileByPath(systemWidePath); - } - addEnvsFolder(roots, condaFolder); + final List results = new ArrayList<>(); + final Sdk sdk = ReadAction.compute(() -> PythonSdkType.findPythonSdk(module)); + try { + final List environments = PyCondaRunKt.listCondaEnvironments(sdk); + for (String environment : environments) { + results.addAll(ReadAction.compute(() -> { + final VirtualFile root = StandardFileSystems.local().findFileByPath(environment); + return StreamEx.of(findInRootDirectory(root)) + .filter(s -> getCondaEnvRoot(s) != null) + .toList(); + })); } } - addEnvsFolder(roots, findPreferredCondaEnvsFolder()); - return roots; - } - - @Nullable - private static VirtualFile findPreferredCondaEnvsFolder() { - @SystemDependent final String path = PyCondaPackageService.getInstance().PREFERRED_CONDA_PATH; - if (path == null) return null; - final VirtualFile conda = StandardFileSystems.local().findFileByPath(path); - if (conda == null) return null; - final VirtualFile binFolder = conda.getParent(); - if (binFolder == null) return null; - return binFolder.getParent(); - } - - private static void addEnvsFolder(@NotNull final List roots, @Nullable final VirtualFile condaFolder) { - if (condaFolder != null) { - final VirtualFile envs = condaFolder.findChild("envs"); - if (envs != null) { - roots.add(envs); - } + catch (ExecutionException e) { + return Collections.emptyList(); } + return results; } @Override public boolean isValidSdkPath(@NotNull File file) { if (!super.isValidSdkPath(file)) return false; - final File bin = file.getParentFile(); - String condaName = "conda"; - if (SystemInfo.isWindows) { - condaName = new File(bin, "envs").exists() ? "conda.exe" : "conda.bat"; - } - if (bin != null) { - final File conda = new File(bin, condaName); - if (conda.exists()) { - return true; - } - final File condaFolder = bin.getParentFile(); - final File condaExecutable = PythonSdkType.findExecutableFile(condaFolder, condaName); - if (condaExecutable != null) return true; - } - return false; + return PythonSdkType.isConda(file.getPath()); } @Nullable @@ -137,6 +86,6 @@ public class CondaEnvSdkFlavor extends CPythonSdkFlavor { @Override public Icon getIcon() { - return PythonIcons.Python.Condaenv; + return PythonIcons.Python.Anaconda; } } diff --git a/python/src/com/jetbrains/python/sdk/flavors/PyCondaRun.kt b/python/src/com/jetbrains/python/sdk/flavors/PyCondaRun.kt index c9f02c59e39a..e75a3503cc25 100644 --- a/python/src/com/jetbrains/python/sdk/flavors/PyCondaRun.kt +++ b/python/src/com/jetbrains/python/sdk/flavors/PyCondaRun.kt @@ -1,11 +1,15 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.jetbrains.python.sdk.flavors +import com.google.gson.Gson +import com.google.gson.JsonSyntaxException +import com.google.gson.annotations.SerializedName import com.intellij.execution.ExecutionException import com.intellij.execution.RunCanceledByUserException import com.intellij.execution.configurations.GeneralCommandLine import com.intellij.execution.process.CapturingProcessHandler import com.intellij.execution.process.ProcessOutput +import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.progress.EmptyProgressIndicator import com.intellij.openapi.progress.ProgressManager import com.intellij.openapi.projectRoots.Sdk @@ -13,18 +17,27 @@ import com.intellij.openapi.util.text.StringUtil import com.jetbrains.python.packaging.PyCondaPackageService import com.jetbrains.python.packaging.PyExecutionException import com.jetbrains.python.sdk.PythonSdkType +import org.jetbrains.concurrency.AsyncPromise +import org.jetbrains.concurrency.Promise -@Throws(PyExecutionException::class) +@Throws(ExecutionException::class) fun runConda(condaExecutable: String, arguments: List): ProcessOutput { return run(condaExecutable, arguments, readCondaEnv(condaExecutable)) } -@Throws(PyExecutionException::class) -fun runConda(sdk: Sdk, arguments: List): ProcessOutput { - return run(findCondaExecutable(sdk), arguments, PythonSdkType.activateVirtualEnv(sdk)) +@Throws(ExecutionException::class) +fun runConda(sdk: Sdk?, arguments: List): ProcessOutput { + val condaExecutable = findCondaExecutable(sdk) + val environment = if (sdk != null) { + PythonSdkType.activateVirtualEnv(sdk) + } + else { + readCondaEnv(condaExecutable) + } + return run(condaExecutable, arguments, environment) } -@Throws(PyExecutionException::class) +@Throws(ExecutionException::class) fun runCondaPython(condaPythonExecutable: String, arguments: List): ProcessOutput { return run(condaPythonExecutable, arguments, PythonSdkType.activateVirtualEnv(condaPythonExecutable)) } @@ -43,8 +56,8 @@ private fun readCondaEnv(condaExecutable: String): Map? { } @Throws(ExecutionException::class) -private fun findCondaExecutable(sdk: Sdk): String { - return PyCondaPackageService.getCondaExecutable(sdk.homePath) ?: throw ExecutionException("Cannot find conda executable") +private fun findCondaExecutable(sdk: Sdk?): String { + return PyCondaPackageService.getCondaExecutable(sdk?.homePath) ?: throw ExecutionException("Cannot find conda executable") } @Throws(PyExecutionException::class) @@ -54,4 +67,14 @@ private fun ProcessOutput.checkExitCode(executable: String, arguments: List { + val output = runConda(sdk, listOf("env", "list", "--json")) + val text = output.stdout + val envList = Gson().fromJson(text, CondaEnvironmentsList::class.java) + return envList.envs +} + +private data class CondaEnvironmentsList(@SerializedName("envs") var envs: List) diff --git a/python/src/com/jetbrains/python/sdk/flavors/VirtualEnvSdkFlavor.java b/python/src/com/jetbrains/python/sdk/flavors/VirtualEnvSdkFlavor.java index f79ff295d542..a91492baa80d 100644 --- a/python/src/com/jetbrains/python/sdk/flavors/VirtualEnvSdkFlavor.java +++ b/python/src/com/jetbrains/python/sdk/flavors/VirtualEnvSdkFlavor.java @@ -15,6 +15,7 @@ */ package com.jetbrains.python.sdk.flavors; +import com.intellij.openapi.application.ReadAction; import com.intellij.openapi.module.Module; import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileUtil; @@ -46,25 +47,27 @@ public class VirtualEnvSdkFlavor extends CPythonSdkFlavor { @Override public Collection suggestHomePaths(@Nullable Module module) { - final List candidates = new ArrayList<>(); - if (module != null) { - final VirtualFile baseDir = PySdkExtKt.getBaseDir(module); - if (baseDir != null) { - candidates.addAll(findInDirectory(baseDir)); + return ReadAction.compute(() -> { + final List candidates = new ArrayList<>(); + if (module != null) { + final VirtualFile baseDir = PySdkExtKt.getBaseDir(module); + if (baseDir != null) { + candidates.addAll(findInBaseDirectory(baseDir)); + } } - } - final VirtualFile path = getDefaultLocation(); - if (path != null) { - candidates.addAll(findInDirectory(path)); - } + final VirtualFile path = getDefaultLocation(); + if (path != null) { + candidates.addAll(findInBaseDirectory(path)); + } - final VirtualFile pyEnvLocation = getPyEnvDefaultLocations(); - if (pyEnvLocation != null) { - candidates.addAll(findInDirectory(pyEnvLocation)); - } + final VirtualFile pyEnvLocation = getPyEnvDefaultLocations(); + if (pyEnvLocation != null) { + candidates.addAll(findInBaseDirectory(pyEnvLocation)); + } - return candidates; + return candidates; + }); } @Nullable @@ -99,24 +102,31 @@ public class VirtualEnvSdkFlavor extends CPythonSdkFlavor { return null; } - public static Collection findInDirectory(VirtualFile rootDir) { + public static Collection findInBaseDirectory(@Nullable VirtualFile baseDir) { List candidates = new ArrayList<>(); - if (rootDir != null) { - rootDir.refresh(true, false); - VirtualFile[] suspects = rootDir.getChildren(); + if (baseDir != null) { + baseDir.refresh(true, false); + VirtualFile[] suspects = baseDir.getChildren(); for (VirtualFile child : suspects) { - if (child.isDirectory()) { - final VirtualFile bin = child.findChild("bin"); - final VirtualFile scripts = child.findChild("Scripts"); - if (bin != null) { - final String interpreter = findInterpreter(bin); - if (interpreter != null) candidates.add(interpreter); - } - if (scripts != null) { - final String interpreter = findInterpreter(scripts); - if (interpreter != null) candidates.add(interpreter); - } - } + candidates.addAll(findInRootDirectory(child)); + } + } + return candidates; + } + + @NotNull + public static Collection findInRootDirectory(@Nullable VirtualFile rootDir) { + final List candidates = new ArrayList<>(); + if (rootDir != null && rootDir.isDirectory()) { + final VirtualFile bin = rootDir.findChild("bin"); + final VirtualFile scripts = rootDir.findChild("Scripts"); + if (bin != null) { + final String interpreter = findInterpreter(bin); + if (interpreter != null) candidates.add(interpreter); + } + if (scripts != null) { + final String interpreter = findInterpreter(scripts); + if (interpreter != null) candidates.add(interpreter); } } return candidates;