PY-34719 PY-34718 PY-33754 PY-33701 Fixed updating lists of conda channels, packages, environments

These are several interconnected fixes, mostly related to using more modern conda API & CLI and making things work
asynchronously.

platform/lang-impl/src/com/intellij/webcore/packaging/InstalledPackagesPanel.java

@@ -278,7 +278,7 @@ public class InstalledPackagesPanel extends JPanel {
     return ObjectUtils.tryCast(myPackageManagementService, PackageManagementServiceEx.class);
   }
 
-  private void updateUninstallUpgrade() {
+  protected void updateUninstallUpgrade() {
     final int[] selected = myPackagesTable.getSelectedRows();
     boolean upgradeAvailable = false;
     boolean canUninstall = selected.length != 0;

platform/lang-impl/src/com/intellij/webcore/packaging/ManagePackagesDialog.java

@@ -171,7 +171,7 @@ public class ManagePackagesDialog extends DialogWrapper {
   }
 
   private void addManageAction() {
-    if (myController.getAllRepositories() != null) {
+    if (myController.canManageRepositories()) {
       myManageButton.addActionListener(new ActionListener() {
         @Override
         public void actionPerformed(ActionEvent event) {

platform/lang-impl/src/com/intellij/webcore/packaging/ManageRepoDialog.java

@@ -15,6 +15,9 @@
  */
 package com.intellij.webcore.packaging;
 
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.application.ModalityState;
+import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.ui.DialogWrapper;
 import com.intellij.openapi.ui.InputValidator;
@@ -22,30 +25,52 @@ import com.intellij.openapi.ui.Messages;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.ui.ToolbarDecorator;
 import com.intellij.ui.components.JBList;
+import com.intellij.util.CatchingConsumer;
 import com.intellij.util.ui.JBUI;
 
 import javax.swing.*;
+import java.util.List;
 
 public class ManageRepoDialog extends DialogWrapper {
   private JPanel myMainPanel;
-  private final JBList myList;
+  private final JBList<String> myList;
   private boolean myEnabled;
+  private static final Logger LOG = Logger.getInstance(ManageRepoDialog.class);
 
   public ManageRepoDialog(Project project, final PackageManagementService controller) {
     super(project, false);
     init();
     setTitle("Manage Repositories");
-    final DefaultListModel repoModel = new DefaultListModel();
-    for(String repoUrl: controller.getAllRepositories()) {
-      repoModel.addElement(repoUrl);
-    }
-    myList = new JBList();
+    myList = new JBList<>();
+    myList.setPaintBusy(true);
+    final DefaultListModel<String> repoModel = new DefaultListModel<>();
+    controller.fetchAllRepositories(new CatchingConsumer<List<String>, Exception>() {
+      @Override
+      public void consume(List<String> repoUrls) {
+        ApplicationManager.getApplication().invokeLater(() -> {
+          if (isDisposed()) return;
+          myList.setPaintBusy(false);
+          for (String repoUrl: repoUrls) {
+            repoModel.addElement(repoUrl);
+          }
+        }, ModalityState.any());
+      }
+
+      @Override
+      public void consume(Exception e) {
+        ApplicationManager.getApplication().invokeLater(() -> {
+          if (isDisposed()) return;
+          myList.setPaintBusy(false);
+          LOG.warn(e);
+        });
+      }
+    });
     myList.setModel(repoModel);
     myList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
 
     myList.addListSelectionListener(event -> {
-      final Object selected = myList.getSelectedValue();
-      myEnabled = controller.canModifyRepository((String) selected);
+      final String selected = myList.getSelectedValue();
+      myEnabled = controller.canModifyRepository(selected);
     });
 
     final ToolbarDecorator decorator = ToolbarDecorator.createDecorator(myList).disableUpDownActions();
@@ -61,7 +86,7 @@ public class ManageRepoDialog extends DialogWrapper {
       }
     });
     decorator.setEditAction(button -> {
-      final String oldValue = (String)myList.getSelectedValue();
+      final String oldValue = myList.getSelectedValue();
 
       String url = Messages.showInputDialog("Please edit repository URL", "Repository URL", null, oldValue, new InputValidator() {
         @Override
@@ -82,7 +107,7 @@ public class ManageRepoDialog extends DialogWrapper {
       }
     });
     decorator.setRemoveAction(button -> {
-      String selected = (String)myList.getSelectedValue();
+      String selected = myList.getSelectedValue();
       controller.removeRepository(selected);
       repoModel.removeElement(selected);
       button.setEnabled(false);

platform/lang-impl/src/com/intellij/webcore/packaging/PackageManagementService.java

@@ -23,6 +23,20 @@ public abstract class PackageManagementService {
     return null;
   }
 
+  /**
+   * An async version of {@link #getAllRepositories()}.
+   */
+  public void fetchAllRepositories(@NotNull CatchingConsumer<? super List<String>, ? super Exception> consumer) {
+    consumer.consume(getAllRepositories());
+  }
+
+  /**
+   * Returns true if the service supports managing repositories.
+   */
+  public boolean canManageRepositories() {
+    return getAllRepositories() != null;
+  }
+
   /**
    * Checks if the user can change the URL of the specified repository or remove it from the list.
    *

python/helpers/conda_packaging_tool.py

@@ -4,11 +4,13 @@ import traceback
 ERROR_WRONG_USAGE = 1
 ERROR_EXCEPTION = 4
 
+
 def usage():
-    sys.stderr.write('Usage: conda_packaging_tool.py <listall>\n')
+    sys.stderr.write('Usage: conda_packaging_tool.py listall | channels | versions PACKAGE\n')
     sys.stderr.flush()
     exit(ERROR_WRONG_USAGE)
 
+
 def do_list_available_packages():
     import conda
     version = conda.__version__
@@ -23,6 +25,7 @@ def do_list_available_packages():
     minor_version = int(version_splitted[1])
 
     if major_version >= 4 and minor_version >= 4:
+        init_context()
         from conda.core.index import get_index
         index = get_index()
     elif major_version == 4 and minor_version >= 2:
@@ -36,20 +39,52 @@ def do_list_available_packages():
         index = common.get_index_trap()
 
     for pkg in index.values():
-        sys.stdout.write("\t".join([pkg["name"], pkg["version"], ":".join(pkg["depends"])])+chr(10))
+        sys.stdout.write("\t".join([pkg["name"], pkg["version"], ":".join(pkg["depends"])]) + chr(10))
         sys.stdout.flush()
 
 
 def do_list_channels():
-    import conda.config as config
-    if hasattr(config, "get_channel_urls"):
-        channels = config.get_channel_urls()
+    context = init_context()
+    if context:
+        channels = context.channels
     else:
-        channels = config.context.channels
-    for channel in channels:
-        if channel != 'defaults':
-            sys.stdout.write(channel+chr(10))
-            sys.stdout.flush()
+        import conda.config as config
+        if hasattr(config, "get_channel_urls"):
+            channels = config.get_channel_urls()
+        else:
+            channels = config.context.channels
+    sys.stdout.write('\n'.join(channels))
+    sys.stdout.write('\n')
+    sys.stdout.flush()
+
+
+def fetch_versions(package):
+    import json
+    from distutils.version import LooseVersion
+    from conda.cli.python_api import run_command, Commands
+
+    stdout, stderr, ret_code = run_command(Commands.SEARCH, package, '--json')
+    if ret_code != 0:
+        raise Exception(stderr)
+    results = json.loads(stdout)
+    results = results.get(package, [])
+    all_versions = (r.get('version') for r in results)
+    return sorted(set(v for v in all_versions if v), key=LooseVersion, reverse=True)
+
+
+def do_list_versions(package):
+    sys.stdout.write('\n'.join(fetch_versions(package)))
+    sys.stderr.write('\n')
+    sys.stdout.flush()
+
+
+def init_context():
+    try:
+        from conda.base.context import context
+    except ImportError:
+        return None
+    context.__init__()
+    return context
 
 
 def main():
@@ -68,6 +103,11 @@ def main():
                 usage()
                 return
             do_list_channels()
+        elif cmd == 'versions':
+            if len(sys.argv) != 3:
+                usage()
+                return
+            do_list_versions(sys.argv[2])
         else:
             usage()
     except Exception:
@@ -75,5 +115,6 @@ def main():
         exit(ERROR_EXCEPTION)
     exit(retcode)
 
+
 if __name__ == '__main__':
     main()

python/ide/src/com/jetbrains/python/PythonSdkConfigurator.kt

@@ -1,8 +1,12 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python
 
+import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.progress.ProgressIndicator
+import com.intellij.openapi.progress.ProgressManager
+import com.intellij.openapi.progress.Task
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.projectRoots.impl.SdkConfigurationUtil
@@ -27,13 +31,22 @@ class PythonSdkConfigurator : DirectoryProjectConfigurator {
     }
 
     fun findDetectedAssociatedEnvironment(module: Module, existingSdks: List<Sdk>): PyDetectedSdk? {
-      detectVirtualEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let {
-        return it
-      }
-      detectCondaEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let {
-        return it
-      }
-      return null
+      // TODO: Move all interpreter detection away from EDT & use proper synchronization for that
+      val progress = ProgressManager.getInstance()
+      return progress.run(object : Task.WithResult<PyDetectedSdk?, Exception>(module.project,
+                                                                              "Looking for Virtual Environments",
+                                                                              false) {
+        override fun compute(indicator: ProgressIndicator): PyDetectedSdk? {
+          indicator.isIndeterminate = true
+          detectVirtualEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let {
+            return it
+          }
+          detectCondaEnvs(module, existingSdks).firstOrNull { it.isAssociatedWithModule(module) }?.let {
+            return it
+          }
+          return null
+        }
+      })
     }
 
     private fun findExistingSystemWideSdk(existingSdks: List<Sdk>) =

python/src/com/jetbrains/python/packaging/PyCondaPackageCache.java

@@ -1,40 +0,0 @@
-// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-package com.jetbrains.python.packaging;
-
-import com.google.common.collect.Multimap;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.ArrayList;
-
-/**
- * @author Mikhail Golubev
- */
-public class PyCondaPackageCache extends PyAbstractPackageCache {
-  private static final String CACHE_FILE_NAME = "conda-cache.json";
-
-  private static PyCondaPackageCache ourInstance;
-
-  @NotNull
-  public static synchronized PyCondaPackageCache getInstance() {
-    if (ourInstance == null) {
-      ourInstance = load(PyCondaPackageCache.class, new PyCondaPackageCache(), getDefaultCachePath(CACHE_FILE_NAME));
-    }
-    return ourInstance;
-  }
-
-  @NotNull
-  public static synchronized PyCondaPackageCache reload(@NotNull Multimap<String, String> nameToVersion) {
-    ourInstance = new PyCondaPackageCache(nameToVersion);
-    store(ourInstance, CACHE_FILE_NAME);
-    return ourInstance;
-  }
-
-  private PyCondaPackageCache() {
-  }
-
-  private PyCondaPackageCache(@NotNull Multimap<String, String> nameToVersion) {
-    for (String name : nameToVersion.keySet()) {
-      myPackages.put(name, new PyAbstractPackageCache.PackageInfo(new ArrayList<>(nameToVersion.get(name))));
-    }
-  }
-}

python/src/com/jetbrains/python/packaging/PyCondaPackageManagerImpl.java

@@ -145,6 +145,11 @@ public class PyCondaPackageManagerImpl extends PyPackageManagerImpl {
     }
   }
 
+  @Override
+  public boolean hasManagement() throws ExecutionException {
+    return useConda || super.hasManagement();
+  }
+
   @NotNull
   private List<PyPackage> parseCondaToolOutput(@NotNull String s) throws ExecutionException {
     final String[] lines = StringUtil.splitByLines(s);

python/src/com/jetbrains/python/packaging/PyCondaPackageService.java

@@ -3,6 +3,7 @@ package com.jetbrains.python.packaging;
 
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Multimaps;
+import com.intellij.execution.ExecutionException;
 import com.intellij.execution.configurations.PathEnvironmentVariableUtil;
 import com.intellij.execution.process.ProcessOutput;
 import com.intellij.openapi.components.*;
@@ -13,7 +14,6 @@ import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.openapi.vfs.StandardFileSystems;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.util.SystemProperties;
-import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.text.VersionComparatorUtil;
 import com.intellij.util.xmlb.XmlSerializerUtil;
 import com.jetbrains.python.PythonHelpersLocator;
@@ -30,8 +30,6 @@ import java.util.*;
 @State(name = "PyCondaPackageService", storages = @Storage(value="conda_packages.xml", roamingType = RoamingType.DISABLED))
 public class PyCondaPackageService implements PersistentStateComponent<PyCondaPackageService> {
   private static final Logger LOG = Logger.getInstance(PyCondaPackageService.class);
-  public Set<String> CONDA_CHANNELS = ContainerUtil.newConcurrentSet();
-  public long LAST_TIME_CHECKED = 0;
   @Nullable @SystemDependent public String PREFERRED_CONDA_PATH = null;
 
   @Override
@@ -48,27 +46,6 @@ public class PyCondaPackageService implements PersistentStateComponent<PyCondaPa
     return ServiceManager.getService(PyCondaPackageService.class);
   }
 
-  public void loadAndGetPackages(boolean force) {
-    if (PyCondaPackageCache.getInstance().getPackageNames().isEmpty() || force) {
-      updatePackagesCache();
-    }
-  }
-
-  public Set<String> loadAndGetChannels() {
-    if (CONDA_CHANNELS.isEmpty()) {
-      updateChannels();
-    }
-    return CONDA_CHANNELS;
-  }
-
-  public void addChannel(@NotNull final String url) {
-    CONDA_CHANNELS.add(url);
-  }
-
-  public void removeChannel(@NotNull final String url) {
-    CONDA_CHANNELS.remove(url);
-  }
-
   @Nullable
   public static String getCondaPython() {
     final String conda = getSystemCondaExecutable();
@@ -107,14 +84,11 @@ public class PyCondaPackageService implements PersistentStateComponent<PyCondaPa
 
   @Nullable
   public static String getCondaExecutable(@Nullable String sdkPath) {
-    if (sdkPath == null) {
-      return null;
+    if (sdkPath != null) {
+      String condaPath = findCondaExecutableRelativeToEnv(sdkPath);
+      if (condaPath != null) return condaPath;
     }
 
-    String condaPath = findCondaExecutableRelativeToEnv(sdkPath);
-
-    if (condaPath != null) return condaPath;
-
     if (StringUtil.isNotEmpty(getInstance().PREFERRED_CONDA_PATH)) {
       return getInstance().PREFERRED_CONDA_PATH;
     }
@@ -140,7 +114,7 @@ public class PyCondaPackageService implements PersistentStateComponent<PyCondaPa
   }
 
   @Nullable
-  public static String getCondaExecutableByName(@NotNull final String condaName) {
+  private static String getCondaExecutableByName(@NotNull final String condaName) {
     final VirtualFile userHome = LocalFileSystem.getInstance().findFileByPath(SystemProperties.getUserHome().replace('\\', '/'));
     if (userHome != null) {
       for (String root : CondaEnvSdkFlavor.CONDA_DEFAULT_ROOTS) {
@@ -152,15 +126,13 @@ public class PyCondaPackageService implements PersistentStateComponent<PyCondaPa
           executableFile = findExecutable(condaName, appData);
           if (executableFile != null) return executableFile;
           condaFolder = LocalFileSystem.getInstance().findFileByPath("C:\\" + root);
-          executableFile = findExecutable(condaName, condaFolder);
-          if (executableFile != null) return executableFile;
         }
         else {
           final String systemWidePath = "/opt/anaconda";
           condaFolder = LocalFileSystem.getInstance().findFileByPath(systemWidePath);
-          executableFile = findExecutable(condaName, condaFolder);
-          if (executableFile != null) return executableFile;
         }
+        executableFile = findExecutable(condaName, condaFolder);
+        if (executableFile != null) return executableFile;
       }
     }
 
@@ -185,51 +157,48 @@ public class PyCondaPackageService implements PersistentStateComponent<PyCondaPa
     return null;
   }
 
-  public void updatePackagesCache() {
-    final String condaPython = getCondaPython();
-    if (condaPython == null) {
-      return;
-    }
-    final String path = PythonHelpersLocator.getHelperPath("conda_packaging_tool.py");
-    final ProcessOutput output;
+  @Nullable
+  public Multimap<String, String> listAllPackagesAndVersions() {
     try {
-      output = PyCondaRunKt.runCondaPython(condaPython, Arrays.asList(path, "listall"));
+      final String output = runCondaPackagingHelper("listall");
+      final Multimap<String, String> nameToVersions =
+        Multimaps.newSortedSetMultimap(new HashMap<>(), () -> new TreeSet<>(VersionComparatorUtil.COMPARATOR.reversed()));
+      for (String line : StringUtil.split(output, "\n")) {
+        final List<String> split = StringUtil.split(line, "\t");
+        if (split.size() < 2) continue;
+        nameToVersions.put(split.get(0), split.get(1));
+      }
+      return nameToVersions;
     }
-    catch (PyExecutionException e) {
+    catch (ExecutionException e) {
       LOG.warn("Failed to get list of conda packages. " + e);
-      return;
+      return null;
     }
-
-    final Multimap<String, String> nameToVersions =
-      Multimaps.newSortedSetMultimap(new HashMap<>(), () -> new TreeSet<>(VersionComparatorUtil.COMPARATOR.reversed()));
-    for (String line : output.getStdoutLines()) {
-      final List<String> split = StringUtil.split(line, "\t");
-      if (split.size() < 2) continue;
-      nameToVersions.put(split.get(0), split.get(1));
-    }
-    PyCondaPackageCache.reload(nameToVersions);
-    LAST_TIME_CHECKED = System.currentTimeMillis();
   }
 
   @NotNull
-  public List<String> getPackageVersions(@NotNull final String packageName) {
-    return ContainerUtil.notNullize(PyCondaPackageCache.getInstance().getVersions(packageName));
+  public List<String> listPackageVersions(@NotNull String packageName) throws ExecutionException {
+    final String output = runCondaPackagingHelper("versions", packageName);
+    return StringUtil.split(output, "\n");
   }
 
-  public void updateChannels() {
+  @Nullable
+  public List<String> listChannels() throws ExecutionException {
+    final String output = runCondaPackagingHelper("channels");
+    return StringUtil.split(output, "\n");
+  }
+
+  @NotNull
+  private static String runCondaPackagingHelper(@NotNull String... args) throws ExecutionException {
+    final List<String> commandArgs = new ArrayList<>();
+    commandArgs.add(PythonHelpersLocator.getHelperPath("conda_packaging_tool.py"));
+    commandArgs.addAll(Arrays.asList(args));
     final String condaPython = getCondaPython();
-    if (condaPython == null) return;
-    final String path = PythonHelpersLocator.getHelperPath("conda_packaging_tool.py");
-    final ProcessOutput output;
-    try {
-      output = PyCondaRunKt.runCondaPython(condaPython, Arrays.asList(path, "channels"));
+    if (condaPython == null) {
+      throw new PyExecutionException("Cannot find Python executable for conda",
+                                     "python", commandArgs, new ProcessOutput());
     }
-    catch (PyExecutionException e) {
-      LOG.warn("Failed to update conda channels. " + e);
-      return;
-    }
-    final List<String> lines = output.getStdoutLines();
-    CONDA_CHANNELS.addAll(lines);
-    LAST_TIME_CHECKED = System.currentTimeMillis();
+    final ProcessOutput output = PyCondaRunKt.runCondaPython(condaPython, commandArgs);
+    return output.getStdout();
   }
 }

python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java

@@ -53,17 +53,6 @@ public class PyPackagesUpdater implements StartupActivity {
         }
       });
     }
-    if (checkCondaUpdateNeeded(project)) {
-      application.executeOnPooledThread(() -> PyCondaPackageService.getInstance().updatePackagesCache());
-    }
-  }
-
-  private static boolean checkCondaUpdateNeeded(Project project) {
-    if (!hasPython(project)) return false;
-    final long timeDelta = System.currentTimeMillis() - PyCondaPackageService.getInstance().LAST_TIME_CHECKED;
-    if (Math.abs(timeDelta) < EXPIRATION_TIMEOUT) return false;
-    LOG.debug("Updating outdated Conda package cache");
-    return true;
   }
 
   private static boolean hasPython(Project project) {

python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java

@@ -16,15 +16,18 @@
 package com.jetbrains.python.packaging.ui;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 import com.intellij.execution.ExecutionException;
 import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.progress.ProgressIndicator;
+import com.intellij.openapi.progress.ProgressManager;
+import com.intellij.openapi.progress.Task;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.projectRoots.Sdk;
 import com.intellij.util.CatchingConsumer;
 import com.intellij.util.containers.ContainerUtil;
 import com.intellij.webcore.packaging.InstalledPackage;
 import com.intellij.webcore.packaging.RepoPackage;
-import com.jetbrains.python.packaging.PyCondaPackageCache;
 import com.jetbrains.python.packaging.PyCondaPackageManagerImpl;
 import com.jetbrains.python.packaging.PyCondaPackageService;
 import com.jetbrains.python.packaging.PyPackageManager;
@@ -32,6 +35,8 @@ import com.jetbrains.python.sdk.flavors.PyCondaRunKt;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 public class PyCondaManagementService extends PyPackageManagementService {
@@ -49,45 +54,77 @@ public class PyCondaManagementService extends PyPackageManagementService {
   @NotNull
   public List<RepoPackage> getAllPackagesCached() {
     if (useConda()) {
-      return getCachedCondaPackages();
+      return Collections.emptyList();
+    }
+    else {
+      return super.getAllPackagesCached();
     }
-    return super.getAllPackagesCached();
   }
 
   @Override
   @NotNull
   public List<RepoPackage> getAllPackages() throws IOException {
     if (useConda()) {
-      PyCondaPackageService.getInstance().loadAndGetPackages(false);
-      return getAllPackagesCached();
+      return reloadAllPackages();
+    }
+    else {
+      return super.getAllPackages();
     }
-    return super.getAllPackages();
   }
 
   @Override
   @NotNull
   public List<RepoPackage> reloadAllPackages() throws IOException {
     if (useConda()) {
-      PyCondaPackageService.getInstance().loadAndGetPackages(true);
-      return getAllPackagesCached();
+      final Multimap<String, String> packages = PyCondaPackageService.getInstance().listAllPackagesAndVersions();
+      if (packages == null) return Collections.emptyList();
+      final List<RepoPackage> results = new ArrayList<>();
+      for (String pkg : packages.keySet()) {
+        results.add(new RepoPackage(pkg, null, ContainerUtil.getFirstItem(packages.get(pkg))));
+      }
+      return results;
     }
     return super.reloadAllPackages();
   }
 
   @Override
-  public List<String> getAllRepositories() {
-    return useConda() ? Lists.newArrayList(PyCondaPackageService.getInstance().loadAndGetChannels()) : super.getAllRepositories();
+  public boolean canManageRepositories() {
+    return true;
+  }
+
+  @Override
+  public void fetchAllRepositories(@NotNull CatchingConsumer<? super List<String>, ? super Exception> consumer) {
+    if (useConda()) {
+      myExecutorService.submit(() -> {
+        try {
+          final List<String> channels = ContainerUtil.notNullize(PyCondaPackageService.getInstance().listChannels());
+          consumer.consume(channels);
+        }
+        catch (ExecutionException e) {
+          consumer.consume(e);
+        }
+      });
+    }
+    else {
+      super.fetchAllRepositories(consumer);
+    }
   }
 
   @Override
   public void addRepository(String repositoryUrl) {
     if (useConda()) {
-      try {
-        PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--add", "channels", repositoryUrl, "--force"));
-      }
-      catch (ExecutionException e) {
-        LOG.warn("Failed to add repository. " + e);
-      }
+      ProgressManager.getInstance().run(new Task.Modal(getProject(), "Adding Conda Channel", true) {
+        @Override
+        public void run(@NotNull ProgressIndicator indicator) {
+          indicator.setIndeterminate(true);
+          try {
+            PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--add", "channels", repositoryUrl, "--force"));
+          }
+          catch (ExecutionException e) {
+            LOG.warn("Failed to add repository. " + e);
+          }
+        }
+      });
     }
     else {
       super.addRepository(repositoryUrl);
@@ -97,12 +134,18 @@ public class PyCondaManagementService extends PyPackageManagementService {
   @Override
   public void removeRepository(String repositoryUrl) {
     if (useConda()) {
-      try {
-        PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--remove", "channels", repositoryUrl, "--force"));
-      }
-      catch (ExecutionException e) {
-        LOG.warn("Failed to remove repository. " + e);
-      }
+      ProgressManager.getInstance().run(new Task.Modal(getProject(), "Removing Conda Channel", true) {
+        @Override
+        public void run(@NotNull ProgressIndicator indicator) {
+          indicator.setIndeterminate(true);
+          try {
+            PyCondaRunKt.runConda(mySdk, Lists.newArrayList("config", "--remove", "channels", repositoryUrl, "--force"));
+          }
+          catch (ExecutionException e) {
+            LOG.warn("Failed to remove repository. " + e);
+          }
+        }
+      });
     }
     else {
       super.removeRepository(repositoryUrl);
@@ -117,7 +160,14 @@ public class PyCondaManagementService extends PyPackageManagementService {
   @Override
   public void fetchPackageVersions(String packageName, CatchingConsumer<List<String>, Exception> consumer) {
     if (useConda()) {
-      consumer.consume(PyCondaPackageService.getInstance().getPackageVersions(packageName));
+      myExecutorService.submit(() -> {
+        try {
+          consumer.consume(PyCondaPackageService.getInstance().listPackageVersions(packageName));
+        }
+        catch (ExecutionException e) {
+          LOG.warn("Failed to fetch versions for '" + packageName + "'. " + e);
+        }
+      });
     }
     else {
       super.fetchPackageVersions(packageName, consumer);
@@ -128,20 +178,23 @@ public class PyCondaManagementService extends PyPackageManagementService {
   public void fetchLatestVersion(@NotNull InstalledPackage pkg, @NotNull CatchingConsumer<String, Exception> consumer) {
     final String packageName = pkg.getName();
     if (useConda()) {
-      final String latestVersion = ContainerUtil.getFirstItem(PyCondaPackageCache.getInstance().getVersions(packageName));
-      consumer.consume(latestVersion);
+      myExecutorService.submit(() -> {
+        try {
+          final String latestVersion = ContainerUtil.getFirstItem(PyCondaPackageService.getInstance().listPackageVersions(packageName));
+          consumer.consume(latestVersion);
+        }
+        catch (ExecutionException e) {
+          LOG.warn("Failed to fetch versions for '" + packageName + "'. " + e);
+        }
+      });
     }
     else {
       super.fetchLatestVersion(pkg, consumer);
     }
   }
 
-  @NotNull
-  private static List<RepoPackage> getCachedCondaPackages() {
-    final PyCondaPackageCache instance = PyCondaPackageCache.getInstance();
-    return ContainerUtil.map(instance.getPackageNames(), name -> {
-      final String latestVersion = ContainerUtil.getFirstItem(instance.getVersions(name));
-      return new RepoPackage(name, null, latestVersion);
-    });
+  @Override
+  public boolean shouldFetchLatestVersionsForOnlyInstalledPackages() {
+    return false;
   }
 }

python/src/com/jetbrains/python/packaging/ui/PyInstalledPackagesPanel.java

@@ -45,7 +45,7 @@ import java.util.Set;
  * @author yole
  */
 public class PyInstalledPackagesPanel extends InstalledPackagesPanel {
-  private boolean myHasManagement = false;
+  private volatile boolean myHasManagement = false;
 
   public PyInstalledPackagesPanel(@NotNull Project project, @NotNull PackagesNotificationPanel area) {
     super(project, area);
@@ -98,6 +98,7 @@ public class PyInstalledPackagesPanel extends InstalledPackagesPanel {
       PyExecutionException exception = null;
       try {
         myHasManagement = PyPackageManager.getInstance(selectedSdk).hasManagement();
+        application.invokeLater(() -> updateUninstallUpgrade(), ModalityState.any());
         if (!myHasManagement) {
           throw new PyExecutionException("Python packaging tools not found", "pip", Collections.emptyList(), "", "", 0,
                                          ImmutableList.of(new PyInstallPackageManagementFix()));

python/src/com/jetbrains/python/sdk/add/PyAddExistingCondaEnvPanel.kt

@@ -15,16 +15,26 @@
  */
 package com.jetbrains.python.sdk.add
 
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.application.ModalityState
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.ui.ValidationInfo
 import com.intellij.ui.components.JBCheckBox
+import com.intellij.util.ui.AsyncProcessIcon
 import com.intellij.util.ui.FormBuilder
 import com.jetbrains.python.sdk.*
+import com.sun.glass.ui.Application
 import icons.PythonIcons
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
 import java.awt.BorderLayout
 import javax.swing.Icon
+import javax.swing.SwingUtilities
+import javax.swing.SwingWorker
 
 /**
  * @author vlan
@@ -34,10 +44,8 @@ class PyAddExistingCondaEnvPanel(private val project: Project?,
                                  private val existingSdks: List<Sdk>,
                                  override var newProjectPath: String?) : PyAddSdkPanel() {
   override val panelName: String = "Existing environment"
-  override val icon: Icon = PythonIcons.Python.Condaenv
-  private val sdkComboBox = PySdkPathChoosingComboBox(detectCondaEnvs(module, existingSdks)
-                                                        .filterNot { it.isAssociatedWithAnotherModule(module) },
-                                                      null)
+  override val icon: Icon = PythonIcons.Python.Anaconda
+  private val sdkComboBox = PySdkPathChoosingComboBox(listOf(), null)
   private val makeSharedField = JBCheckBox("Make available to all projects")
 
   init {
@@ -47,6 +55,17 @@ class PyAddExistingCondaEnvPanel(private val project: Project?,
       .addComponent(makeSharedField)
       .panel
     add(formPanel, BorderLayout.NORTH)
+    ApplicationManager.getApplication().executeOnPooledThread(object: Runnable {
+      override fun run() {
+        if (module != null && module.isDisposed) return
+        val sdks = detectCondaEnvs(module, existingSdks)
+        ApplicationManager.getApplication().invokeLater({
+          sdks.forEach {
+            sdkComboBox.childComponent.addItem(it)
+          }
+        }, ModalityState.any())
+      }
+    })
   }
 
   override fun validateAll(): List<ValidationInfo> = listOfNotNull(validateSdkComboBox(sdkComboBox))

python/src/com/jetbrains/python/sdk/flavors/CondaEnvSdkFlavor.java

@@ -15,26 +15,26 @@
  */
 package com.jetbrains.python.sdk.flavors;
 
+import com.intellij.execution.ExecutionException;
+import com.intellij.openapi.application.ReadAction;
 import com.intellij.openapi.module.Module;
-import com.intellij.openapi.util.SystemInfo;
-import com.intellij.openapi.vfs.LocalFileSystem;
+import com.intellij.openapi.projectRoots.Sdk;
 import com.intellij.openapi.vfs.StandardFileSystems;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.util.SystemProperties;
-import com.jetbrains.python.packaging.PyCondaPackageService;
 import com.jetbrains.python.sdk.PythonSdkType;
 import icons.PythonIcons;
+import one.util.streamex.StreamEx;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
-import org.jetbrains.annotations.SystemDependent;
 
 import javax.swing.*;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
-import static com.jetbrains.python.sdk.flavors.VirtualEnvSdkFlavor.findInDirectory;
+import static com.jetbrains.python.sdk.flavors.VirtualEnvSdkFlavor.findInRootDirectory;
 
 public class CondaEnvSdkFlavor extends CPythonSdkFlavor {
   private CondaEnvSdkFlavor() {
@@ -47,80 +47,29 @@ public class CondaEnvSdkFlavor extends CPythonSdkFlavor {
 
   @Override
   public Collection<String> suggestHomePaths(@Nullable Module module) {
-    List<String> candidates = new ArrayList<>();
-
-    for (VirtualFile file : getCondaDefaultLocations()) {
-      candidates.addAll(findInDirectory(file));
-    }
-
-    return candidates;
-  }
-
-  public static List<VirtualFile> getCondaDefaultLocations() {
-    List<VirtualFile> roots = new ArrayList<>();
-    final VirtualFile userHome = LocalFileSystem.getInstance().findFileByPath(SystemProperties.getUserHome().replace('\\','/'));
-    if (userHome != null) {
-      final VirtualFile condaHidden = userHome.findChild(".conda");
-      if (condaHidden != null) {
-        addEnvsFolder(roots, condaHidden);
-      }
-      for (String root : CONDA_DEFAULT_ROOTS) {
-        VirtualFile condaFolder = userHome.findChild(root);
-        addEnvsFolder(roots, condaFolder);
-        if (SystemInfo.isWindows) {
-          final VirtualFile appData = userHome.findFileByRelativePath("AppData\\Local\\Continuum\\" + root);
-          addEnvsFolder(roots, appData);
-          condaFolder = LocalFileSystem.getInstance().findFileByPath("C:\\" + root);
-        }
-        else {
-          final String systemWidePath = "/opt/anaconda";
-          condaFolder = LocalFileSystem.getInstance().findFileByPath(systemWidePath);
-        }
-        addEnvsFolder(roots, condaFolder);
+    final List<String> results = new ArrayList<>();
+    final Sdk sdk = ReadAction.compute(() -> PythonSdkType.findPythonSdk(module));
+    try {
+      final List<String> environments = PyCondaRunKt.listCondaEnvironments(sdk);
+      for (String environment : environments) {
+        results.addAll(ReadAction.compute(() -> {
+          final VirtualFile root = StandardFileSystems.local().findFileByPath(environment);
+          return StreamEx.of(findInRootDirectory(root))
+            .filter(s -> getCondaEnvRoot(s) != null)
+            .toList();
+        }));
       }
     }
-    addEnvsFolder(roots, findPreferredCondaEnvsFolder());
-    return roots;
-  }
-
-  @Nullable
-  private static VirtualFile findPreferredCondaEnvsFolder() {
-    @SystemDependent final String path = PyCondaPackageService.getInstance().PREFERRED_CONDA_PATH;
-    if (path == null) return null;
-    final VirtualFile conda = StandardFileSystems.local().findFileByPath(path);
-    if (conda == null) return null;
-    final VirtualFile binFolder = conda.getParent();
-    if (binFolder == null) return null;
-    return binFolder.getParent();
-  }
-
-  private static void addEnvsFolder(@NotNull final List<VirtualFile> roots, @Nullable final VirtualFile condaFolder) {
-    if (condaFolder != null) {
-      final VirtualFile envs = condaFolder.findChild("envs");
-      if (envs != null) {
-        roots.add(envs);
-      }
+    catch (ExecutionException e) {
+      return Collections.emptyList();
     }
+    return results;
   }
 
   @Override
   public boolean isValidSdkPath(@NotNull File file) {
     if (!super.isValidSdkPath(file)) return false;
-    final File bin = file.getParentFile();
-    String condaName = "conda";
-    if (SystemInfo.isWindows) {
-      condaName = new File(bin, "envs").exists() ? "conda.exe" : "conda.bat";
-    }
-    if (bin != null) {
-      final File conda = new File(bin, condaName);
-      if (conda.exists()) {
-        return true;
-      }
-      final File condaFolder = bin.getParentFile();
-      final File condaExecutable = PythonSdkType.findExecutableFile(condaFolder, condaName);
-      if (condaExecutable != null) return true;
-    }
-    return false;
+    return PythonSdkType.isConda(file.getPath());
   }
 
   @Nullable
@@ -137,6 +86,6 @@ public class CondaEnvSdkFlavor extends CPythonSdkFlavor {
 
   @Override
   public Icon getIcon() {
-    return PythonIcons.Python.Condaenv;
+    return PythonIcons.Python.Anaconda;
   }
 }

python/src/com/jetbrains/python/sdk/flavors/PyCondaRun.kt

@@ -1,11 +1,15 @@
 // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.jetbrains.python.sdk.flavors
 
+import com.google.gson.Gson
+import com.google.gson.JsonSyntaxException
+import com.google.gson.annotations.SerializedName
 import com.intellij.execution.ExecutionException
 import com.intellij.execution.RunCanceledByUserException
 import com.intellij.execution.configurations.GeneralCommandLine
 import com.intellij.execution.process.CapturingProcessHandler
 import com.intellij.execution.process.ProcessOutput
+import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.progress.EmptyProgressIndicator
 import com.intellij.openapi.progress.ProgressManager
 import com.intellij.openapi.projectRoots.Sdk
@@ -13,18 +17,27 @@ import com.intellij.openapi.util.text.StringUtil
 import com.jetbrains.python.packaging.PyCondaPackageService
 import com.jetbrains.python.packaging.PyExecutionException
 import com.jetbrains.python.sdk.PythonSdkType
+import org.jetbrains.concurrency.AsyncPromise
+import org.jetbrains.concurrency.Promise
 
-@Throws(PyExecutionException::class)
+@Throws(ExecutionException::class)
 fun runConda(condaExecutable: String, arguments: List<String>): ProcessOutput {
   return run(condaExecutable, arguments, readCondaEnv(condaExecutable))
 }
 
-@Throws(PyExecutionException::class)
-fun runConda(sdk: Sdk, arguments: List<String>): ProcessOutput {
-  return run(findCondaExecutable(sdk), arguments, PythonSdkType.activateVirtualEnv(sdk))
+@Throws(ExecutionException::class)
+fun runConda(sdk: Sdk?, arguments: List<String>): ProcessOutput {
+  val condaExecutable = findCondaExecutable(sdk)
+  val environment = if (sdk != null) {
+    PythonSdkType.activateVirtualEnv(sdk)
+  }
+  else {
+    readCondaEnv(condaExecutable)
+  }
+  return run(condaExecutable, arguments, environment)
 }
 
-@Throws(PyExecutionException::class)
+@Throws(ExecutionException::class)
 fun runCondaPython(condaPythonExecutable: String, arguments: List<String>): ProcessOutput {
   return run(condaPythonExecutable, arguments, PythonSdkType.activateVirtualEnv(condaPythonExecutable))
 }
@@ -43,8 +56,8 @@ private fun readCondaEnv(condaExecutable: String): Map<String, String>? {
 }
 
 @Throws(ExecutionException::class)
-private fun findCondaExecutable(sdk: Sdk): String {
-  return PyCondaPackageService.getCondaExecutable(sdk.homePath) ?: throw ExecutionException("Cannot find conda executable")
+private fun findCondaExecutable(sdk: Sdk?): String {
+  return PyCondaPackageService.getCondaExecutable(sdk?.homePath) ?: throw ExecutionException("Cannot find conda executable")
 }
 
 @Throws(PyExecutionException::class)
@@ -54,4 +67,14 @@ private fun ProcessOutput.checkExitCode(executable: String, arguments: List<Stri
     else "Non-zero exit code"
     throw PyExecutionException(message, executable, arguments, this)
   }
-}
+}
+
+@Throws(ExecutionException::class, JsonSyntaxException::class)
+fun listCondaEnvironments(sdk: Sdk?): List<String> {
+  val output = runConda(sdk, listOf("env", "list", "--json"))
+  val text = output.stdout
+  val envList = Gson().fromJson(text, CondaEnvironmentsList::class.java)
+  return envList.envs
+}
+
+private data class CondaEnvironmentsList(@SerializedName("envs") var envs: List<String>)

python/src/com/jetbrains/python/sdk/flavors/VirtualEnvSdkFlavor.java

@@ -15,6 +15,7 @@
  */
 package com.jetbrains.python.sdk.flavors;
 
+import com.intellij.openapi.application.ReadAction;
 import com.intellij.openapi.module.Module;
 import com.intellij.openapi.util.SystemInfo;
 import com.intellij.openapi.util.io.FileUtil;
@@ -46,25 +47,27 @@ public class VirtualEnvSdkFlavor extends CPythonSdkFlavor {
 
   @Override
   public Collection<String> suggestHomePaths(@Nullable Module module) {
-    final List<String> candidates = new ArrayList<>();
-    if (module != null) {
-      final VirtualFile baseDir = PySdkExtKt.getBaseDir(module);
-      if (baseDir != null) {
-        candidates.addAll(findInDirectory(baseDir));
+    return ReadAction.compute(() -> {
+      final List<String> candidates = new ArrayList<>();
+      if (module != null) {
+        final VirtualFile baseDir = PySdkExtKt.getBaseDir(module);
+        if (baseDir != null) {
+          candidates.addAll(findInBaseDirectory(baseDir));
+        }
       }
-    }
 
-    final VirtualFile path = getDefaultLocation();
-    if (path != null) {
-      candidates.addAll(findInDirectory(path));
-    }
+      final VirtualFile path = getDefaultLocation();
+      if (path != null) {
+        candidates.addAll(findInBaseDirectory(path));
+      }
 
-    final VirtualFile pyEnvLocation = getPyEnvDefaultLocations();
-    if (pyEnvLocation != null) {
-      candidates.addAll(findInDirectory(pyEnvLocation));
-    }
+      final VirtualFile pyEnvLocation = getPyEnvDefaultLocations();
+      if (pyEnvLocation != null) {
+        candidates.addAll(findInBaseDirectory(pyEnvLocation));
+      }
 
-    return candidates;
+      return candidates;
+    });
   }
 
   @Nullable
@@ -99,24 +102,31 @@ public class VirtualEnvSdkFlavor extends CPythonSdkFlavor {
     return null;
   }
 
-  public static Collection<String> findInDirectory(VirtualFile rootDir) {
+  public static Collection<String> findInBaseDirectory(@Nullable VirtualFile baseDir) {
     List<String> candidates = new ArrayList<>();
-    if (rootDir != null) {
-      rootDir.refresh(true, false);
-      VirtualFile[] suspects = rootDir.getChildren();
+    if (baseDir != null) {
+      baseDir.refresh(true, false);
+      VirtualFile[] suspects = baseDir.getChildren();
       for (VirtualFile child : suspects) {
-        if (child.isDirectory()) {
-          final VirtualFile bin = child.findChild("bin");
-          final VirtualFile scripts = child.findChild("Scripts");
-          if (bin != null) {
-            final String interpreter = findInterpreter(bin);
-            if (interpreter != null) candidates.add(interpreter);
-          }
-          if (scripts != null) {
-            final String interpreter = findInterpreter(scripts);
-            if (interpreter != null) candidates.add(interpreter);
-          }
-        }
+        candidates.addAll(findInRootDirectory(child));
+      }
+    }
+    return candidates;
+  }
+
+  @NotNull
+  public static Collection<String> findInRootDirectory(@Nullable VirtualFile rootDir) {
+    final List<String> candidates = new ArrayList<>();
+    if (rootDir != null && rootDir.isDirectory()) {
+      final VirtualFile bin = rootDir.findChild("bin");
+      final VirtualFile scripts = rootDir.findChild("Scripts");
+      if (bin != null) {
+        final String interpreter = findInterpreter(bin);
+        if (interpreter != null) candidates.add(interpreter);
+      }
+      if (scripts != null) {
+        final String interpreter = findInterpreter(scripts);
+        if (interpreter != null) candidates.add(interpreter);
       }
     }
     return candidates;