PY-27360 Initial implementation of a package cache for Conda

python/src/com/jetbrains/python/packaging/CondaPackageCache.java

@@ -0,0 +1,112 @@
+// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.packaging;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.SerializedName;
+import com.intellij.openapi.application.PathManager;
+import com.intellij.openapi.diagnostic.Logger;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+/**
+ * @author Mikhail Golubev
+ */
+public class CondaPackageCache {
+  private static final Logger LOG = Logger.getInstance(PyPIPackageCache.class);
+  private static final Gson ourGson = new GsonBuilder().create();
+  private static CondaPackageCache ourInstance = null;
+
+  @NotNull
+  public static synchronized CondaPackageCache getInstance() {
+    if (ourInstance == null) {
+      ourInstance = new CondaPackageCache();
+      try (Reader reader = Files.newBufferedReader(getCachePath(), StandardCharsets.UTF_8)) {
+        ourInstance = ourGson.fromJson(reader, CondaPackageCache.class);
+        LOG.info("Loaded " + ourInstance.myPackages.size() + " packages from " + getCachePath());
+      }
+      catch (IOException exception) {
+        LOG.warn("Cannot load Conda package cache from the filesystem", exception);
+      }
+    }
+    return ourInstance;
+  }
+
+  @NotNull
+  public static synchronized CondaPackageCache reload(@NotNull Map<String, List<String>> packageNames) {
+    ourInstance = new CondaPackageCache(packageNames);
+    try {
+      Files.createDirectories(getCachePath().getParent());
+      try (Writer writer = Files.newBufferedWriter(getCachePath(), StandardCharsets.UTF_8)) {
+        ourGson.toJson(ourInstance, writer);
+      }
+    }
+    catch (IOException exception) {
+      LOG.warn("Cannot save Conda package cache to the filesystem", exception);
+    }
+    return ourInstance;
+  }
+
+  @NotNull
+  private static Path getCachePath() {
+    return Paths.get(PathManager.getSystemPath(), "python_packages", "conda-cache.json");
+  }
+
+  public CondaPackageCache() {
+
+  }
+
+  private CondaPackageCache(@NotNull Map<String, List<String>> nameToVersion) {
+    for (Map.Entry<String, List<String>> entry : nameToVersion.entrySet()) {
+      myPackages.put(entry.getKey(), new PackageInfo(entry.getValue()));
+    }
+  }
+
+  public boolean containsPackage(@NotNull String packageName) {
+    return myPackages.containsKey(packageName);
+  }
+
+  @NotNull
+  public Set<String> getPackageNames() {
+    return Collections.unmodifiableSet(myPackages.keySet());
+  }
+
+  /**
+   * Returns available package versions sorted in the reversed order using {@link com.intellij.webcore.packaging.PackageVersionComparator}.
+   * @param packageName case-insensitive name of a package
+   */
+  @NotNull
+  public List<String> getVersions(@NotNull String packageName) {
+    return Collections.unmodifiableList(myPackages.get(packageName).getVersions());
+  }
+
+  @SerializedName("packages")
+  private final TreeMap<String, PackageInfo> myPackages = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
+  private static class PackageInfo {
+
+    @SerializedName("v")
+    private List<String> myVersions;
+
+    public PackageInfo(@NotNull List<String> versions) {
+      myVersions = versions;
+    }
+
+    @SuppressWarnings("unused")
+    public PackageInfo() {
+    }
+
+    @NotNull
+    public List<String> getVersions() {
+      return Collections.unmodifiableList(myVersions);
+    }
+  }
+}

python/src/com/jetbrains/python/packaging/PyCondaPackageService.java

@@ -40,10 +40,7 @@ import java.util.*;
 @State(name = "PyCondaPackageService", storages = @Storage(value="conda_packages.xml", roamingType = RoamingType.DISABLED))
 public class PyCondaPackageService implements PersistentStateComponent<PyCondaPackageService> {
   private static final Logger LOG = Logger.getInstance(PyCondaPackageService.class);
-  public Map<String, String> CONDA_PACKAGES = ContainerUtil.newConcurrentMap();
-  public Map<String, List<String>> PACKAGES_TO_RELEASES = new HashMap<>();
   public Set<String> CONDA_CHANNELS = ContainerUtil.newConcurrentSet();
-
   public long LAST_TIME_CHECKED = 0;
 
   @Override
@@ -60,15 +57,10 @@ public class PyCondaPackageService implements PersistentStateComponent<PyCondaPa
     return ServiceManager.getService(PyCondaPackageService.class);
   }
 
-  public Map<String, String> getCondaPackages() {
-    return CONDA_PACKAGES;
-  }
-
-  public Map<String, String> loadAndGetPackages(boolean force) {
-    if (CONDA_PACKAGES.isEmpty() || force) {
+  public void loadAndGetPackages(boolean force) {
+    if (CondaPackageCache.getInstance().getPackageNames().isEmpty() || force) {
       updatePackagesCache();
     }
-    return CONDA_PACKAGES;
   }
 
   public Set<String> loadAndGetChannels() {
@@ -189,42 +181,26 @@ public class PyCondaPackageService implements PersistentStateComponent<PyCondaPa
       LOG.warn(output.getStderr());
       return;
     }
-    CONDA_PACKAGES.clear();
-    PACKAGES_TO_RELEASES.clear();
-    final List<String> lines = output.getStdoutLines();
-    for (String line : lines) {
+
+    final Map<String, List<String>> nameToVersions = new HashMap<>();
+    for (String line : output.getStdoutLines()) {
       final List<String> split = StringUtil.split(line, "\t");
       if (split.size() < 2) continue;
-      final String aPackage = CONDA_PACKAGES.get(split.get(0));
-      if (aPackage != null) {
-        if (VersionComparatorUtil.compare(aPackage, split.get(1)) < 0)
-          CONDA_PACKAGES.put(split.get(0), split.get(1));
-      }
-      else {
-        CONDA_PACKAGES.put(split.get(0), split.get(1));
-      }
-
-      if (PACKAGES_TO_RELEASES.containsKey(split.get(0))) {
-        final List<String> versions = PACKAGES_TO_RELEASES.get(split.get(0));
-        if (!versions.contains(split.get(1))) {
-          versions.add(split.get(1));
-        }
-      }
-      else {
-        final ArrayList<String> versions = new ArrayList<>();
-        versions.add(split.get(1));
-        PACKAGES_TO_RELEASES.put(split.get(0), versions);
+      final String packageName = split.get(0);
+      final String packageVersion = split.get(1);
+      final List<String> versions = nameToVersions.computeIfAbsent(packageName, k -> new ArrayList<>());
+      final int sortedIndex = Collections.binarySearch(versions, packageVersion, VersionComparatorUtil.COMPARATOR.reversed());
+      if (sortedIndex < 0) {
+        versions.add(-(sortedIndex + 1), packageVersion);
       }
     }
+    CondaPackageCache.reload(nameToVersions);
     LAST_TIME_CHECKED = System.currentTimeMillis();
   }
 
   @NotNull
   public List<String> getPackageVersions(@NotNull final String packageName) {
-    if (PACKAGES_TO_RELEASES.containsKey(packageName)) {
-      return PACKAGES_TO_RELEASES.get(packageName);
-    }
-    return Collections.emptyList();
+    return CondaPackageCache.getInstance().getVersions(packageName);
   }
 
   public void updateChannels() {

python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java

@@ -25,16 +25,17 @@ import com.intellij.openapi.project.Project;
 import com.intellij.openapi.projectRoots.Sdk;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.CatchingConsumer;
-import com.intellij.webcore.packaging.PackageVersionComparator;
+import com.intellij.util.containers.ContainerUtil;
 import com.intellij.webcore.packaging.RepoPackage;
-import com.jetbrains.python.packaging.*;
+import com.jetbrains.python.packaging.CondaPackageCache;
+import com.jetbrains.python.packaging.PyCondaPackageManagerImpl;
+import com.jetbrains.python.packaging.PyCondaPackageService;
+import com.jetbrains.python.packaging.PyPackageManager;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
 public class PyCondaManagementService extends PyPackageManagementService {
   private static final Logger LOG = Logger.getInstance(PyCondaManagementService.class);
@@ -50,19 +51,30 @@ public class PyCondaManagementService extends PyPackageManagementService {
   @Override
   @NotNull
   public List<RepoPackage> getAllPackagesCached() {
-    return useConda() ? versionMapToPackageList(PyCondaPackageService.getInstance().getCondaPackages()) : super.getAllPackagesCached();
+    if (useConda()) {
+      return getCachedCondaPackages();
+    }
+    return super.getAllPackagesCached();
   }
 
   @Override
   @NotNull
   public List<RepoPackage> getAllPackages() throws IOException {
-    return useConda() ? versionMapToPackageList(PyCondaPackageService.getInstance().loadAndGetPackages(false)) : super.getAllPackages();
+    if (useConda()) {
+      PyCondaPackageService.getInstance().loadAndGetPackages(false);
+      return getAllPackagesCached();
+    }
+    return super.getAllPackages();
   }
 
   @Override
   @NotNull
   public List<RepoPackage> reloadAllPackages() throws IOException {
-    return useConda() ? versionMapToPackageList(PyCondaPackageService.getInstance().loadAndGetPackages(true)) : super.reloadAllPackages();
+    if (useConda()) {
+      PyCondaPackageService.getInstance().loadAndGetPackages(true);
+      return getAllPackagesCached();
+    }
+    return super.reloadAllPackages();
   }
 
   @Override
@@ -132,9 +144,7 @@ public class PyCondaManagementService extends PyPackageManagementService {
   @Override
   public void fetchPackageVersions(String packageName, CatchingConsumer<List<String>, Exception> consumer) {
     if (useConda()) {
-      final List<String> versions = PyCondaPackageService.getInstance().getPackageVersions(packageName);
-      Collections.sort(versions, Collections.reverseOrder(new PackageVersionComparator()));
-      consumer.consume(versions);
+      consumer.consume(PyCondaPackageService.getInstance().getPackageVersions(packageName));
     }
     else {
       super.fetchPackageVersions(packageName, consumer);
@@ -142,13 +152,11 @@ public class PyCondaManagementService extends PyPackageManagementService {
   }
 
   @NotNull
-  protected static List<RepoPackage> versionMapToPackageList(@NotNull Map<String, String> packageToVersionMap) {
-    final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty();
-    final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : "";
-    final List<RepoPackage> packages = new ArrayList<>();
-    for (Map.Entry<String, String> entry : packageToVersionMap.entrySet()) {
-      packages.add(new RepoPackage(entry.getKey(), url, entry.getValue()));
-    }
-    return packages;
+  private static List<RepoPackage> getCachedCondaPackages() {
+    final CondaPackageCache instance = CondaPackageCache.getInstance();
+    return ContainerUtil.map(instance.getPackageNames(), name -> {
+      final String latestVersion = ContainerUtil.getFirstItem(instance.getVersions(name));
+      return new RepoPackage(name, null, latestVersion);
+    });
   }
 }