PY-24412 Keep names of PyPI packages separately from the packaging settings

So as not to load the whole cache of available packages on EDT just
to get a timestamp for its invalidation in PyPackagesUpdater.

One simplification I made is that we no longer store the latest versions
of these packages in persisted settings. It would to much of a hassle
to keep it in sync then, and overall it's not that beneficial since
we're not able to get both package names and their versions in one
request due to the latest changes in PyPI API.

python/src/com/jetbrains/python/packaging/PyPIPackageCache.java

@@ -0,0 +1,92 @@
+// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.jetbrains.python.packaging;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.SerializedName;
+import com.intellij.openapi.application.PathManager;
+import com.intellij.openapi.diagnostic.Logger;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author Mikhail Golubev
+ */
+public class PyPIPackageCache {
+  private static final Logger LOG = Logger.getInstance(PyPIPackageCache.class);
+  private static final Gson ourGson = new GsonBuilder().create();
+  private static PyPIPackageCache ourInstance = null;
+
+  @NotNull
+  public static synchronized PyPIPackageCache getInstance() {
+    if (ourInstance == null) {
+      ourInstance = new PyPIPackageCache();
+      try (Reader reader = Files.newBufferedReader(getCachePath(), StandardCharsets.UTF_8)) {
+        ourInstance = ourGson.fromJson(reader, PyPIPackageCache.class);
+        LOG.info("Loaded " + ourInstance.getPackageNames().size() + " packages from " + getCachePath());
+      }
+      catch (IOException exception) {
+        LOG.warn("Cannot load PyPI package cache from the filesystem", exception);
+      }
+    }
+    return ourInstance;
+  }
+
+  @NotNull
+  public static synchronized PyPIPackageCache reload(@NotNull List<String> packageNames) {
+    ourInstance = new PyPIPackageCache(packageNames);
+    try {
+      Files.createDirectories(getCachePath().getParent());
+      try (Writer writer = Files.newBufferedWriter(getCachePath(), StandardCharsets.UTF_8)) {
+        ourGson.toJson(ourInstance, writer);
+      }
+    }
+    catch (IOException exception) {
+      LOG.warn("Cannot save PyPI package cache to the filesystem", exception);
+    }
+    return ourInstance;
+  }
+
+  // For GSON
+  private PyPIPackageCache() {
+  }
+
+  public PyPIPackageCache(@NotNull List<String> packageNames) {
+    myPackageNames = packageNames;
+  }
+
+  @NotNull
+  private static Path getCachePath() {
+    return Paths.get(PathManager.getSystemPath(), "python_packages", "pypi-cache.json");
+  }
+
+  @SerializedName("packages")
+  private List<String> myPackageNames = new ArrayList<>();
+
+  @NotNull
+  public List<String> getPackageNames() {
+    return Collections.unmodifiableList(myPackageNames);
+  }
+
+  /**
+   * Checks that the given name is among those available at PyPI <em>case-insensitively</em>.
+   * <p>
+   * Note that if the cache hasn't been initialized yet or there was an error during its loading,
+   * {@link #getInstance()} returns an empty sentinel value, and, therefore, this method will return {@code false}.
+   * It's worth writing code analysis so that this value doesn't lead to false positives in the editor
+   * when the cache is merely not ready.
+   */
+  public boolean containsPackage(@NotNull String name) {
+    return Collections.binarySearch(myPackageNames, name, String.CASE_INSENSITIVE_ORDER) >= 0;
+  }
+}

python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java

@@ -5,7 +5,6 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.annotations.SerializedName;
@@ -39,7 +38,6 @@ import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 public class PyPIPackageUtil {
   private static final Logger LOG = Logger.getInstance(PyPIPackageUtil.class);
@@ -83,7 +81,7 @@ public class PyPIPackageUtil {
    * 
    * @see #getAdditionalPackages() 
    */
-  private volatile Set<RepoPackage> myAdditionalPackages = null;
+  private volatile List<RepoPackage> myAdditionalPackages = null;
 
   /**
    * Contains cached package information retrieved through PyPI's JSON API.
@@ -101,13 +99,6 @@ public class PyPIPackageUtil {
       }
     });
   
-  /**
-   * Lowercased package names for fast check that some package is available in PyPI.
-   * TODO find the way to get rid of it, it's not a good idea to store 85k+ entries in memory twice
-   */
-  @Nullable private volatile Set<String> myPackageNames = null;
-
-
   /**
    * Prevents simultaneous updates of {@link PyPackageService#PY_PACKAGES}
    * because the corresponding response contains tons of data and multiple
@@ -161,15 +152,20 @@ public class PyPIPackageUtil {
   }
 
   @NotNull
-  public Set<RepoPackage> getAdditionalPackages() throws IOException {
-    if (myAdditionalPackages == null) {
-      final Set<RepoPackage> packages = new TreeSet<>();
+  public List<RepoPackage> getAdditionalPackages() {
+    return myAdditionalPackages != null ? Collections.unmodifiableList(myAdditionalPackages) : Collections.emptyList();
+  }
+
+  @NotNull
+  public List<RepoPackage> loadAndGetAdditionalPackages(boolean alwaysRefresh) throws IOException {
+    if (myAdditionalPackages == null || alwaysRefresh) {
+      final Set<RepoPackage> uniquePackages = new TreeSet<>();
       for (String url : PyPackageService.getInstance().additionalRepositories) {
-        packages.addAll(getPackagesFromAdditionalRepository(url));
+        uniquePackages.addAll(getPackagesFromAdditionalRepository(url));
       }
-      myAdditionalPackages = packages;
+      myAdditionalPackages = new ArrayList<>(uniquePackages);
     }
-    return Collections.unmodifiableSet(myAdditionalPackages);
+    return Collections.unmodifiableList(myAdditionalPackages);
   }
 
   @NotNull
@@ -203,11 +199,6 @@ public class PyPIPackageUtil {
     return result;
   }
 
-  public void clearPackagesCache() {
-    PyPackageService.getInstance().PY_PACKAGES.clear();
-    myAdditionalPackages = null;
-  }
-
   public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer<PackageDetails.Info, Exception> callback) {
     ApplicationManager.getApplication().executeOnPooledThread(() -> {
       try {
@@ -262,9 +253,7 @@ public class PyPIPackageUtil {
   private String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException {
     LOG.debug("Requesting the latest PyPI version for the package " + packageName);
     final List<String> versions = getPackageVersionsFromPyPI(packageName, true);
-    final String latest = ContainerUtil.getFirstItem(versions);
-    getPyPIPackages().put(packageName, StringUtil.notNullize(latest));
-    return latest;
+    return ContainerUtil.getFirstItem(versions);
   }
 
   /**
@@ -295,9 +284,9 @@ public class PyPIPackageUtil {
 
   @Nullable
   public String fetchLatestPackageVersion(@NotNull String packageName) throws IOException {
-    String version = getPyPIPackages().get(packageName);
-    // Package is on PyPI but it's version is unknown
-    if (version != null && version.isEmpty()) {
+    String version = null;
+    // Package is on PyPI not a, say, some system package on Ubuntu
+    if (PyPIPackageCache.getInstance().containsPackage(packageName)) {
       version = getLatestPackageVersionFromPyPI(packageName);
     }
     if (!PyPackageService.getInstance().additionalRepositories.isEmpty()) {
@@ -349,27 +338,29 @@ public class PyPIPackageUtil {
     return repository + suffix;
   }
 
-  public void updatePyPICache(@NotNull PyPackageService service) throws IOException {
+  public void updatePyPICache() throws IOException {
+    final PyPackageService service = PyPackageService.getInstance();
     service.LAST_TIME_CHECKED = System.currentTimeMillis();
-
-    service.PY_PACKAGES.clear();
     if (service.PYPI_REMOVED) return;
-    parsePyPIList(parsePyPIListFromWeb(PYPI_LIST_URL, true), service);
+    final List<String> decodedNames = parsePyPIList(parsePyPIListFromWeb(PYPI_LIST_URL, true));
+    PyPIPackageCache.reload(decodedNames);
   }
 
-  private void parsePyPIList(@NotNull List<String> packages, @NotNull PyPackageService service) {
-    myPackageNames = null;
+  @NotNull
+  private static List<String> parsePyPIList(@NotNull List<String> packages) {
+    final List<String> decodedNames = new ArrayList<>();
     for (String pyPackage : packages) {
       try {
         final String packageName = URLDecoder.decode(pyPackage, "UTF-8");
         if (!packageName.contains(" ")) {
-          service.PY_PACKAGES.put(packageName, "");
+          decodedNames.add(packageName);
         }
       }
       catch (UnsupportedEncodingException e) {
         LOG.warn(e.getMessage());
       }
     }
+    return decodedNames;
   }
 
   @NotNull
@@ -418,37 +409,22 @@ public class PyPIPackageUtil {
     });
   }
 
-  @NotNull
-  public Collection<String> getPackageNames() {
-    final Map<String, String> pyPIPackages = getPyPIPackages();
-    final ArrayList<String> list = Lists.newArrayList(pyPIPackages.keySet());
-    Collections.sort(list);
-    return list;
-  }
-
-  @NotNull
-  public Map<String, String> loadAndGetPackages() throws IOException {
-    // The map returned by getPyPIPackages() is already thread-safe;
-    // this lock is solely to prevent multiple threads from updating
+  public void loadAndGetPackages() throws IOException {
+    // This lock is solely to prevent multiple threads from updating
     // the mammoth cache of PyPI packages simultaneously.
     synchronized (myPyPIPackageCacheUpdateLock) {
-      if (getPyPIPackages().isEmpty()) {
-        updatePyPICache(PyPackageService.getInstance());
+      final PyPIPackageCache cache = PyPIPackageCache.getInstance();
+      if (cache.getPackageNames().isEmpty()) {
+        updatePyPICache();
       }
-      return getPyPIPackages();
     }
   }
 
-  @NotNull
-  public static Map<String, String> getPyPIPackages() {
-    return PyPackageService.getInstance().PY_PACKAGES;
-  }
-
+  /**
+   * @see PyPIPackageCache#containsPackage(String)
+   */
   public boolean isInPyPI(@NotNull String packageName) {
-    if (myPackageNames == null) {
-      myPackageNames = getPyPIPackages().keySet().stream().map(name -> name.toLowerCase(Locale.ENGLISH)).collect(Collectors.toSet());
-    }
-    return myPackageNames != null && myPackageNames.contains(packageName.toLowerCase(Locale.ENGLISH));
+    return PyPIPackageCache.getInstance().containsPackage(packageName);
   }
 
   @SuppressWarnings("FieldMayBeFinal")

python/src/com/jetbrains/python/packaging/PyPackageService.java

@@ -30,10 +30,9 @@ public class PyPackageService implements
                               PersistentStateComponent<PyPackageService> {
   public volatile Map<String, Boolean> sdkToUsersite = ContainerUtil.newConcurrentMap();
   public volatile List<String> additionalRepositories = ContainerUtil.createConcurrentList();
-  public volatile Map<String, String> PY_PACKAGES = ContainerUtil.newConcurrentMap();
   @SystemIndependent public volatile String virtualEnvBasePath;
   public volatile Boolean PYPI_REMOVED = false;
-  
+
   public long LAST_TIME_CHECKED = 0;
 
   @Override

python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java

@@ -46,9 +46,7 @@ public class PyPackagesUpdater implements StartupActivity {
     if (checkNeeded(project)) {
       application.executeOnPooledThread(() -> {
         try {
-          final PyPackageService service = PyPackageService.getInstance();
-          PyPIPackageUtil.INSTANCE.updatePyPICache(service);
-          service.LAST_TIME_CHECKED = System.currentTimeMillis();
+          PyPIPackageUtil.INSTANCE.updatePyPICache();
         }
         catch (IOException e) {
           LOG.warn(e.getMessage());

python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java

@@ -27,15 +27,14 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.CatchingConsumer;
 import com.intellij.webcore.packaging.PackageVersionComparator;
 import com.intellij.webcore.packaging.RepoPackage;
-import com.jetbrains.python.packaging.PyCondaPackageManagerImpl;
-import com.jetbrains.python.packaging.PyCondaPackageService;
-import com.jetbrains.python.packaging.PyPackageManager;
+import com.jetbrains.python.packaging.*;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 public class PyCondaManagementService extends PyPackageManagementService {
   private static final Logger LOG = Logger.getInstance(PyCondaManagementService.class);
@@ -141,4 +140,15 @@ public class PyCondaManagementService extends PyPackageManagementService {
       super.fetchPackageVersions(packageName, consumer);
     }
   }
+
+  @NotNull
+  protected static List<RepoPackage> versionMapToPackageList(@NotNull Map<String, String> packageToVersionMap) {
+    final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty();
+    final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : "";
+    final List<RepoPackage> packages = new ArrayList<>();
+    for (Map.Entry<String, String> entry : packageToVersionMap.entrySet()) {
+      packages.add(new RepoPackage(entry.getKey(), url, entry.getValue()));
+    }
+    return packages;
+  }
 }

python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java

@@ -10,6 +10,7 @@ import com.intellij.openapi.util.SystemInfo;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.CatchingConsumer;
 import com.intellij.util.concurrency.AppExecutorUtil;
+import com.intellij.util.containers.ContainerUtil;
 import com.intellij.util.ui.JBUI;
 import com.intellij.webcore.packaging.InstalledPackage;
 import com.intellij.webcore.packaging.PackageManagementServiceEx;
@@ -93,34 +94,36 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
   @NotNull
   @Override
   public List<RepoPackage> getAllPackages() throws IOException {
-    final Map<String, String> packageToVersionMap = PyPIPackageUtil.INSTANCE.loadAndGetPackages();
-    final List<RepoPackage> packages = versionMapToPackageList(packageToVersionMap);
-    packages.addAll(PyPIPackageUtil.INSTANCE.getAdditionalPackages());
-    return packages;
-  }
-
-  @NotNull
-  protected static List<RepoPackage> versionMapToPackageList(@NotNull Map<String, String> packageToVersionMap) {
-    final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty();
-    final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : "";
-    final List<RepoPackage> packages = new ArrayList<>();
-    for (Map.Entry<String, String> entry : packageToVersionMap.entrySet()) {
-      packages.add(new RepoPackage(entry.getKey(), url, entry.getValue()));
-    }
-    return packages;
+    PyPIPackageUtil.INSTANCE.loadAndGetPackages();
+    PyPIPackageUtil.INSTANCE.loadAndGetAdditionalPackages(false);
+    return getAllPackagesCached();
   }
 
   @NotNull
   @Override
   public List<RepoPackage> reloadAllPackages() throws IOException {
-    PyPIPackageUtil.INSTANCE.clearPackagesCache();
-    return getAllPackages();
+    PyPIPackageUtil.INSTANCE.updatePyPICache();
+    PyPIPackageUtil.INSTANCE.loadAndGetAdditionalPackages(true);
+    return getAllPackagesCached();
   }
 
   @NotNull
   @Override
   public List<RepoPackage> getAllPackagesCached() {
-    return versionMapToPackageList(PyPIPackageUtil.getPyPIPackages());
+    final List<RepoPackage> pypiPackages = getCachedPyPIPackages();
+    final List<RepoPackage> additionalPackages = PyPIPackageUtil.INSTANCE.getAdditionalPackages();
+    // Make a copy, since ManagePackagesDialog attempts to change the passed collection directly
+    final List<RepoPackage> result = new ArrayList<>(pypiPackages);
+    result.addAll(additionalPackages);
+    return result;
+  }
+
+  @NotNull
+  private static List<RepoPackage> getCachedPyPIPackages() {
+    // Don't show URL next to the package name in "Available Packages" if only PyPI is in use
+    final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty();
+    final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : "";
+    return ContainerUtil.map(PyPIPackageCache.getInstance().getPackageNames(), name -> new RepoPackage(name, url, null));
   }
 
   @Override

python/src/com/jetbrains/python/statistics/PyPackageUsagesCollector.java

@@ -24,7 +24,7 @@ import com.intellij.openapi.module.Module;
 import com.intellij.openapi.module.ModuleManager;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.projectRoots.Sdk;
-import com.jetbrains.python.packaging.PyPIPackageUtil;
+import com.jetbrains.python.packaging.PyPIPackageCache;
 import com.jetbrains.python.packaging.PyPackageManager;
 import com.jetbrains.python.packaging.PyRequirement;
 import com.jetbrains.python.sdk.PythonSdkType;
@@ -51,7 +51,7 @@ public class PyPackageUsagesCollector extends AbstractProjectsUsagesCollector {
         ApplicationManager.getApplication().runReadAction(() -> {
           List<PyRequirement> requirements = PyPackageManager.getInstance(pythonSdk).getRequirements(m);
           if (requirements != null) {
-            Collection<String> packages = new HashSet<>(PyPIPackageUtil.INSTANCE.getPackageNames());
+            Collection<String> packages = new HashSet<>(PyPIPackageCache.getInstance().getPackageNames());
             for (PyRequirement requirement : requirements) {
               String name = requirement.getName();
               if (packages.contains(name)) {