diff --git a/python/src/com/jetbrains/python/packaging/PyPIPackageCache.java b/python/src/com/jetbrains/python/packaging/PyPIPackageCache.java new file mode 100644 index 000000000000..3e2356054392 --- /dev/null +++ b/python/src/com/jetbrains/python/packaging/PyPIPackageCache.java @@ -0,0 +1,92 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.packaging; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.diagnostic.Logger; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * @author Mikhail Golubev + */ +public class PyPIPackageCache { + private static final Logger LOG = Logger.getInstance(PyPIPackageCache.class); + private static final Gson ourGson = new GsonBuilder().create(); + private static PyPIPackageCache ourInstance = null; + + @NotNull + public static synchronized PyPIPackageCache getInstance() { + if (ourInstance == null) { + ourInstance = new PyPIPackageCache(); + try (Reader reader = Files.newBufferedReader(getCachePath(), StandardCharsets.UTF_8)) { + ourInstance = ourGson.fromJson(reader, PyPIPackageCache.class); + LOG.info("Loaded " + ourInstance.getPackageNames().size() + " packages from " + getCachePath()); + } + catch (IOException exception) { + LOG.warn("Cannot load PyPI package cache from the filesystem", exception); + } + } + return ourInstance; + } + + @NotNull + public static synchronized PyPIPackageCache reload(@NotNull List packageNames) { + ourInstance = new PyPIPackageCache(packageNames); + try { + Files.createDirectories(getCachePath().getParent()); + try (Writer writer = Files.newBufferedWriter(getCachePath(), StandardCharsets.UTF_8)) { + ourGson.toJson(ourInstance, writer); + } + } + catch (IOException exception) { + LOG.warn("Cannot save PyPI package cache to the filesystem", exception); + } + return ourInstance; + } + + // For GSON + private PyPIPackageCache() { + } + + public PyPIPackageCache(@NotNull List packageNames) { + myPackageNames = packageNames; + } + + @NotNull + private static Path getCachePath() { + return Paths.get(PathManager.getSystemPath(), "python_packages", "pypi-cache.json"); + } + + @SerializedName("packages") + private List myPackageNames = new ArrayList<>(); + + @NotNull + public List getPackageNames() { + return Collections.unmodifiableList(myPackageNames); + } + + /** + * Checks that the given name is among those available at PyPI case-insensitively. + *

+ * Note that if the cache hasn't been initialized yet or there was an error during its loading, + * {@link #getInstance()} returns an empty sentinel value, and, therefore, this method will return {@code false}. + * It's worth writing code analysis so that this value doesn't lead to false positives in the editor + * when the cache is merely not ready. + */ + public boolean containsPackage(@NotNull String name) { + return Collections.binarySearch(myPackageNames, name, String.CASE_INSENSITIVE_ORDER) >= 0; + } +} diff --git a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java index c0b76532d3fb..7888ca5aa6f5 100644 --- a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java +++ b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java @@ -5,7 +5,6 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.SerializedName; @@ -39,7 +38,6 @@ import java.util.*; import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; public class PyPIPackageUtil { private static final Logger LOG = Logger.getInstance(PyPIPackageUtil.class); @@ -83,7 +81,7 @@ public class PyPIPackageUtil { * * @see #getAdditionalPackages() */ - private volatile Set myAdditionalPackages = null; + private volatile List myAdditionalPackages = null; /** * Contains cached package information retrieved through PyPI's JSON API. @@ -101,13 +99,6 @@ public class PyPIPackageUtil { } }); - /** - * Lowercased package names for fast check that some package is available in PyPI. - * TODO find the way to get rid of it, it's not a good idea to store 85k+ entries in memory twice - */ - @Nullable private volatile Set myPackageNames = null; - - /** * Prevents simultaneous updates of {@link PyPackageService#PY_PACKAGES} * because the corresponding response contains tons of data and multiple @@ -161,15 +152,20 @@ public class PyPIPackageUtil { } @NotNull - public Set getAdditionalPackages() throws IOException { - if (myAdditionalPackages == null) { - final Set packages = new TreeSet<>(); + public List getAdditionalPackages() { + return myAdditionalPackages != null ? Collections.unmodifiableList(myAdditionalPackages) : Collections.emptyList(); + } + + @NotNull + public List loadAndGetAdditionalPackages(boolean alwaysRefresh) throws IOException { + if (myAdditionalPackages == null || alwaysRefresh) { + final Set uniquePackages = new TreeSet<>(); for (String url : PyPackageService.getInstance().additionalRepositories) { - packages.addAll(getPackagesFromAdditionalRepository(url)); + uniquePackages.addAll(getPackagesFromAdditionalRepository(url)); } - myAdditionalPackages = packages; + myAdditionalPackages = new ArrayList<>(uniquePackages); } - return Collections.unmodifiableSet(myAdditionalPackages); + return Collections.unmodifiableList(myAdditionalPackages); } @NotNull @@ -203,11 +199,6 @@ public class PyPIPackageUtil { return result; } - public void clearPackagesCache() { - PyPackageService.getInstance().PY_PACKAGES.clear(); - myAdditionalPackages = null; - } - public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer callback) { ApplicationManager.getApplication().executeOnPooledThread(() -> { try { @@ -262,9 +253,7 @@ public class PyPIPackageUtil { private String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException { LOG.debug("Requesting the latest PyPI version for the package " + packageName); final List versions = getPackageVersionsFromPyPI(packageName, true); - final String latest = ContainerUtil.getFirstItem(versions); - getPyPIPackages().put(packageName, StringUtil.notNullize(latest)); - return latest; + return ContainerUtil.getFirstItem(versions); } /** @@ -295,9 +284,9 @@ public class PyPIPackageUtil { @Nullable public String fetchLatestPackageVersion(@NotNull String packageName) throws IOException { - String version = getPyPIPackages().get(packageName); - // Package is on PyPI but it's version is unknown - if (version != null && version.isEmpty()) { + String version = null; + // Package is on PyPI not a, say, some system package on Ubuntu + if (PyPIPackageCache.getInstance().containsPackage(packageName)) { version = getLatestPackageVersionFromPyPI(packageName); } if (!PyPackageService.getInstance().additionalRepositories.isEmpty()) { @@ -349,27 +338,29 @@ public class PyPIPackageUtil { return repository + suffix; } - public void updatePyPICache(@NotNull PyPackageService service) throws IOException { + public void updatePyPICache() throws IOException { + final PyPackageService service = PyPackageService.getInstance(); service.LAST_TIME_CHECKED = System.currentTimeMillis(); - - service.PY_PACKAGES.clear(); if (service.PYPI_REMOVED) return; - parsePyPIList(parsePyPIListFromWeb(PYPI_LIST_URL, true), service); + final List decodedNames = parsePyPIList(parsePyPIListFromWeb(PYPI_LIST_URL, true)); + PyPIPackageCache.reload(decodedNames); } - private void parsePyPIList(@NotNull List packages, @NotNull PyPackageService service) { - myPackageNames = null; + @NotNull + private static List parsePyPIList(@NotNull List packages) { + final List decodedNames = new ArrayList<>(); for (String pyPackage : packages) { try { final String packageName = URLDecoder.decode(pyPackage, "UTF-8"); if (!packageName.contains(" ")) { - service.PY_PACKAGES.put(packageName, ""); + decodedNames.add(packageName); } } catch (UnsupportedEncodingException e) { LOG.warn(e.getMessage()); } } + return decodedNames; } @NotNull @@ -418,37 +409,22 @@ public class PyPIPackageUtil { }); } - @NotNull - public Collection getPackageNames() { - final Map pyPIPackages = getPyPIPackages(); - final ArrayList list = Lists.newArrayList(pyPIPackages.keySet()); - Collections.sort(list); - return list; - } - - @NotNull - public Map loadAndGetPackages() throws IOException { - // The map returned by getPyPIPackages() is already thread-safe; - // this lock is solely to prevent multiple threads from updating + public void loadAndGetPackages() throws IOException { + // This lock is solely to prevent multiple threads from updating // the mammoth cache of PyPI packages simultaneously. synchronized (myPyPIPackageCacheUpdateLock) { - if (getPyPIPackages().isEmpty()) { - updatePyPICache(PyPackageService.getInstance()); + final PyPIPackageCache cache = PyPIPackageCache.getInstance(); + if (cache.getPackageNames().isEmpty()) { + updatePyPICache(); } - return getPyPIPackages(); } } - @NotNull - public static Map getPyPIPackages() { - return PyPackageService.getInstance().PY_PACKAGES; - } - + /** + * @see PyPIPackageCache#containsPackage(String) + */ public boolean isInPyPI(@NotNull String packageName) { - if (myPackageNames == null) { - myPackageNames = getPyPIPackages().keySet().stream().map(name -> name.toLowerCase(Locale.ENGLISH)).collect(Collectors.toSet()); - } - return myPackageNames != null && myPackageNames.contains(packageName.toLowerCase(Locale.ENGLISH)); + return PyPIPackageCache.getInstance().containsPackage(packageName); } @SuppressWarnings("FieldMayBeFinal") diff --git a/python/src/com/jetbrains/python/packaging/PyPackageService.java b/python/src/com/jetbrains/python/packaging/PyPackageService.java index df7e1e4f5c88..7cff6f2d22c0 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackageService.java +++ b/python/src/com/jetbrains/python/packaging/PyPackageService.java @@ -30,10 +30,9 @@ public class PyPackageService implements PersistentStateComponent { public volatile Map sdkToUsersite = ContainerUtil.newConcurrentMap(); public volatile List additionalRepositories = ContainerUtil.createConcurrentList(); - public volatile Map PY_PACKAGES = ContainerUtil.newConcurrentMap(); @SystemIndependent public volatile String virtualEnvBasePath; public volatile Boolean PYPI_REMOVED = false; - + public long LAST_TIME_CHECKED = 0; @Override diff --git a/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java b/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java index 97748515fda6..c44be4e1752f 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java +++ b/python/src/com/jetbrains/python/packaging/PyPackagesUpdater.java @@ -46,9 +46,7 @@ public class PyPackagesUpdater implements StartupActivity { if (checkNeeded(project)) { application.executeOnPooledThread(() -> { try { - final PyPackageService service = PyPackageService.getInstance(); - PyPIPackageUtil.INSTANCE.updatePyPICache(service); - service.LAST_TIME_CHECKED = System.currentTimeMillis(); + PyPIPackageUtil.INSTANCE.updatePyPICache(); } catch (IOException e) { LOG.warn(e.getMessage()); diff --git a/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java b/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java index 9d26b64990f9..004f4f396eeb 100644 --- a/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java +++ b/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java @@ -27,15 +27,14 @@ import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.CatchingConsumer; import com.intellij.webcore.packaging.PackageVersionComparator; import com.intellij.webcore.packaging.RepoPackage; -import com.jetbrains.python.packaging.PyCondaPackageManagerImpl; -import com.jetbrains.python.packaging.PyCondaPackageService; -import com.jetbrains.python.packaging.PyPackageManager; +import com.jetbrains.python.packaging.*; import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; public class PyCondaManagementService extends PyPackageManagementService { private static final Logger LOG = Logger.getInstance(PyCondaManagementService.class); @@ -141,4 +140,15 @@ public class PyCondaManagementService extends PyPackageManagementService { super.fetchPackageVersions(packageName, consumer); } } + + @NotNull + protected static List versionMapToPackageList(@NotNull Map packageToVersionMap) { + final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty(); + final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : ""; + final List packages = new ArrayList<>(); + for (Map.Entry entry : packageToVersionMap.entrySet()) { + packages.add(new RepoPackage(entry.getKey(), url, entry.getValue())); + } + return packages; + } } diff --git a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java index 0afa22e61baa..68e82d5de4b9 100644 --- a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java +++ b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java @@ -10,6 +10,7 @@ import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.CatchingConsumer; import com.intellij.util.concurrency.AppExecutorUtil; +import com.intellij.util.containers.ContainerUtil; import com.intellij.util.ui.JBUI; import com.intellij.webcore.packaging.InstalledPackage; import com.intellij.webcore.packaging.PackageManagementServiceEx; @@ -93,34 +94,36 @@ public class PyPackageManagementService extends PackageManagementServiceEx { @NotNull @Override public List getAllPackages() throws IOException { - final Map packageToVersionMap = PyPIPackageUtil.INSTANCE.loadAndGetPackages(); - final List packages = versionMapToPackageList(packageToVersionMap); - packages.addAll(PyPIPackageUtil.INSTANCE.getAdditionalPackages()); - return packages; - } - - @NotNull - protected static List versionMapToPackageList(@NotNull Map packageToVersionMap) { - final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty(); - final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : ""; - final List packages = new ArrayList<>(); - for (Map.Entry entry : packageToVersionMap.entrySet()) { - packages.add(new RepoPackage(entry.getKey(), url, entry.getValue())); - } - return packages; + PyPIPackageUtil.INSTANCE.loadAndGetPackages(); + PyPIPackageUtil.INSTANCE.loadAndGetAdditionalPackages(false); + return getAllPackagesCached(); } @NotNull @Override public List reloadAllPackages() throws IOException { - PyPIPackageUtil.INSTANCE.clearPackagesCache(); - return getAllPackages(); + PyPIPackageUtil.INSTANCE.updatePyPICache(); + PyPIPackageUtil.INSTANCE.loadAndGetAdditionalPackages(true); + return getAllPackagesCached(); } @NotNull @Override public List getAllPackagesCached() { - return versionMapToPackageList(PyPIPackageUtil.getPyPIPackages()); + final List pypiPackages = getCachedPyPIPackages(); + final List additionalPackages = PyPIPackageUtil.INSTANCE.getAdditionalPackages(); + // Make a copy, since ManagePackagesDialog attempts to change the passed collection directly + final List result = new ArrayList<>(pypiPackages); + result.addAll(additionalPackages); + return result; + } + + @NotNull + private static List getCachedPyPIPackages() { + // Don't show URL next to the package name in "Available Packages" if only PyPI is in use + final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty(); + final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : ""; + return ContainerUtil.map(PyPIPackageCache.getInstance().getPackageNames(), name -> new RepoPackage(name, url, null)); } @Override diff --git a/python/src/com/jetbrains/python/statistics/PyPackageUsagesCollector.java b/python/src/com/jetbrains/python/statistics/PyPackageUsagesCollector.java index c3e7f3251b43..b0632236c703 100644 --- a/python/src/com/jetbrains/python/statistics/PyPackageUsagesCollector.java +++ b/python/src/com/jetbrains/python/statistics/PyPackageUsagesCollector.java @@ -24,7 +24,7 @@ import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleManager; import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; -import com.jetbrains.python.packaging.PyPIPackageUtil; +import com.jetbrains.python.packaging.PyPIPackageCache; import com.jetbrains.python.packaging.PyPackageManager; import com.jetbrains.python.packaging.PyRequirement; import com.jetbrains.python.sdk.PythonSdkType; @@ -51,7 +51,7 @@ public class PyPackageUsagesCollector extends AbstractProjectsUsagesCollector { ApplicationManager.getApplication().runReadAction(() -> { List requirements = PyPackageManager.getInstance(pythonSdk).getRequirements(m); if (requirements != null) { - Collection packages = new HashSet<>(PyPIPackageUtil.INSTANCE.getPackageNames()); + Collection packages = new HashSet<>(PyPIPackageCache.getInstance().getPackageNames()); for (PyRequirement requirement : requirements) { String name = requirement.getName(); if (packages.contains(name)) {