From e93970a179be6bce42b7430a632ecd18ce2e14bc Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Wed, 6 Dec 2017 17:41:58 +0300 Subject: [PATCH] PY-27360 Initial implementation of a package cache for Conda --- .../python/packaging/CondaPackageCache.java | 112 ++++++++++++++++++ .../packaging/PyCondaPackageService.java | 50 ++------ .../ui/PyCondaManagementService.java | 44 ++++--- 3 files changed, 151 insertions(+), 55 deletions(-) create mode 100644 python/src/com/jetbrains/python/packaging/CondaPackageCache.java diff --git a/python/src/com/jetbrains/python/packaging/CondaPackageCache.java b/python/src/com/jetbrains/python/packaging/CondaPackageCache.java new file mode 100644 index 000000000000..6691c7ca850a --- /dev/null +++ b/python/src/com/jetbrains/python/packaging/CondaPackageCache.java @@ -0,0 +1,112 @@ +// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.jetbrains.python.packaging; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.diagnostic.Logger; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +/** + * @author Mikhail Golubev + */ +public class CondaPackageCache { + private static final Logger LOG = Logger.getInstance(PyPIPackageCache.class); + private static final Gson ourGson = new GsonBuilder().create(); + private static CondaPackageCache ourInstance = null; + + @NotNull + public static synchronized CondaPackageCache getInstance() { + if (ourInstance == null) { + ourInstance = new CondaPackageCache(); + try (Reader reader = Files.newBufferedReader(getCachePath(), StandardCharsets.UTF_8)) { + ourInstance = ourGson.fromJson(reader, CondaPackageCache.class); + LOG.info("Loaded " + ourInstance.myPackages.size() + " packages from " + getCachePath()); + } + catch (IOException exception) { + LOG.warn("Cannot load Conda package cache from the filesystem", exception); + } + } + return ourInstance; + } + + @NotNull + public static synchronized CondaPackageCache reload(@NotNull Map> packageNames) { + ourInstance = new CondaPackageCache(packageNames); + try { + Files.createDirectories(getCachePath().getParent()); + try (Writer writer = Files.newBufferedWriter(getCachePath(), StandardCharsets.UTF_8)) { + ourGson.toJson(ourInstance, writer); + } + } + catch (IOException exception) { + LOG.warn("Cannot save Conda package cache to the filesystem", exception); + } + return ourInstance; + } + + @NotNull + private static Path getCachePath() { + return Paths.get(PathManager.getSystemPath(), "python_packages", "conda-cache.json"); + } + + public CondaPackageCache() { + + } + + private CondaPackageCache(@NotNull Map> nameToVersion) { + for (Map.Entry> entry : nameToVersion.entrySet()) { + myPackages.put(entry.getKey(), new PackageInfo(entry.getValue())); + } + } + + public boolean containsPackage(@NotNull String packageName) { + return myPackages.containsKey(packageName); + } + + @NotNull + public Set getPackageNames() { + return Collections.unmodifiableSet(myPackages.keySet()); + } + + /** + * Returns available package versions sorted in the reversed order using {@link com.intellij.webcore.packaging.PackageVersionComparator}. + * @param packageName case-insensitive name of a package + */ + @NotNull + public List getVersions(@NotNull String packageName) { + return Collections.unmodifiableList(myPackages.get(packageName).getVersions()); + } + + @SerializedName("packages") + private final TreeMap myPackages = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + private static class PackageInfo { + + @SerializedName("v") + private List myVersions; + + public PackageInfo(@NotNull List versions) { + myVersions = versions; + } + + @SuppressWarnings("unused") + public PackageInfo() { + } + + @NotNull + public List getVersions() { + return Collections.unmodifiableList(myVersions); + } + } +} diff --git a/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java b/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java index a2cfb92fa49a..dd7adb4035c3 100644 --- a/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java +++ b/python/src/com/jetbrains/python/packaging/PyCondaPackageService.java @@ -40,10 +40,7 @@ import java.util.*; @State(name = "PyCondaPackageService", storages = @Storage(value="conda_packages.xml", roamingType = RoamingType.DISABLED)) public class PyCondaPackageService implements PersistentStateComponent { private static final Logger LOG = Logger.getInstance(PyCondaPackageService.class); - public Map CONDA_PACKAGES = ContainerUtil.newConcurrentMap(); - public Map> PACKAGES_TO_RELEASES = new HashMap<>(); public Set CONDA_CHANNELS = ContainerUtil.newConcurrentSet(); - public long LAST_TIME_CHECKED = 0; @Override @@ -60,15 +57,10 @@ public class PyCondaPackageService implements PersistentStateComponent getCondaPackages() { - return CONDA_PACKAGES; - } - - public Map loadAndGetPackages(boolean force) { - if (CONDA_PACKAGES.isEmpty() || force) { + public void loadAndGetPackages(boolean force) { + if (CondaPackageCache.getInstance().getPackageNames().isEmpty() || force) { updatePackagesCache(); } - return CONDA_PACKAGES; } public Set loadAndGetChannels() { @@ -189,42 +181,26 @@ public class PyCondaPackageService implements PersistentStateComponent lines = output.getStdoutLines(); - for (String line : lines) { + + final Map> nameToVersions = new HashMap<>(); + for (String line : output.getStdoutLines()) { final List split = StringUtil.split(line, "\t"); if (split.size() < 2) continue; - final String aPackage = CONDA_PACKAGES.get(split.get(0)); - if (aPackage != null) { - if (VersionComparatorUtil.compare(aPackage, split.get(1)) < 0) - CONDA_PACKAGES.put(split.get(0), split.get(1)); - } - else { - CONDA_PACKAGES.put(split.get(0), split.get(1)); - } - - if (PACKAGES_TO_RELEASES.containsKey(split.get(0))) { - final List versions = PACKAGES_TO_RELEASES.get(split.get(0)); - if (!versions.contains(split.get(1))) { - versions.add(split.get(1)); - } - } - else { - final ArrayList versions = new ArrayList<>(); - versions.add(split.get(1)); - PACKAGES_TO_RELEASES.put(split.get(0), versions); + final String packageName = split.get(0); + final String packageVersion = split.get(1); + final List versions = nameToVersions.computeIfAbsent(packageName, k -> new ArrayList<>()); + final int sortedIndex = Collections.binarySearch(versions, packageVersion, VersionComparatorUtil.COMPARATOR.reversed()); + if (sortedIndex < 0) { + versions.add(-(sortedIndex + 1), packageVersion); } } + CondaPackageCache.reload(nameToVersions); LAST_TIME_CHECKED = System.currentTimeMillis(); } @NotNull public List getPackageVersions(@NotNull final String packageName) { - if (PACKAGES_TO_RELEASES.containsKey(packageName)) { - return PACKAGES_TO_RELEASES.get(packageName); - } - return Collections.emptyList(); + return CondaPackageCache.getInstance().getVersions(packageName); } public void updateChannels() { diff --git a/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java b/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java index 004f4f396eeb..24b8ca2058fe 100644 --- a/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java +++ b/python/src/com/jetbrains/python/packaging/ui/PyCondaManagementService.java @@ -25,16 +25,17 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.CatchingConsumer; -import com.intellij.webcore.packaging.PackageVersionComparator; +import com.intellij.util.containers.ContainerUtil; import com.intellij.webcore.packaging.RepoPackage; -import com.jetbrains.python.packaging.*; +import com.jetbrains.python.packaging.CondaPackageCache; +import com.jetbrains.python.packaging.PyCondaPackageManagerImpl; +import com.jetbrains.python.packaging.PyCondaPackageService; +import com.jetbrains.python.packaging.PyPackageManager; import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; public class PyCondaManagementService extends PyPackageManagementService { private static final Logger LOG = Logger.getInstance(PyCondaManagementService.class); @@ -50,19 +51,30 @@ public class PyCondaManagementService extends PyPackageManagementService { @Override @NotNull public List getAllPackagesCached() { - return useConda() ? versionMapToPackageList(PyCondaPackageService.getInstance().getCondaPackages()) : super.getAllPackagesCached(); + if (useConda()) { + return getCachedCondaPackages(); + } + return super.getAllPackagesCached(); } @Override @NotNull public List getAllPackages() throws IOException { - return useConda() ? versionMapToPackageList(PyCondaPackageService.getInstance().loadAndGetPackages(false)) : super.getAllPackages(); + if (useConda()) { + PyCondaPackageService.getInstance().loadAndGetPackages(false); + return getAllPackagesCached(); + } + return super.getAllPackages(); } @Override @NotNull public List reloadAllPackages() throws IOException { - return useConda() ? versionMapToPackageList(PyCondaPackageService.getInstance().loadAndGetPackages(true)) : super.reloadAllPackages(); + if (useConda()) { + PyCondaPackageService.getInstance().loadAndGetPackages(true); + return getAllPackagesCached(); + } + return super.reloadAllPackages(); } @Override @@ -132,9 +144,7 @@ public class PyCondaManagementService extends PyPackageManagementService { @Override public void fetchPackageVersions(String packageName, CatchingConsumer, Exception> consumer) { if (useConda()) { - final List versions = PyCondaPackageService.getInstance().getPackageVersions(packageName); - Collections.sort(versions, Collections.reverseOrder(new PackageVersionComparator())); - consumer.consume(versions); + consumer.consume(PyCondaPackageService.getInstance().getPackageVersions(packageName)); } else { super.fetchPackageVersions(packageName, consumer); @@ -142,13 +152,11 @@ public class PyCondaManagementService extends PyPackageManagementService { } @NotNull - protected static List versionMapToPackageList(@NotNull Map packageToVersionMap) { - final boolean customRepoConfigured = !PyPackageService.getInstance().additionalRepositories.isEmpty(); - final String url = customRepoConfigured ? PyPIPackageUtil.PYPI_LIST_URL : ""; - final List packages = new ArrayList<>(); - for (Map.Entry entry : packageToVersionMap.entrySet()) { - packages.add(new RepoPackage(entry.getKey(), url, entry.getValue())); - } - return packages; + private static List getCachedCondaPackages() { + final CondaPackageCache instance = CondaPackageCache.getInstance(); + return ContainerUtil.map(instance.getPackageNames(), name -> { + final String latestVersion = ContainerUtil.getFirstItem(instance.getVersions(name)); + return new RepoPackage(name, null, latestVersion); + }); } }