diff --git a/python/python-community.iml b/python/python-community.iml index e0aca64302e3..de44e2740069 100644 --- a/python/python-community.iml +++ b/python/python-community.iml @@ -23,6 +23,7 @@ + diff --git a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java index 20e066313647..133780a244b2 100644 --- a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java +++ b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java @@ -16,6 +16,9 @@ package com.jetbrains.python.packaging; import com.google.common.collect.Lists; +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.intellij.openapi.application.ApplicationInfo; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.ApplicationNamesInfo; @@ -26,31 +29,22 @@ import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.CatchingConsumer; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.io.HttpRequests; -import com.intellij.util.net.HttpConfigurable; import com.intellij.webcore.packaging.PackageVersionComparator; import com.intellij.webcore.packaging.RepoPackage; import com.jetbrains.python.PythonHelpersLocator; -import org.apache.xmlrpc.*; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; import javax.swing.text.MutableAttributeSet; import javax.swing.text.html.HTML; import javax.swing.text.html.HTMLEditorKit; import javax.swing.text.html.parser.ParserDelegator; -import java.io.*; -import java.net.MalformedURLException; -import java.net.URL; +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -59,9 +53,11 @@ import java.util.stream.Collectors; /** * User: catherine */ -@SuppressWarnings("UseOfObsoleteCollectionType") public class PyPIPackageUtil { private static final Logger LOG = Logger.getInstance(PyPIPackageUtil.class); + private static final Gson GSON = new GsonBuilder() + .setFieldNamingStrategy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .create(); private static final String PYPI_HOST = "https://pypi.python.org"; public static final String PYPI_URL = PYPI_HOST + "/pypi"; @@ -73,8 +69,7 @@ public class PyPIPackageUtil { private static final Set ourAdditionalPackageNames = new TreeSet<>(); public static final PyPIPackageUtil INSTANCE = new PyPIPackageUtil(); - private XmlRpcClient myXmlRpcClient; - private final Map packageToDetails = new HashMap<>(); + private final Map myPackageToDetails = new HashMap<>(); @Nullable private volatile Set myPackageNames = null; @@ -94,18 +89,6 @@ public class PyPIPackageUtil { */ private final Object myPyPIPackageCacheUpdateLock = new Object(); - - private PyPIPackageUtil() { - try { - final DefaultXmlRpcTransportFactory factory = new PyPIXmlRpcTransportFactory(new URL(PYPI_URL)); - factory.setProperty("timeout", 1000); - myXmlRpcClient = new XmlRpcClient(new URL(PYPI_URL), factory); - } - catch (MalformedURLException e) { - LOG.warn(e); - } - } - /** * Value for "User Agent" HTTP header in form: PyCharm/2016.2 EAP */ @@ -185,40 +168,30 @@ public class PyPIPackageUtil { ourAdditionalPackageNames.clear(); } - public void addPackageDetails(@NonNls String packageName, Hashtable details) { - packageToDetails.put(packageName, details); - } - - @Nullable - private Hashtable getPackageDetails(@NotNull String packageName) { - return packageToDetails.get(packageName); - } - - public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer callback) { - final Hashtable details = getPackageDetails(packageName); - if (details != null) { - callback.consume(details); - return; - } + public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer callback) { ApplicationManager.getApplication().executeOnPooledThread(() -> { - final Vector params = new Vector<>(); - params.add(packageName); try { - final String version = fetchLatestPackageVersion(packageName); - if (version != null) { - params.add(version); - final Object result = myXmlRpcClient.execute("release_data", params); - if (result != null) { - callback.consume((Hashtable)result); - } - } + final PackageDetails packageDetails = refreshAndGetPackageDetailsFromPyPI(packageName, false); + callback.consume(packageDetails.getInfo()); } - catch (Exception e) { + catch (IOException e) { callback.consume(e); } }); } + @NotNull + private PackageDetails refreshAndGetPackageDetailsFromPyPI(@NotNull String packageName, boolean alwaysRefresh) throws IOException { + PackageDetails details = myPackageToDetails.get(packageName); + if (alwaysRefresh || details == null) { + details = HttpRequests.request(PYPI_URL + "/" + packageName + "/json") + .userAgent(getUserAgent()) + .connect(request -> GSON.fromJson(request.getReader(), PackageDetails.class)); + myPackageToDetails.put(packageName, details); + } + return details; + } + public void addPackageReleases(@NotNull String packageName, @NotNull List releases) { ourPackageToReleases.put(packageName, releases); } @@ -226,19 +199,13 @@ public class PyPIPackageUtil { public void usePackageReleases(@NotNull String packageName, @NotNull CatchingConsumer, Exception> callback) { ApplicationManager.getApplication().executeOnPooledThread(() -> { try { - final List releases = getPackageVersionsFromAdditionalRepositories(packageName); - if (releases == null) { - final Vector params = new Vector<>(); - params.add(packageName); - params.add("show_hidden=True"); - final Object result = myXmlRpcClient.execute("package_releases", params); - if (result != null) { - //noinspection unchecked - callback.consume((List)result); - } + final List releasesFromSimpleIndex = getPackageVersionsFromAdditionalRepositories(packageName); + if (releasesFromSimpleIndex == null) { + final List releasesFromPyPI = getPackageVersionsFromPyPI(packageName, true); + callback.consume(releasesFromPyPI); } else { - callback.consume(releases); + callback.consume(releasesFromSimpleIndex); } } catch (Exception e) { @@ -247,6 +214,31 @@ public class PyPIPackageUtil { }); } + /** + * Fetches available package versions using JSON API of PyPI. + */ + @NotNull + private List getPackageVersionsFromPyPI(@NotNull String packageName, + boolean force) throws IOException { + final PackageDetails details = refreshAndGetPackageDetailsFromPyPI(packageName, force); + final List result = details.getReleases(); + result.sort(PackageVersionComparator.VERSION_COMPARATOR.reversed()); + return Collections.unmodifiableList(result); + } + + @Nullable + private String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException { + LOG.debug("Requesting the latest PyPI version for the package " + packageName); + final List versions = getPackageVersionsFromPyPI(packageName, true); + final String latest = ContainerUtil.getFirstItem(versions); + getPyPIPackages().put(packageName, StringUtil.notNullize(latest)); + return latest; + } + + /** + * Fetches available package versions by scrapping the page containing package archives. + * It's primarily used for additional repositories since, e.g. devpi doesn't provide another way to get this information. + */ @Nullable private static List getPackageVersionsFromAdditionalRepositories(@NotNull @NonNls String packageName) throws IOException { if (ourPackageToReleases.containsKey(packageName)) { @@ -254,7 +246,7 @@ public class PyPIPackageUtil { } final List repositories = PyPackageService.getInstance().additionalRepositories; for (String repository : repositories) { - final List versions = getVersionsFromRepository(packageName, repository); + final List versions = parsePackageVersionsFromArchives(composeSimpleUrl(packageName, repository)); if (!versions.isEmpty()) { ourPackageToReleases.put(packageName, versions); return versions; @@ -269,28 +261,8 @@ public class PyPIPackageUtil { return ContainerUtil.getFirstItem(versions); } - @NotNull - private static List getPackageVersionsFromPyPI(@NotNull String packageName) throws IOException { - return getVersionsFromRepository(packageName, PYPI_LIST_URL); - } - @Nullable - private static String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException { - LOG.debug("Requesting the latest PyPI version for the package " + packageName); - final List versions = getPackageVersionsFromPyPI(packageName); - final String latest = ContainerUtil.getFirstItem(versions); - getPyPIPackages().put(packageName, StringUtil.notNullize(latest)); - return latest; - } - - @NotNull - private static List getVersionsFromRepository(@NotNull String packageName, @NotNull String repository) throws IOException { - final String packageArchivesSimpleUrl = composeSimpleUrl(packageName, repository); - return parsePackageVersionsFromArchives(packageArchivesSimpleUrl); - } - - @Nullable - public static String fetchLatestPackageVersion(@NotNull String packageName) throws IOException { + public String fetchLatestPackageVersion(@NotNull String packageName) throws IOException { String version = getPyPIPackages().get(packageName); // Package is on PyPI but it's version is unknown if (version != null && version.isEmpty()) { @@ -443,67 +415,54 @@ public class PyPIPackageUtil { return myPackageNames != null && myPackageNames.contains(packageName.toLowerCase(Locale.ENGLISH)); } - private static class PyPIXmlRpcTransport extends DefaultXmlRpcTransport { - public PyPIXmlRpcTransport(URL url) { - super(url); + @SuppressWarnings("FieldMayBeFinal") + public static final class PackageDetails { + public static final class Info { + private String version = ""; + private String author = ""; + + private String authorEmail = ""; + private String homePage = ""; + private String summary = ""; + + + @NotNull + public String getVersion() { + return version; + } + + @NotNull + public String getAuthor() { + return author; + } + + @NotNull + public String getAuthorEmail() { + return authorEmail; + } + + @NotNull + public String getHomePage() { + return homePage; + } + + @NotNull + public String getSummary() { + return summary; + } } - @Override - public InputStream sendXmlRpc(@NotNull byte[] request) throws IOException { - // Create a trust manager that does not validate certificate for this connection - final TrustManager[] trustAllCerts = new TrustManager[]{new PyPITrustManager()}; + private Info info = new Info(); + private Map releases = Collections.emptyMap(); - try { - final SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, trustAllCerts, new SecureRandom()); - - final HttpConfigurable settings = HttpConfigurable.getInstance(); - con = settings.openConnection(PYPI_HOST + "/pypi?%3Aaction=index"); - if (con instanceof HttpsURLConnection) { - ((HttpsURLConnection)con).setSSLSocketFactory(sslContext.getSocketFactory()); - } - con.setDoInput(true); - con.setDoOutput(true); - con.setUseCaches(false); - con.setAllowUserInteraction(false); - con.setRequestProperty("Content-Length", - Integer.toString(request.length)); - con.setRequestProperty("Content-Type", "text/xml"); - if (auth != null) - { - con.setRequestProperty("Authorization", "Basic " + auth); - } - final OutputStream out = con.getOutputStream(); - out.write(request); - out.flush(); - out.close(); - return con.getInputStream(); - } - catch (@NotNull NoSuchAlgorithmException | KeyManagementException e) { - LOG.warn(e.getMessage()); - } - return super.sendXmlRpc(request); - } - } - - private static class PyPIXmlRpcTransportFactory extends DefaultXmlRpcTransportFactory { - public PyPIXmlRpcTransportFactory(URL url) { - super(url); + @NotNull + public Info getInfo() { + return info; } @NotNull - @Override - public XmlRpcTransport createTransport() throws XmlRpcClientException { - return new PyPIXmlRpcTransport(url); + public List getReleases() { + return new ArrayList<>(releases.keySet()); } } - - private static class PyPITrustManager implements X509TrustManager { - @Override - public X509Certificate[] getAcceptedIssuers(){return null;} - @Override - public void checkClientTrusted(X509Certificate[] certs, String authType){} - @Override - public void checkServerTrusted(X509Certificate[] certs, String authType){} - } } diff --git a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java index bacbed9e8d05..c10bb098b3e5 100644 --- a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java +++ b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java @@ -28,6 +28,7 @@ import com.intellij.webcore.packaging.InstalledPackage; import com.intellij.webcore.packaging.PackageManagementServiceEx; import com.intellij.webcore.packaging.RepoPackage; import com.jetbrains.python.packaging.*; +import com.jetbrains.python.packaging.PyPIPackageUtil.PackageDetails; import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.sdk.PySdkUtil; import com.jetbrains.python.sdk.PythonSdkType; @@ -43,7 +44,6 @@ import java.util.regex.Pattern; /** * @author yole */ -@SuppressWarnings("UseOfObsoleteCollectionType") public class PyPackageManagementService extends PackageManagementServiceEx { @NotNull private static final Pattern PATTERN_ERROR_LINE = Pattern.compile(".*error:.*", Pattern.CASE_INSENSITIVE); @NonNls private static final String TEXT_PREFIX = "" + @@ -255,11 +255,10 @@ public class PyPackageManagementService extends PackageManagementServiceEx { @Override public void fetchPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer consumer) { - PyPIPackageUtil.INSTANCE.fillPackageDetails(packageName, new CatchingConsumer() { + PyPIPackageUtil.INSTANCE.fillPackageDetails(packageName, new CatchingConsumer() { @Override - public void consume(Hashtable details) { - PyPIPackageUtil.INSTANCE.addPackageDetails(packageName, details); - consumer.consume(formatPackageDetails(details)); + public void consume(PackageDetails.Info details) { + consumer.consume(formatPackageInfo(details)); } @Override @@ -269,31 +268,31 @@ public class PyPackageManagementService extends PackageManagementServiceEx { }); } - private static String formatPackageDetails(@NotNull Hashtable details) { - final Object description = details.get("summary"); + private static String formatPackageInfo(@NotNull PackageDetails.Info info) { final StringBuilder stringBuilder = new StringBuilder(TEXT_PREFIX); - if (description instanceof String) { + final String description = info.getSummary(); + if (StringUtil.isNotEmpty(description)) { stringBuilder.append(description).append("
"); } - final Object version = details.get("version"); - if (version instanceof String && !StringUtil.isEmpty((String)version)) { + final String version = info.getVersion(); + if (StringUtil.isNotEmpty(version)) { stringBuilder.append("

Version

"); stringBuilder.append(version); } - final Object author = details.get("author"); - if (author instanceof String && !StringUtil.isEmpty((String)author)) { + final String author = info.getAuthor(); + if (StringUtil.isNotEmpty(author)) { stringBuilder.append("

Author

"); stringBuilder.append(author).append("

"); } - final Object authorEmail = details.get("author_email"); - if (authorEmail instanceof String && !StringUtil.isEmpty((String)authorEmail)) { + final String authorEmail = info.getAuthorEmail(); + if (StringUtil.isNotEmpty(authorEmail)) { stringBuilder.append("
"); stringBuilder.append(composeHref("mailto:" + authorEmail)); } - final Object homePage = details.get("home_page"); - if (homePage instanceof String && !StringUtil.isEmpty((String)homePage)) { + final String homePage = info.getHomePage(); + if (StringUtil.isNotEmpty(homePage)) { stringBuilder.append("
"); - stringBuilder.append(composeHref((String)homePage)); + stringBuilder.append(composeHref(homePage)); } stringBuilder.append(TEXT_SUFFIX); return stringBuilder.toString(); @@ -389,7 +388,7 @@ public class PyPackageManagementService extends PackageManagementServiceEx { ApplicationManager.getApplication().executeOnPooledThread(() -> { try { PyPIPackageUtil.INSTANCE.loadAndGetPackages(); - final String version = PyPIPackageUtil.fetchLatestPackageVersion(pkg.getName()); + final String version = PyPIPackageUtil.INSTANCE.fetchLatestPackageVersion(pkg.getName()); consumer.consume(StringUtil.notNullize(version)); } catch (IOException e) {