diff --git a/python/python-community.iml b/python/python-community.iml
index e0aca64302e3..de44e2740069 100644
--- a/python/python-community.iml
+++ b/python/python-community.iml
@@ -23,6 +23,7 @@
+
diff --git a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java
index 20e066313647..133780a244b2 100644
--- a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java
+++ b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java
@@ -16,6 +16,9 @@
package com.jetbrains.python.packaging;
import com.google.common.collect.Lists;
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import com.intellij.openapi.application.ApplicationInfo;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.application.ApplicationNamesInfo;
@@ -26,31 +29,22 @@ import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.CatchingConsumer;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.io.HttpRequests;
-import com.intellij.util.net.HttpConfigurable;
import com.intellij.webcore.packaging.PackageVersionComparator;
import com.intellij.webcore.packaging.RepoPackage;
import com.jetbrains.python.PythonHelpersLocator;
-import org.apache.xmlrpc.*;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
import javax.swing.text.MutableAttributeSet;
import javax.swing.text.html.HTML;
import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.parser.ParserDelegator;
-import java.io.*;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
-import java.security.KeyManagementException;
-import java.security.NoSuchAlgorithmException;
-import java.security.SecureRandom;
-import java.security.cert.X509Certificate;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,9 +53,11 @@ import java.util.stream.Collectors;
/**
* User: catherine
*/
-@SuppressWarnings("UseOfObsoleteCollectionType")
public class PyPIPackageUtil {
private static final Logger LOG = Logger.getInstance(PyPIPackageUtil.class);
+ private static final Gson GSON = new GsonBuilder()
+ .setFieldNamingStrategy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
+ .create();
private static final String PYPI_HOST = "https://pypi.python.org";
public static final String PYPI_URL = PYPI_HOST + "/pypi";
@@ -73,8 +69,7 @@ public class PyPIPackageUtil {
private static final Set ourAdditionalPackageNames = new TreeSet<>();
public static final PyPIPackageUtil INSTANCE = new PyPIPackageUtil();
- private XmlRpcClient myXmlRpcClient;
- private final Map packageToDetails = new HashMap<>();
+ private final Map myPackageToDetails = new HashMap<>();
@Nullable private volatile Set myPackageNames = null;
@@ -94,18 +89,6 @@ public class PyPIPackageUtil {
*/
private final Object myPyPIPackageCacheUpdateLock = new Object();
-
- private PyPIPackageUtil() {
- try {
- final DefaultXmlRpcTransportFactory factory = new PyPIXmlRpcTransportFactory(new URL(PYPI_URL));
- factory.setProperty("timeout", 1000);
- myXmlRpcClient = new XmlRpcClient(new URL(PYPI_URL), factory);
- }
- catch (MalformedURLException e) {
- LOG.warn(e);
- }
- }
-
/**
* Value for "User Agent" HTTP header in form: PyCharm/2016.2 EAP
*/
@@ -185,40 +168,30 @@ public class PyPIPackageUtil {
ourAdditionalPackageNames.clear();
}
- public void addPackageDetails(@NonNls String packageName, Hashtable details) {
- packageToDetails.put(packageName, details);
- }
-
- @Nullable
- private Hashtable getPackageDetails(@NotNull String packageName) {
- return packageToDetails.get(packageName);
- }
-
- public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer callback) {
- final Hashtable details = getPackageDetails(packageName);
- if (details != null) {
- callback.consume(details);
- return;
- }
+ public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer callback) {
ApplicationManager.getApplication().executeOnPooledThread(() -> {
- final Vector params = new Vector<>();
- params.add(packageName);
try {
- final String version = fetchLatestPackageVersion(packageName);
- if (version != null) {
- params.add(version);
- final Object result = myXmlRpcClient.execute("release_data", params);
- if (result != null) {
- callback.consume((Hashtable)result);
- }
- }
+ final PackageDetails packageDetails = refreshAndGetPackageDetailsFromPyPI(packageName, false);
+ callback.consume(packageDetails.getInfo());
}
- catch (Exception e) {
+ catch (IOException e) {
callback.consume(e);
}
});
}
+ @NotNull
+ private PackageDetails refreshAndGetPackageDetailsFromPyPI(@NotNull String packageName, boolean alwaysRefresh) throws IOException {
+ PackageDetails details = myPackageToDetails.get(packageName);
+ if (alwaysRefresh || details == null) {
+ details = HttpRequests.request(PYPI_URL + "/" + packageName + "/json")
+ .userAgent(getUserAgent())
+ .connect(request -> GSON.fromJson(request.getReader(), PackageDetails.class));
+ myPackageToDetails.put(packageName, details);
+ }
+ return details;
+ }
+
public void addPackageReleases(@NotNull String packageName, @NotNull List releases) {
ourPackageToReleases.put(packageName, releases);
}
@@ -226,19 +199,13 @@ public class PyPIPackageUtil {
public void usePackageReleases(@NotNull String packageName, @NotNull CatchingConsumer, Exception> callback) {
ApplicationManager.getApplication().executeOnPooledThread(() -> {
try {
- final List releases = getPackageVersionsFromAdditionalRepositories(packageName);
- if (releases == null) {
- final Vector params = new Vector<>();
- params.add(packageName);
- params.add("show_hidden=True");
- final Object result = myXmlRpcClient.execute("package_releases", params);
- if (result != null) {
- //noinspection unchecked
- callback.consume((List)result);
- }
+ final List releasesFromSimpleIndex = getPackageVersionsFromAdditionalRepositories(packageName);
+ if (releasesFromSimpleIndex == null) {
+ final List releasesFromPyPI = getPackageVersionsFromPyPI(packageName, true);
+ callback.consume(releasesFromPyPI);
}
else {
- callback.consume(releases);
+ callback.consume(releasesFromSimpleIndex);
}
}
catch (Exception e) {
@@ -247,6 +214,31 @@ public class PyPIPackageUtil {
});
}
+ /**
+ * Fetches available package versions using JSON API of PyPI.
+ */
+ @NotNull
+ private List getPackageVersionsFromPyPI(@NotNull String packageName,
+ boolean force) throws IOException {
+ final PackageDetails details = refreshAndGetPackageDetailsFromPyPI(packageName, force);
+ final List result = details.getReleases();
+ result.sort(PackageVersionComparator.VERSION_COMPARATOR.reversed());
+ return Collections.unmodifiableList(result);
+ }
+
+ @Nullable
+ private String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException {
+ LOG.debug("Requesting the latest PyPI version for the package " + packageName);
+ final List versions = getPackageVersionsFromPyPI(packageName, true);
+ final String latest = ContainerUtil.getFirstItem(versions);
+ getPyPIPackages().put(packageName, StringUtil.notNullize(latest));
+ return latest;
+ }
+
+ /**
+ * Fetches available package versions by scrapping the page containing package archives.
+ * It's primarily used for additional repositories since, e.g. devpi doesn't provide another way to get this information.
+ */
@Nullable
private static List getPackageVersionsFromAdditionalRepositories(@NotNull @NonNls String packageName) throws IOException {
if (ourPackageToReleases.containsKey(packageName)) {
@@ -254,7 +246,7 @@ public class PyPIPackageUtil {
}
final List repositories = PyPackageService.getInstance().additionalRepositories;
for (String repository : repositories) {
- final List versions = getVersionsFromRepository(packageName, repository);
+ final List versions = parsePackageVersionsFromArchives(composeSimpleUrl(packageName, repository));
if (!versions.isEmpty()) {
ourPackageToReleases.put(packageName, versions);
return versions;
@@ -269,28 +261,8 @@ public class PyPIPackageUtil {
return ContainerUtil.getFirstItem(versions);
}
- @NotNull
- private static List getPackageVersionsFromPyPI(@NotNull String packageName) throws IOException {
- return getVersionsFromRepository(packageName, PYPI_LIST_URL);
- }
-
@Nullable
- private static String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException {
- LOG.debug("Requesting the latest PyPI version for the package " + packageName);
- final List versions = getPackageVersionsFromPyPI(packageName);
- final String latest = ContainerUtil.getFirstItem(versions);
- getPyPIPackages().put(packageName, StringUtil.notNullize(latest));
- return latest;
- }
-
- @NotNull
- private static List getVersionsFromRepository(@NotNull String packageName, @NotNull String repository) throws IOException {
- final String packageArchivesSimpleUrl = composeSimpleUrl(packageName, repository);
- return parsePackageVersionsFromArchives(packageArchivesSimpleUrl);
- }
-
- @Nullable
- public static String fetchLatestPackageVersion(@NotNull String packageName) throws IOException {
+ public String fetchLatestPackageVersion(@NotNull String packageName) throws IOException {
String version = getPyPIPackages().get(packageName);
// Package is on PyPI but it's version is unknown
if (version != null && version.isEmpty()) {
@@ -443,67 +415,54 @@ public class PyPIPackageUtil {
return myPackageNames != null && myPackageNames.contains(packageName.toLowerCase(Locale.ENGLISH));
}
- private static class PyPIXmlRpcTransport extends DefaultXmlRpcTransport {
- public PyPIXmlRpcTransport(URL url) {
- super(url);
+ @SuppressWarnings("FieldMayBeFinal")
+ public static final class PackageDetails {
+ public static final class Info {
+ private String version = "";
+ private String author = "";
+
+ private String authorEmail = "";
+ private String homePage = "";
+ private String summary = "";
+
+
+ @NotNull
+ public String getVersion() {
+ return version;
+ }
+
+ @NotNull
+ public String getAuthor() {
+ return author;
+ }
+
+ @NotNull
+ public String getAuthorEmail() {
+ return authorEmail;
+ }
+
+ @NotNull
+ public String getHomePage() {
+ return homePage;
+ }
+
+ @NotNull
+ public String getSummary() {
+ return summary;
+ }
}
- @Override
- public InputStream sendXmlRpc(@NotNull byte[] request) throws IOException {
- // Create a trust manager that does not validate certificate for this connection
- final TrustManager[] trustAllCerts = new TrustManager[]{new PyPITrustManager()};
+ private Info info = new Info();
+ private Map releases = Collections.emptyMap();
- try {
- final SSLContext sslContext = SSLContext.getInstance("TLS");
- sslContext.init(null, trustAllCerts, new SecureRandom());
-
- final HttpConfigurable settings = HttpConfigurable.getInstance();
- con = settings.openConnection(PYPI_HOST + "/pypi?%3Aaction=index");
- if (con instanceof HttpsURLConnection) {
- ((HttpsURLConnection)con).setSSLSocketFactory(sslContext.getSocketFactory());
- }
- con.setDoInput(true);
- con.setDoOutput(true);
- con.setUseCaches(false);
- con.setAllowUserInteraction(false);
- con.setRequestProperty("Content-Length",
- Integer.toString(request.length));
- con.setRequestProperty("Content-Type", "text/xml");
- if (auth != null)
- {
- con.setRequestProperty("Authorization", "Basic " + auth);
- }
- final OutputStream out = con.getOutputStream();
- out.write(request);
- out.flush();
- out.close();
- return con.getInputStream();
- }
- catch (@NotNull NoSuchAlgorithmException | KeyManagementException e) {
- LOG.warn(e.getMessage());
- }
- return super.sendXmlRpc(request);
- }
- }
-
- private static class PyPIXmlRpcTransportFactory extends DefaultXmlRpcTransportFactory {
- public PyPIXmlRpcTransportFactory(URL url) {
- super(url);
+ @NotNull
+ public Info getInfo() {
+ return info;
}
@NotNull
- @Override
- public XmlRpcTransport createTransport() throws XmlRpcClientException {
- return new PyPIXmlRpcTransport(url);
+ public List getReleases() {
+ return new ArrayList<>(releases.keySet());
}
}
-
- private static class PyPITrustManager implements X509TrustManager {
- @Override
- public X509Certificate[] getAcceptedIssuers(){return null;}
- @Override
- public void checkClientTrusted(X509Certificate[] certs, String authType){}
- @Override
- public void checkServerTrusted(X509Certificate[] certs, String authType){}
- }
}
diff --git a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java
index bacbed9e8d05..c10bb098b3e5 100644
--- a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java
+++ b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java
@@ -28,6 +28,7 @@ import com.intellij.webcore.packaging.InstalledPackage;
import com.intellij.webcore.packaging.PackageManagementServiceEx;
import com.intellij.webcore.packaging.RepoPackage;
import com.jetbrains.python.packaging.*;
+import com.jetbrains.python.packaging.PyPIPackageUtil.PackageDetails;
import com.jetbrains.python.psi.LanguageLevel;
import com.jetbrains.python.sdk.PySdkUtil;
import com.jetbrains.python.sdk.PythonSdkType;
@@ -43,7 +44,6 @@ import java.util.regex.Pattern;
/**
* @author yole
*/
-@SuppressWarnings("UseOfObsoleteCollectionType")
public class PyPackageManagementService extends PackageManagementServiceEx {
@NotNull private static final Pattern PATTERN_ERROR_LINE = Pattern.compile(".*error:.*", Pattern.CASE_INSENSITIVE);
@NonNls private static final String TEXT_PREFIX = "" +
@@ -255,11 +255,10 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
@Override
public void fetchPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer consumer) {
- PyPIPackageUtil.INSTANCE.fillPackageDetails(packageName, new CatchingConsumer() {
+ PyPIPackageUtil.INSTANCE.fillPackageDetails(packageName, new CatchingConsumer() {
@Override
- public void consume(Hashtable details) {
- PyPIPackageUtil.INSTANCE.addPackageDetails(packageName, details);
- consumer.consume(formatPackageDetails(details));
+ public void consume(PackageDetails.Info details) {
+ consumer.consume(formatPackageInfo(details));
}
@Override
@@ -269,31 +268,31 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
});
}
- private static String formatPackageDetails(@NotNull Hashtable details) {
- final Object description = details.get("summary");
+ private static String formatPackageInfo(@NotNull PackageDetails.Info info) {
final StringBuilder stringBuilder = new StringBuilder(TEXT_PREFIX);
- if (description instanceof String) {
+ final String description = info.getSummary();
+ if (StringUtil.isNotEmpty(description)) {
stringBuilder.append(description).append("
");
}
- final Object version = details.get("version");
- if (version instanceof String && !StringUtil.isEmpty((String)version)) {
+ final String version = info.getVersion();
+ if (StringUtil.isNotEmpty(version)) {
stringBuilder.append("Version
");
stringBuilder.append(version);
}
- final Object author = details.get("author");
- if (author instanceof String && !StringUtil.isEmpty((String)author)) {
+ final String author = info.getAuthor();
+ if (StringUtil.isNotEmpty(author)) {
stringBuilder.append("Author
");
stringBuilder.append(author).append("
");
}
- final Object authorEmail = details.get("author_email");
- if (authorEmail instanceof String && !StringUtil.isEmpty((String)authorEmail)) {
+ final String authorEmail = info.getAuthorEmail();
+ if (StringUtil.isNotEmpty(authorEmail)) {
stringBuilder.append("
");
stringBuilder.append(composeHref("mailto:" + authorEmail));
}
- final Object homePage = details.get("home_page");
- if (homePage instanceof String && !StringUtil.isEmpty((String)homePage)) {
+ final String homePage = info.getHomePage();
+ if (StringUtil.isNotEmpty(homePage)) {
stringBuilder.append("
");
- stringBuilder.append(composeHref((String)homePage));
+ stringBuilder.append(composeHref(homePage));
}
stringBuilder.append(TEXT_SUFFIX);
return stringBuilder.toString();
@@ -389,7 +388,7 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
ApplicationManager.getApplication().executeOnPooledThread(() -> {
try {
PyPIPackageUtil.INSTANCE.loadAndGetPackages();
- final String version = PyPIPackageUtil.fetchLatestPackageVersion(pkg.getName());
+ final String version = PyPIPackageUtil.INSTANCE.fetchLatestPackageVersion(pkg.getName());
consumer.consume(StringUtil.notNullize(version));
}
catch (IOException e) {