mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-09-27 10:03:11 +07:00
PY-20102 Use PyPI's JSON REST API to get packages information and versions
This commit is contained in:
Mikhail Golubev
@@ -23,6 +23,7 @@
|
||||
<orderEntry type="module" module-name="spellchecker" />
|
||||
<orderEntry type="module" module-name="xdebugger-impl" />
|
||||
<orderEntry type="module" module-name="xml-psi-impl" />
|
||||
<orderEntry type="library" name="gson" level="project" />
|
||||
</component>
|
||||
</module>
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@
|
||||
package com.jetbrains.python.packaging;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.gson.FieldNamingPolicy;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.intellij.openapi.application.ApplicationInfo;
|
||||
import com.intellij.openapi.application.ApplicationManager;
|
||||
import com.intellij.openapi.application.ApplicationNamesInfo;
|
||||
@@ -26,31 +29,22 @@ import com.intellij.openapi.util.text.StringUtil;
|
||||
import com.intellij.util.CatchingConsumer;
|
||||
import com.intellij.util.containers.ContainerUtil;
|
||||
import com.intellij.util.io.HttpRequests;
|
||||
import com.intellij.util.net.HttpConfigurable;
|
||||
import com.intellij.webcore.packaging.PackageVersionComparator;
|
||||
import com.intellij.webcore.packaging.RepoPackage;
|
||||
import com.jetbrains.python.PythonHelpersLocator;
|
||||
import org.apache.xmlrpc.*;
|
||||
import org.jetbrains.annotations.NonNls;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import javax.net.ssl.HttpsURLConnection;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.TrustManager;
|
||||
import javax.net.ssl.X509TrustManager;
|
||||
import javax.swing.text.MutableAttributeSet;
|
||||
import javax.swing.text.html.HTML;
|
||||
import javax.swing.text.html.HTMLEditorKit;
|
||||
import javax.swing.text.html.parser.ParserDelegator;
|
||||
import java.io.*;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLDecoder;
|
||||
import java.security.KeyManagementException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -59,9 +53,11 @@ import java.util.stream.Collectors;
|
||||
/**
|
||||
* User: catherine
|
||||
*/
|
||||
@SuppressWarnings("UseOfObsoleteCollectionType")
|
||||
public class PyPIPackageUtil {
|
||||
private static final Logger LOG = Logger.getInstance(PyPIPackageUtil.class);
|
||||
private static final Gson GSON = new GsonBuilder()
|
||||
.setFieldNamingStrategy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
|
||||
.create();
|
||||
|
||||
private static final String PYPI_HOST = "https://pypi.python.org";
|
||||
public static final String PYPI_URL = PYPI_HOST + "/pypi";
|
||||
@@ -73,8 +69,7 @@ public class PyPIPackageUtil {
|
||||
private static final Set<RepoPackage> ourAdditionalPackageNames = new TreeSet<>();
|
||||
|
||||
public static final PyPIPackageUtil INSTANCE = new PyPIPackageUtil();
|
||||
private XmlRpcClient myXmlRpcClient;
|
||||
private final Map<String, Hashtable> packageToDetails = new HashMap<>();
|
||||
private final Map<String, PackageDetails> myPackageToDetails = new HashMap<>();
|
||||
@Nullable private volatile Set<String> myPackageNames = null;
|
||||
|
||||
|
||||
@@ -94,18 +89,6 @@ public class PyPIPackageUtil {
|
||||
*/
|
||||
private final Object myPyPIPackageCacheUpdateLock = new Object();
|
||||
|
||||
|
||||
private PyPIPackageUtil() {
|
||||
try {
|
||||
final DefaultXmlRpcTransportFactory factory = new PyPIXmlRpcTransportFactory(new URL(PYPI_URL));
|
||||
factory.setProperty("timeout", 1000);
|
||||
myXmlRpcClient = new XmlRpcClient(new URL(PYPI_URL), factory);
|
||||
}
|
||||
catch (MalformedURLException e) {
|
||||
LOG.warn(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Value for "User Agent" HTTP header in form: PyCharm/2016.2 EAP
|
||||
*/
|
||||
@@ -185,40 +168,30 @@ public class PyPIPackageUtil {
|
||||
ourAdditionalPackageNames.clear();
|
||||
}
|
||||
|
||||
public void addPackageDetails(@NonNls String packageName, Hashtable details) {
|
||||
packageToDetails.put(packageName, details);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private Hashtable getPackageDetails(@NotNull String packageName) {
|
||||
return packageToDetails.get(packageName);
|
||||
}
|
||||
|
||||
public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer<Hashtable, Exception> callback) {
|
||||
final Hashtable details = getPackageDetails(packageName);
|
||||
if (details != null) {
|
||||
callback.consume(details);
|
||||
return;
|
||||
}
|
||||
public void fillPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer<PackageDetails.Info, Exception> callback) {
|
||||
ApplicationManager.getApplication().executeOnPooledThread(() -> {
|
||||
final Vector<String> params = new Vector<>();
|
||||
params.add(packageName);
|
||||
try {
|
||||
final String version = fetchLatestPackageVersion(packageName);
|
||||
if (version != null) {
|
||||
params.add(version);
|
||||
final Object result = myXmlRpcClient.execute("release_data", params);
|
||||
if (result != null) {
|
||||
callback.consume((Hashtable)result);
|
||||
}
|
||||
}
|
||||
final PackageDetails packageDetails = refreshAndGetPackageDetailsFromPyPI(packageName, false);
|
||||
callback.consume(packageDetails.getInfo());
|
||||
}
|
||||
catch (Exception e) {
|
||||
catch (IOException e) {
|
||||
callback.consume(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private PackageDetails refreshAndGetPackageDetailsFromPyPI(@NotNull String packageName, boolean alwaysRefresh) throws IOException {
|
||||
PackageDetails details = myPackageToDetails.get(packageName);
|
||||
if (alwaysRefresh || details == null) {
|
||||
details = HttpRequests.request(PYPI_URL + "/" + packageName + "/json")
|
||||
.userAgent(getUserAgent())
|
||||
.connect(request -> GSON.fromJson(request.getReader(), PackageDetails.class));
|
||||
myPackageToDetails.put(packageName, details);
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
public void addPackageReleases(@NotNull String packageName, @NotNull List<String> releases) {
|
||||
ourPackageToReleases.put(packageName, releases);
|
||||
}
|
||||
@@ -226,19 +199,13 @@ public class PyPIPackageUtil {
|
||||
public void usePackageReleases(@NotNull String packageName, @NotNull CatchingConsumer<List<String>, Exception> callback) {
|
||||
ApplicationManager.getApplication().executeOnPooledThread(() -> {
|
||||
try {
|
||||
final List<String> releases = getPackageVersionsFromAdditionalRepositories(packageName);
|
||||
if (releases == null) {
|
||||
final Vector<String> params = new Vector<>();
|
||||
params.add(packageName);
|
||||
params.add("show_hidden=True");
|
||||
final Object result = myXmlRpcClient.execute("package_releases", params);
|
||||
if (result != null) {
|
||||
//noinspection unchecked
|
||||
callback.consume((List<String>)result);
|
||||
}
|
||||
final List<String> releasesFromSimpleIndex = getPackageVersionsFromAdditionalRepositories(packageName);
|
||||
if (releasesFromSimpleIndex == null) {
|
||||
final List<String> releasesFromPyPI = getPackageVersionsFromPyPI(packageName, true);
|
||||
callback.consume(releasesFromPyPI);
|
||||
}
|
||||
else {
|
||||
callback.consume(releases);
|
||||
callback.consume(releasesFromSimpleIndex);
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
@@ -247,6 +214,31 @@ public class PyPIPackageUtil {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches available package versions using JSON API of PyPI.
|
||||
*/
|
||||
@NotNull
|
||||
private List<String> getPackageVersionsFromPyPI(@NotNull String packageName,
|
||||
boolean force) throws IOException {
|
||||
final PackageDetails details = refreshAndGetPackageDetailsFromPyPI(packageName, force);
|
||||
final List<String> result = details.getReleases();
|
||||
result.sort(PackageVersionComparator.VERSION_COMPARATOR.reversed());
|
||||
return Collections.unmodifiableList(result);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException {
|
||||
LOG.debug("Requesting the latest PyPI version for the package " + packageName);
|
||||
final List<String> versions = getPackageVersionsFromPyPI(packageName, true);
|
||||
final String latest = ContainerUtil.getFirstItem(versions);
|
||||
getPyPIPackages().put(packageName, StringUtil.notNullize(latest));
|
||||
return latest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches available package versions by scrapping the page containing package archives.
|
||||
* It's primarily used for additional repositories since, e.g. devpi doesn't provide another way to get this information.
|
||||
*/
|
||||
@Nullable
|
||||
private static List<String> getPackageVersionsFromAdditionalRepositories(@NotNull @NonNls String packageName) throws IOException {
|
||||
if (ourPackageToReleases.containsKey(packageName)) {
|
||||
@@ -254,7 +246,7 @@ public class PyPIPackageUtil {
|
||||
}
|
||||
final List<String> repositories = PyPackageService.getInstance().additionalRepositories;
|
||||
for (String repository : repositories) {
|
||||
final List<String> versions = getVersionsFromRepository(packageName, repository);
|
||||
final List<String> versions = parsePackageVersionsFromArchives(composeSimpleUrl(packageName, repository));
|
||||
if (!versions.isEmpty()) {
|
||||
ourPackageToReleases.put(packageName, versions);
|
||||
return versions;
|
||||
@@ -269,28 +261,8 @@ public class PyPIPackageUtil {
|
||||
return ContainerUtil.getFirstItem(versions);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static List<String> getPackageVersionsFromPyPI(@NotNull String packageName) throws IOException {
|
||||
return getVersionsFromRepository(packageName, PYPI_LIST_URL);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private static String getLatestPackageVersionFromPyPI(@NotNull String packageName) throws IOException {
|
||||
LOG.debug("Requesting the latest PyPI version for the package " + packageName);
|
||||
final List<String> versions = getPackageVersionsFromPyPI(packageName);
|
||||
final String latest = ContainerUtil.getFirstItem(versions);
|
||||
getPyPIPackages().put(packageName, StringUtil.notNullize(latest));
|
||||
return latest;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static List<String> getVersionsFromRepository(@NotNull String packageName, @NotNull String repository) throws IOException {
|
||||
final String packageArchivesSimpleUrl = composeSimpleUrl(packageName, repository);
|
||||
return parsePackageVersionsFromArchives(packageArchivesSimpleUrl);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static String fetchLatestPackageVersion(@NotNull String packageName) throws IOException {
|
||||
public String fetchLatestPackageVersion(@NotNull String packageName) throws IOException {
|
||||
String version = getPyPIPackages().get(packageName);
|
||||
// Package is on PyPI but it's version is unknown
|
||||
if (version != null && version.isEmpty()) {
|
||||
@@ -443,67 +415,54 @@ public class PyPIPackageUtil {
|
||||
return myPackageNames != null && myPackageNames.contains(packageName.toLowerCase(Locale.ENGLISH));
|
||||
}
|
||||
|
||||
private static class PyPIXmlRpcTransport extends DefaultXmlRpcTransport {
|
||||
public PyPIXmlRpcTransport(URL url) {
|
||||
super(url);
|
||||
@SuppressWarnings("FieldMayBeFinal")
|
||||
public static final class PackageDetails {
|
||||
public static final class Info {
|
||||
private String version = "";
|
||||
private String author = "";
|
||||
|
||||
private String authorEmail = "";
|
||||
private String homePage = "";
|
||||
private String summary = "";
|
||||
|
||||
|
||||
@NotNull
|
||||
public String getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public String getAuthor() {
|
||||
return author;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public String getAuthorEmail() {
|
||||
return authorEmail;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public String getHomePage() {
|
||||
return homePage;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public String getSummary() {
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream sendXmlRpc(@NotNull byte[] request) throws IOException {
|
||||
// Create a trust manager that does not validate certificate for this connection
|
||||
final TrustManager[] trustAllCerts = new TrustManager[]{new PyPITrustManager()};
|
||||
private Info info = new Info();
|
||||
private Map<String, Object> releases = Collections.emptyMap();
|
||||
|
||||
try {
|
||||
final SSLContext sslContext = SSLContext.getInstance("TLS");
|
||||
sslContext.init(null, trustAllCerts, new SecureRandom());
|
||||
|
||||
final HttpConfigurable settings = HttpConfigurable.getInstance();
|
||||
con = settings.openConnection(PYPI_HOST + "/pypi?%3Aaction=index");
|
||||
if (con instanceof HttpsURLConnection) {
|
||||
((HttpsURLConnection)con).setSSLSocketFactory(sslContext.getSocketFactory());
|
||||
}
|
||||
con.setDoInput(true);
|
||||
con.setDoOutput(true);
|
||||
con.setUseCaches(false);
|
||||
con.setAllowUserInteraction(false);
|
||||
con.setRequestProperty("Content-Length",
|
||||
Integer.toString(request.length));
|
||||
con.setRequestProperty("Content-Type", "text/xml");
|
||||
if (auth != null)
|
||||
{
|
||||
con.setRequestProperty("Authorization", "Basic " + auth);
|
||||
}
|
||||
final OutputStream out = con.getOutputStream();
|
||||
out.write(request);
|
||||
out.flush();
|
||||
out.close();
|
||||
return con.getInputStream();
|
||||
}
|
||||
catch (@NotNull NoSuchAlgorithmException | KeyManagementException e) {
|
||||
LOG.warn(e.getMessage());
|
||||
}
|
||||
return super.sendXmlRpc(request);
|
||||
}
|
||||
}
|
||||
|
||||
private static class PyPIXmlRpcTransportFactory extends DefaultXmlRpcTransportFactory {
|
||||
public PyPIXmlRpcTransportFactory(URL url) {
|
||||
super(url);
|
||||
@NotNull
|
||||
public Info getInfo() {
|
||||
return info;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public XmlRpcTransport createTransport() throws XmlRpcClientException {
|
||||
return new PyPIXmlRpcTransport(url);
|
||||
public List<String> getReleases() {
|
||||
return new ArrayList<>(releases.keySet());
|
||||
}
|
||||
}
|
||||
|
||||
private static class PyPITrustManager implements X509TrustManager {
|
||||
@Override
|
||||
public X509Certificate[] getAcceptedIssuers(){return null;}
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] certs, String authType){}
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] certs, String authType){}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import com.intellij.webcore.packaging.InstalledPackage;
|
||||
import com.intellij.webcore.packaging.PackageManagementServiceEx;
|
||||
import com.intellij.webcore.packaging.RepoPackage;
|
||||
import com.jetbrains.python.packaging.*;
|
||||
import com.jetbrains.python.packaging.PyPIPackageUtil.PackageDetails;
|
||||
import com.jetbrains.python.psi.LanguageLevel;
|
||||
import com.jetbrains.python.sdk.PySdkUtil;
|
||||
import com.jetbrains.python.sdk.PythonSdkType;
|
||||
@@ -43,7 +44,6 @@ import java.util.regex.Pattern;
|
||||
/**
|
||||
* @author yole
|
||||
*/
|
||||
@SuppressWarnings("UseOfObsoleteCollectionType")
|
||||
public class PyPackageManagementService extends PackageManagementServiceEx {
|
||||
@NotNull private static final Pattern PATTERN_ERROR_LINE = Pattern.compile(".*error:.*", Pattern.CASE_INSENSITIVE);
|
||||
@NonNls private static final String TEXT_PREFIX = "<html><head>" +
|
||||
@@ -255,11 +255,10 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
|
||||
|
||||
@Override
|
||||
public void fetchPackageDetails(@NotNull String packageName, @NotNull CatchingConsumer<String, Exception> consumer) {
|
||||
PyPIPackageUtil.INSTANCE.fillPackageDetails(packageName, new CatchingConsumer<Hashtable, Exception>() {
|
||||
PyPIPackageUtil.INSTANCE.fillPackageDetails(packageName, new CatchingConsumer<PackageDetails.Info, Exception>() {
|
||||
@Override
|
||||
public void consume(Hashtable details) {
|
||||
PyPIPackageUtil.INSTANCE.addPackageDetails(packageName, details);
|
||||
consumer.consume(formatPackageDetails(details));
|
||||
public void consume(PackageDetails.Info details) {
|
||||
consumer.consume(formatPackageInfo(details));
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -269,31 +268,31 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
|
||||
});
|
||||
}
|
||||
|
||||
private static String formatPackageDetails(@NotNull Hashtable details) {
|
||||
final Object description = details.get("summary");
|
||||
private static String formatPackageInfo(@NotNull PackageDetails.Info info) {
|
||||
final StringBuilder stringBuilder = new StringBuilder(TEXT_PREFIX);
|
||||
if (description instanceof String) {
|
||||
final String description = info.getSummary();
|
||||
if (StringUtil.isNotEmpty(description)) {
|
||||
stringBuilder.append(description).append("<br/>");
|
||||
}
|
||||
final Object version = details.get("version");
|
||||
if (version instanceof String && !StringUtil.isEmpty((String)version)) {
|
||||
final String version = info.getVersion();
|
||||
if (StringUtil.isNotEmpty(version)) {
|
||||
stringBuilder.append("<h4>Version</h4>");
|
||||
stringBuilder.append(version);
|
||||
}
|
||||
final Object author = details.get("author");
|
||||
if (author instanceof String && !StringUtil.isEmpty((String)author)) {
|
||||
final String author = info.getAuthor();
|
||||
if (StringUtil.isNotEmpty(author)) {
|
||||
stringBuilder.append("<h4>Author</h4>");
|
||||
stringBuilder.append(author).append("<br/><br/>");
|
||||
}
|
||||
final Object authorEmail = details.get("author_email");
|
||||
if (authorEmail instanceof String && !StringUtil.isEmpty((String)authorEmail)) {
|
||||
final String authorEmail = info.getAuthorEmail();
|
||||
if (StringUtil.isNotEmpty(authorEmail)) {
|
||||
stringBuilder.append("<br/>");
|
||||
stringBuilder.append(composeHref("mailto:" + authorEmail));
|
||||
}
|
||||
final Object homePage = details.get("home_page");
|
||||
if (homePage instanceof String && !StringUtil.isEmpty((String)homePage)) {
|
||||
final String homePage = info.getHomePage();
|
||||
if (StringUtil.isNotEmpty(homePage)) {
|
||||
stringBuilder.append("<br/>");
|
||||
stringBuilder.append(composeHref((String)homePage));
|
||||
stringBuilder.append(composeHref(homePage));
|
||||
}
|
||||
stringBuilder.append(TEXT_SUFFIX);
|
||||
return stringBuilder.toString();
|
||||
@@ -389,7 +388,7 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
|
||||
ApplicationManager.getApplication().executeOnPooledThread(() -> {
|
||||
try {
|
||||
PyPIPackageUtil.INSTANCE.loadAndGetPackages();
|
||||
final String version = PyPIPackageUtil.fetchLatestPackageVersion(pkg.getName());
|
||||
final String version = PyPIPackageUtil.INSTANCE.fetchLatestPackageVersion(pkg.getName());
|
||||
consumer.consume(StringUtil.notNullize(version));
|
||||
}
|
||||
catch (IOException e) {
|
||||
|
||||
Reference in New Issue
Block a user