From 177c0b274ffd4c266193c2fd1ebb604c20ddf03c Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Mon, 25 Jan 2016 20:32:06 +0300 Subject: [PATCH] PY-13896 devpi support and make pypi.python.org optional --- .../python/packaging/PyPIPackageUtil.java | 128 ++++++++++-------- .../python/packaging/PyPackageService.java | 3 +- .../ui/PyPackageManagementService.java | 4 +- 3 files changed, 75 insertions(+), 60 deletions(-) diff --git a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java index 66a5a5384352..c4d6104922a4 100644 --- a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java +++ b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java @@ -17,6 +17,7 @@ package com.jetbrains.python.packaging; import com.google.common.collect.Lists; import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.io.HttpRequests; @@ -94,64 +95,62 @@ public class PyPIPackageUtil { } } - public static Set getPackageNames(final String url) throws IOException { - final TreeSet names = new TreeSet(); - final HTMLEditorKit.ParserCallback callback = - new HTMLEditorKit.ParserCallback() { - HTML.Tag myTag; - @Override - public void handleStartTag(HTML.Tag tag, - MutableAttributeSet set, - int i) { - myTag = tag; - } - - public void handleText(char[] data, int pos) { - if (myTag != null && "a".equals(myTag.toString())) { - names.add(String.valueOf(data)); - } - } - }; - - try { - final URL repositoryUrl = new URL(url); - final InputStream is = repositoryUrl.openStream(); - final Reader reader = new InputStreamReader(is); - try{ - new ParserDelegator().parse(reader, callback, true); + @NotNull + private static Pair splitNameVersion(@NotNull final String pyPackage) { + int dashInd = pyPackage.lastIndexOf("-"); + if (dashInd >= 0) { + final String name = pyPackage.substring(0, dashInd); + final String version = pyPackage.substring(dashInd); + if (StringUtil.containsAlphaCharacters(version)) { + return Pair.create(pyPackage, null); } - catch (IOException e) { - LOG.warn(e); + return Pair.create(name, version); + } + return Pair.create(pyPackage, null); + } + + public void fillAdditionalPackages(@NotNull final String url) { + final boolean simpleIndex = url.endsWith("simple/"); + final List packagesList = parsePyPIListFromWeb(url, simpleIndex); + + for (String pyPackage : packagesList) { + if (simpleIndex) { + final Pair nameVersion = splitNameVersion(pyPackage); + myAdditionalPackageNames.add(new RepoPackage(nameVersion.getFirst(), url, nameVersion.getSecond())); } - finally { - reader.close(); + else { + try { + Pattern repositoryPattern = Pattern.compile(url + "([^/]*)/([^/]*)$"); + final Matcher matcher = repositoryPattern.matcher(URLDecoder.decode(pyPackage, "UTF-8")); + if (matcher.find()) { + final String packageName = matcher.group(1); + final String packageVersion = matcher.group(2); + if (!packageName.contains(" ")) + myAdditionalPackageNames.add(new RepoPackage(packageName, url, packageVersion)); + } + } + catch (UnsupportedEncodingException e) { + LOG.warn(e.getMessage()); + } } } - catch (MalformedURLException e) { - LOG.warn(e); - } - - return names; } public Set getAdditionalPackageNames() { - if (myAdditionalPackageNames == null) { + if (myAdditionalPackageNames == null || myAdditionalPackageNames.isEmpty()) { myAdditionalPackageNames = new TreeSet(); for (String url : PyPackageService.getInstance().additionalRepositories) { - try { - for (String pyPackage : getPackageNames(url)) { - if (!pyPackage.contains(" ")) - myAdditionalPackageNames.add(new RepoPackage(pyPackage, url)); - } - } - catch (IOException e) { - LOG.warn(e); - } + fillAdditionalPackages(url); } } return myAdditionalPackageNames; } + public void clearPackagesCache() { + PyPackageService.getInstance().PY_PACKAGES.clear(); + if (myAdditionalPackageNames != null) myAdditionalPackageNames.clear(); + } + public void addPackageDetails(@NonNls String packageName, Hashtable details) { packageToDetails.put(packageName, details); } @@ -213,13 +212,15 @@ public class PyPIPackageUtil { } public void updatePyPICache(final PyPackageService service) throws IOException { - myPackageNames = null; + service.LAST_TIME_CHECKED = System.currentTimeMillis(); + service.PY_PACKAGES.clear(); if (service.PYPI_REMOVED) return; - parsePyPIList(getPyPIListFromWeb(), service); + parsePyPIList(parsePyPIListFromWeb(PYPI_LIST_URL, false), service); } public void parsePyPIList(final List packages, final PyPackageService service) { + myPackageNames = null; for (String pyPackage : packages) { try { final Matcher matcher = PYPI_PATTERN.matcher(URLDecoder.decode(pyPackage, "UTF-8")); @@ -236,31 +237,46 @@ public class PyPIPackageUtil { } } - @Nullable - public List getPyPIListFromWeb() { - return HttpRequests.request(PYPI_LIST_URL).connect(new HttpRequests.RequestProcessor>() { + @NotNull + public List parsePyPIListFromWeb(@NotNull final String url, final boolean isSimpleIndex) { + return HttpRequests.request(url).connect(new HttpRequests.RequestProcessor>() { @Override public List process(@NotNull HttpRequests.Request request) throws IOException { final List packages = new ArrayList(); Reader reader = request.getReader(); new ParserDelegator().parse(reader, new HTMLEditorKit.ParserCallback() { boolean inTable = false; + HTML.Tag myTag; @Override public void handleStartTag(HTML.Tag tag, MutableAttributeSet set, int i) { - if ("table".equals(tag.toString())) { - inTable = !inTable; - } + myTag = tag; + if (!isSimpleIndex) { + if ("table".equals(tag.toString())) { + inTable = !inTable; + } - if (inTable && "a".equals(tag.toString())) { - packages.add(String.valueOf(set.getAttribute(HTML.Attribute.HREF))); + if (inTable && "a".equals(tag.toString())) { + packages.add(String.valueOf(set.getAttribute(HTML.Attribute.HREF))); + } + } + } + + @Override + public void handleText(char[] data, int pos) { + if (isSimpleIndex) { + if (myTag != null && "a".equals(myTag.toString())) { + packages.add(String.valueOf(data)); + } } } @Override public void handleEndTag(HTML.Tag tag, int i) { - if ("table".equals(tag.toString())) { - inTable = !inTable; + if (!isSimpleIndex) { + if ("table".equals(tag.toString())) { + inTable = !inTable; + } } } }, true); diff --git a/python/src/com/jetbrains/python/packaging/PyPackageService.java b/python/src/com/jetbrains/python/packaging/PyPackageService.java index d1e03125a625..33062a8f8d2c 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackageService.java +++ b/python/src/com/jetbrains/python/packaging/PyPackageService.java @@ -56,12 +56,13 @@ public class PyPackageService implements sdkToUsersite.put(sdk, useUsersite); } - public void addRepository(final String repository) { + public void addRepository(String repository) { if (repository == null) return; if (repository.startsWith(PyPIPackageUtil.PYPI_HOST)) { PYPI_REMOVED = false; } else { + if (!repository.endsWith("/")) repository += "/"; additionalRepositories.add(repository); } } diff --git a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java index 6f500df4e02b..1e4c98e28074 100644 --- a/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java +++ b/python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java @@ -105,9 +105,7 @@ public class PyPackageManagementService extends PackageManagementService { @Override public List reloadAllPackages() throws IOException { - final PyPackageService service = PyPackageService.getInstance(); - PyPIPackageUtil.INSTANCE.updatePyPICache(service); - service.LAST_TIME_CHECKED = System.currentTimeMillis(); + PyPIPackageUtil.INSTANCE.clearPackagesCache(); return getAllPackages(); }