PY-18405 Version of devpi package is always latest

python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java

@@ -65,7 +65,7 @@ public class PyPIPackageUtil {
 
   private XmlRpcClient myXmlRpcClient;
   private Map<String, Hashtable> packageToDetails = new HashMap<String, Hashtable>();
-  private Map<String, List<String>> packageToReleases = new HashMap<String, List<String>>();
+  private static Map<String, List<String>> packageToReleases = new HashMap<String, List<String>>();
   private Pattern PYPI_PATTERN = Pattern.compile("/pypi/([^/]*)/(.*)");
   private static Set<RepoPackage> ourAdditionalPackageNames = new TreeSet<RepoPackage>();
   @Nullable private volatile Set<String> myPackageNames = null;
@@ -98,9 +98,9 @@ public class PyPIPackageUtil {
   @NotNull
   private static Pair<String, String> splitNameVersion(@NotNull final String pyPackage) {
     int dashInd = pyPackage.lastIndexOf("-");
-    if (dashInd >= 0) {
+    if (dashInd >= 0 && dashInd+1 < pyPackage.length()) {
       final String name = pyPackage.substring(0, dashInd);
-      final String version =  pyPackage.substring(dashInd);
+      final String version = pyPackage.substring(dashInd+1);
       if (StringUtil.containsAlphaCharacters(version)) {
         return Pair.create(pyPackage, null);
       }
@@ -170,7 +170,14 @@ public class PyPIPackageUtil {
       final Vector<String> params = new Vector<String>();
       params.add(packageName);
       try {
-        params.add(getPyPIPackages().get(packageName));
+        String version = getPyPIPackages().get(packageName);
+        if (version == null) {
+          final List<String> releases = getPackageReleases(packageName);
+          if (releases != null && !releases.isEmpty()) {
+            version = releases.get(0);
+          }
+        }
+        params.add(version);
         myXmlRpcClient.executeAsync("release_data", params, callback);
       }
       catch (Exception ignored) {
@@ -199,11 +206,65 @@ public class PyPIPackageUtil {
   }
 
   @Nullable
-  public List<String> getPackageReleases(@NonNls String packageName) {
+  public static List<String> getPackageReleases(@NonNls String packageName) {
     if (packageToReleases.containsKey(packageName)) return packageToReleases.get(packageName);
+    final List<String> repositories = PyPackageService.getInstance().additionalRepositories;
+    if (!repositories.isEmpty()) {
+      for (String repository : repositories) {
+        repository = composeSimpleUrl(packageName, repository);
+
+        final List<String> versions = parsePackageVersions(repository);
+        if (!versions.isEmpty()) {
+          packageToReleases.put(packageName, versions);
+          return versions;
+        }
+      }
+    }
     return null;
   }
 
+  private static List<String> parsePackageVersions(@NotNull final String repository) {
+    return HttpRequests.request(repository).connect(new HttpRequests.RequestProcessor<List<String>>() {
+      @Override
+      public List<String> process(@NotNull HttpRequests.Request request) throws IOException {
+        final List<String> versions = new ArrayList<String>();
+        Reader reader = request.getReader();
+        new ParserDelegator().parse(reader, new HTMLEditorKit.ParserCallback() {
+          HTML.Tag myTag;
+
+          @Override
+          public void handleStartTag(HTML.Tag tag, MutableAttributeSet set, int i) {
+            myTag = tag;
+          }
+
+          @Override
+          public void handleText(char[] data, int pos) {
+            if (myTag != null && "a".equals(myTag.toString())) {
+              String packageVersion = String.valueOf(data);
+              final String suffix = ".tar.gz";
+              if (!packageVersion.endsWith(suffix)) return;
+              packageVersion = StringUtil.trimEnd(packageVersion, suffix);
+              versions.add(splitNameVersion(packageVersion).second);
+            }
+          }
+
+        }, true);
+        return versions;
+      }
+    }, Collections.emptyList(), LOG);
+  }
+
+  @NotNull
+  private static String composeSimpleUrl(@NonNls @NotNull final String packageName, @NotNull final String rep) {
+    String suffix = "";
+    final String repository = StringUtil.trimEnd(rep, "/");
+    if (!repository.endsWith("+simple")) {
+      suffix = "/+simple";
+    }
+    suffix += "/" + packageName;
+    return repository + suffix;
+  }
+
   private PyPIPackageUtil() {
     try {
       DefaultXmlRpcTransportFactory factory = new PyPIXmlRpcTransportFactory(new URL(PYPI_URL));
@@ -286,7 +347,7 @@ public class PyPIPackageUtil {
         }, true);
         return packages;
       }
-    }, Collections.<String>emptyList(), LOG);
+    }, Collections.emptyList(), LOG);
   }
 
   public Collection<String> getPackageNames() {

python/src/com/jetbrains/python/packaging/ui/PyPackageManagementService.java

@@ -399,6 +399,10 @@ public class PyPackageManagementService extends PackageManagementServiceEx {
     }
     catch (IOException ignored) {
     }
+    final List<String> releases = PyPIPackageUtil.getPackageReleases(pkg.getName());
+    if (releases != null && !releases.isEmpty()) {
+      version = releases.get(0);
+    }
     consumer.consume(version);
   }
 }