diff --git a/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java b/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java
index f05a63174dcd..69fc837a5912 100644
--- a/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java
+++ b/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java
@@ -17,38 +17,91 @@ package com.jetbrains.python.packaging;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.fileEditor.FileDocumentManager;
-import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.openapi.vfs.LocalFileSystem;
import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.util.Function;
import com.intellij.webcore.packaging.PackageVersionComparator;
import com.jetbrains.python.packaging.requirement.PyRequirementRelation;
import com.jetbrains.python.packaging.requirement.PyRequirementVersionSpec;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
/**
* @author vlan
* @see [PEP-0386]
*/
public class PyRequirement {
- private static final Pattern NAME = Pattern.compile("\\s*(\\w(\\w|[-.])*)\\s*(.*)");
- private static final Pattern VERSION_SPEC = Pattern.compile("\\s*(<=?|>=?|==|!=)\\s*((\\w|[-.])+)");
- private static final Pattern EDITABLE_EGG = Pattern.compile("\\s*(-e)?\\s*([^#]*)(#egg=(.*))?");
- private static final Pattern RECURSIVE_REQUIREMENT = Pattern.compile("^-r\\s*(.*)");
- private static final Pattern VCS_PATH = Pattern.compile(".*/([^/]+)/?");
- @NotNull private final String myName;
- @NotNull private final List myVersionSpecs;
- @Nullable private final String myURL;
- private final boolean myEditable;
+ @NotNull
+ private static final String EDITABLE_GROUP = "editable";
+
+ @NotNull
+ private static final String EDITABLE_REGEXP = "((?<" + EDITABLE_GROUP + ">-e|--editable)[ \t]+)?";
+
+ @NotNull
+ private static final String USER_AT_REGEXP = "[\\w-]+@";
+
+ @NotNull
+ private static final String PATH_GROUP = "path";
+
+ @NotNull
+ private static final String PATH_REGEXP = "(?<" + PATH_GROUP + ">[^@#\\s]+)";
+
+ @NotNull
+ private static final String REVISION_REGEXP = "(@[^#\\s]+)?";
+
+ @NotNull
+ private static final String EGG_GROUP = "egg";
+
+ @NotNull
+ private static final String EGG_REGEXP = "(#egg=(?<" + EGG_GROUP + ">\\S+))?";
+
+ @NotNull
+ private static final String COMMENT_REGEXP = "([ \t]+#.*)?";
+
+ @NotNull
+ private static final String ARCHIVE_NAME_GROUP = "name";
+
+ @NotNull
+ private static final Pattern GITHUB_ARCHIVE_URL =
+ Pattern.compile("https?://github\\.com/[^/\\s]+/(?<" + ARCHIVE_NAME_GROUP + ">[^/\\s]+)/archive/.+");
+
+ @NotNull
+ private static final Pattern ARCHIVE_URL =
+ Pattern.compile("https?://\\S+/" +
+ "(?<" + ARCHIVE_NAME_GROUP + ">\\S+)" +
+ "(\\.tar\\.gz|\\.zip)(#(sha1|sha224|sha256|sha384|sha512|md5)=\\w+)?" + COMMENT_REGEXP);
+
+ // supports: git+user@...
+ @NotNull
+ private static final Pattern GIT_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP +
+ "git\\+" + USER_AT_REGEXP + "[^:\\s]+:" +
+ PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP);
+
+ @NotNull
+ private static final Pattern BZR_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP +
+ "bzr\\+lp:" +
+ PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP);
+
+ // supports: (bzr|git|hg|svn)(+smth)?://...
+ @NotNull
+ private static final Pattern VCS_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP +
+ "(bzr|git|hg|svn)(\\+[A-Za-z]+)?://?[^/]+/" +
+ PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP);
+
+ @NotNull
+ private final String myName;
+
+ @NotNull
+ private final String myOptions;
+
+ @NotNull
+ private final List myVersionSpecs;
public PyRequirement(@NotNull String name) {
this(name, Collections.emptyList());
@@ -61,8 +114,7 @@ public class PyRequirement {
public PyRequirement(@NotNull String name, @NotNull List versionSpecs) {
myName = name;
myVersionSpecs = versionSpecs;
- myURL = null;
- myEditable = false;
+ myOptions = toString();
}
public PyRequirement(@NotNull String name, @Nullable String version, @NotNull String url, boolean editable) {
@@ -73,61 +125,34 @@ public class PyRequirement {
else {
myVersionSpecs = Collections.emptyList();
}
- myURL = url;
- myEditable = editable;
+ myOptions = url;
}
@NotNull
+ public String getName() {
+ return myName;
+ }
+
+ @NotNull
+ public String toOptions() {
+ return myOptions;
+ }
+
@Override
public String toString() {
- return myName + StringUtil.join(myVersionSpecs,
- new Function() {
- @Override
- public String fun(PyRequirementVersionSpec spec) {
- return spec.toString();
- }
- },
- ","
- );
- }
-
- @NotNull
- public List toOptions() {
- final List results = new ArrayList();
- if (myEditable) {
- results.add("-e");
- }
- if (myURL != null) {
- final int size = myVersionSpecs.size();
- assert size <= 1;
- final String urlAndName = myURL + "#egg=" + myName;
- if (size == 0) {
- results.add(urlAndName);
- }
- else {
- final PyRequirementVersionSpec versionSpec = myVersionSpecs.get(0);
- assert versionSpec.getRelation() == PyRequirementRelation.EQ;
- results.add(urlAndName + "-" + versionSpec.getVersion());
- }
- return results;
- }
- else {
- results.add(toString());
- }
- return results;
+ return myName + StringUtil.join(myVersionSpecs, ",");
}
@Override
public boolean equals(Object o) {
- if (this == o) return true;
+ if (o == this) return true;
if (o == null || getClass() != o.getClass()) return false;
PyRequirement that = (PyRequirement)o;
- if (myEditable != that.myEditable) return false;
if (!myName.equals(that.myName)) return false;
- if (myURL != null ? !myURL.equals(that.myURL) : that.myURL != null) return false;
if (!myVersionSpecs.equals(that.myVersionSpecs)) return false;
+ if (!myOptions.equals(that.myOptions)) return false;
return true;
}
@@ -136,8 +161,7 @@ public class PyRequirement {
public int hashCode() {
int result = myName.hashCode();
result = 31 * result + myVersionSpecs.hashCode();
- result = 31 * result + (myURL != null ? myURL.hashCode() : 0);
- result = 31 * result + (myEditable ? 1 : 0);
+ result = 31 * result + myOptions.hashCode();
return result;
}
@@ -147,27 +171,20 @@ public class PyRequirement {
if (normalizeName(myName).equalsIgnoreCase(pkg.getName())) {
for (PyRequirementVersionSpec spec : myVersionSpecs) {
final int cmp = PackageVersionComparator.VERSION_COMPARATOR.compare(pkg.getVersion(), spec.getVersion());
- final PyRequirementRelation relation = spec.getRelation();
- if (!relation.isSuccessful(cmp)) {
+
+ if (!spec.getRelation().isSuccessful(cmp)) {
return null;
}
}
return pkg;
}
}
+
return null;
}
- /**
- * Parses requirement string as described in [pep-0386].
- * For example: "myPackage<=10.6a3"
- *
- * @param line requirement to parse
- * @return requirement
- * @throws IllegalArgumentException if line can't be parsed
- */
@NotNull
- public static PyRequirement fromStringGuaranteed(@NotNull final String line) {
+ public static PyRequirement fromStringGuaranteed(@NotNull String line) {
final PyRequirement requirement = fromString(line);
if (requirement == null) {
throw new IllegalArgumentException("Failed to parse " + line);
@@ -177,142 +194,169 @@ public class PyRequirement {
@Nullable
public static PyRequirement fromString(@NotNull String line) {
- // TODO: Extras, multi-line requirements '\'
- final PyRequirement editableEgg = parseEditableEgg(line);
- if (editableEgg != null) {
- return editableEgg;
+ final PyRequirement githubArchiveUrl = parseGithubArchiveUrl(line);
+ if (githubArchiveUrl != null) {
+ return githubArchiveUrl;
}
- final Matcher nameMatcher = NAME.matcher(line);
- if (!nameMatcher.matches()) {
- return null;
+
+ final PyRequirement archiveUrl = parseArchiveUrl(line);
+ if (archiveUrl != null) {
+ return archiveUrl;
}
- final String name = nameMatcher.group(1);
- final String rest = nameMatcher.group(3);
- final List versionSpecs = new ArrayList();
- if (!rest.trim().isEmpty()) {
- final Matcher versionSpecMatcher = VERSION_SPEC.matcher(rest);
- while (versionSpecMatcher.find()) {
- final String rel = versionSpecMatcher.group(1);
- final String version = versionSpecMatcher.group(2);
- final PyRequirementRelation relation = PyRequirementRelation.fromString(rel);
- if (relation == null) {
- return null;
- }
- versionSpecs.add(new PyRequirementVersionSpec(relation, version));
- }
+
+ final PyRequirement vcsProjectUrl = parseVcsProjectUrl(line);
+ if (vcsProjectUrl != null) {
+ return vcsProjectUrl;
}
- return new PyRequirement(name, versionSpecs);
+
+ return null; // TODO
}
@NotNull
- public static List parse(@NotNull String s) {
- return parse(s, null, new HashSet());
+ public static List parse(@NotNull String text) {
+ return parse(text, null, new HashSet<>());
}
@NotNull
public static List parse(@NotNull VirtualFile file) {
- return parse(file, new HashSet());
+ return parse(loadText(file), file, new HashSet<>());
}
@NotNull
- private static List parse(@NotNull VirtualFile file, @NotNull Set visited) {
- if (!visited.contains(file)) {
- visited.add(file);
- final Document document = FileDocumentManager.getInstance().getDocument(file);
- if (document != null) {
- return parse(document.getText(), file, visited);
- }
- }
- return Collections.emptyList();
- }
-
- @NotNull
- public String getName() {
- return myName;
- }
-
- @NotNull
- private static List parse(@NotNull String s, @Nullable VirtualFile anchor, @NotNull Set visited) {
- final Set result = new LinkedHashSet();
- for (String line : StringUtil.splitByLines(s)) {
- final String trimmed = line.trim();
- if (!trimmed.isEmpty()) {
- final PyRequirement req = fromString(line);
- if (req != null) {
- result.add(req);
- }
- else if (anchor != null) {
- result.addAll(parseRecursiveRequirement(trimmed, anchor, visited));
- }
- }
- }
- return new ArrayList(result);
- }
-
- @NotNull
- private static List parseRecursiveRequirement(@NotNull String trimmedLine, @NotNull VirtualFile anchor,
- @NotNull Set visited) {
- final Matcher matcher = RECURSIVE_REQUIREMENT.matcher(trimmedLine);
- if (matcher.matches()) {
- final String fileName = FileUtil.toSystemIndependentName(matcher.group(1));
- final VirtualFile dir = anchor.getParent();
- if (dir != null) {
- VirtualFile file = dir.findFileByRelativePath(fileName);
- if (file == null) {
- file = LocalFileSystem.getInstance().findFileByPath(fileName);
- }
- if (file != null) {
- return parse(file, visited);
- }
- }
- }
- return Collections.emptyList();
+ private static List parse(@NotNull String text,
+ @Nullable VirtualFile containingFile,
+ @NotNull Set visitedFiles) {
+ return Arrays
+ .stream(StringUtil.splitByLines(text))
+ .map(String::trim)
+ .filter(line -> !line.isEmpty())
+ .map(PyRequirement::fromString)
+ .filter(req -> req != null)
+ .collect(Collectors.toCollection(LinkedHashSet::new))
+ .stream()
+ .collect(Collectors.toList());
}
@Nullable
- private static PyRequirement parseEditableEgg(@NotNull String line) {
- final Matcher editableEggMatcher = EDITABLE_EGG.matcher(line);
- if (!editableEggMatcher.matches()) {
- return null;
+ private static PyRequirement parseGithubArchiveUrl(@NotNull String line) {
+ final Matcher matcher = GITHUB_ARCHIVE_URL.matcher(line);
+
+ if (matcher.matches()) {
+ return new PyRequirement(matcher.group(ARCHIVE_NAME_GROUP), null, line, false);
}
- final boolean editable = editableEggMatcher.group(1) != null;
- final String url = editableEggMatcher.group(2);
- String egg = editableEggMatcher.group(4);
- if (url == null) {
- return null;
+
+ return null;
+ }
+
+ @Nullable
+ private static PyRequirement parseArchiveUrl(@NotNull String line) {
+ final Matcher matcher = ARCHIVE_URL.matcher(line);
+
+ if (matcher.matches()) {
+ final Pair nameAndVersion = parseNameAndVersion(matcher.group(ARCHIVE_NAME_GROUP));
+
+ return new PyRequirement(nameAndVersion.getFirst(), nameAndVersion.getSecond(), line, false);
}
- if (egg == null) {
- try {
- final URI uri = new URI(url);
- if (uri.getScheme() != null) {
- String path = uri.getPath();
- if (path != null) {
- final String[] split = path.split("@", 2);
- path = split[0];
- final Matcher vcsPathMatcher = VCS_PATH.matcher(path);
- if (!vcsPathMatcher.matches()) {
- return null;
- }
- egg = vcsPathMatcher.group(1);
- final String gitSuffix = ".git";
- egg = StringUtil.trimEnd(egg, gitSuffix);
- }
- }
- }
- catch (URISyntaxException e) {
- return null;
- }
+
+ return null;
+ }
+
+ @Nullable
+ private static PyRequirement parseVcsProjectUrl(@NotNull String line) {
+ final Matcher vcsMatcher = VCS_PROJECT_URL.matcher(line);
+ if (vcsMatcher.matches()) {
+ return createVcsRequirement(line, vcsMatcher);
}
- if (egg == null) {
- return null;
+
+ final Matcher gitMatcher = GIT_PROJECT_URL.matcher(line);
+ if (gitMatcher.matches()) {
+ return createVcsRequirement(line, gitMatcher);
}
+
+ final Matcher bzrMatcher = BZR_PROJECT_URL.matcher(line);
+ if (bzrMatcher.matches()) {
+ return createVcsRequirement(line, bzrMatcher);
+ }
+
+ return null;
+ }
+
+ @NotNull
+ private static String loadText(@NotNull VirtualFile file) {
+ final Document document = FileDocumentManager.getInstance().getDocument(file);
+
+ return document == null ? "" : document.getText();
+ }
+
+ @NotNull
+ private static PyRequirement createVcsRequirement(@NotNull String line, @NotNull Matcher matcher) {
+ final boolean editable = matcher.group(EDITABLE_GROUP) != null;
+ final String path = matcher.group(PATH_GROUP);
+ final String egg = matcher.group(EGG_GROUP);
+
+ final String project = extractProject(dropTrunk(dropRevision(path)));
+ final Pair nameAndVersion = parseNameAndVersion(egg == null ? StringUtil.trimEnd(project, ".git") : egg);
+
+ return new PyRequirement(nameAndVersion.getFirst(), nameAndVersion.getSecond(), line, editable);
+ }
+
+ @NotNull
+ private static String dropRevision(@NotNull String path) {
+ final int atIndex = path.lastIndexOf("@");
+
+ if (atIndex != -1) {
+ return path.substring(0, atIndex);
+ }
+
+ return path;
+ }
+
+ @NotNull
+ private static String dropTrunk(@NotNull String path) {
+ final String slashTrunk = "/trunk";
+
+ if (path.endsWith(slashTrunk)) {
+ return path.substring(0, path.length() - slashTrunk.length());
+ }
+
+ final String slashTrunkSlash = "/trunk/";
+
+ if (path.endsWith(slashTrunkSlash)) {
+ return path.substring(0, path.length() - slashTrunkSlash.length());
+ }
+
+ return path;
+ }
+
+ @NotNull
+ private static String extractProject(@NotNull String path) {
+ final int end = path.endsWith("/") ? path.length() - 1 : path.length();
+ final int slashIndex = path.lastIndexOf("/", end - 1);
+
+ if (slashIndex != -1) {
+ return path.substring(slashIndex + 1, end);
+ }
+
+ if (end != path.length()) {
+ return path.substring(0, end);
+ }
+
+ return path;
+ }
+
+ @NotNull
+ private static Pair parseNameAndVersion(@NotNull String name) {
boolean isName = true;
final List nameParts = new ArrayList();
final List versionParts = new ArrayList();
- for (String part : StringUtil.split(egg, "-")) {
- if (part.matches("[0-9].*") || "dev".equals(part)) {
+
+ for (String part : StringUtil.split(name, "-")) {
+ final boolean partStartsWithDigit = !part.isEmpty() && Character.isDigit(part.charAt(0));
+
+ if (partStartsWithDigit || "dev".equals(part)) {
isName = false;
}
+
if (isName) {
nameParts.add(part);
}
@@ -320,9 +364,18 @@ public class PyRequirement {
versionParts.add(part);
}
}
- final String name = normalizeName(StringUtil.join(nameParts, "-"));
- final String version = !versionParts.isEmpty() ? normalizeVersion(StringUtil.join(versionParts, "-")) : null;
- return new PyRequirement(name, version, url, editable);
+
+ return Pair.create(normalizeNameParts(nameParts), normalizeVersionParts(versionParts));
+ }
+
+ @NotNull
+ private static String normalizeNameParts(@NotNull List nameParts) {
+ return normalizeName(StringUtil.join(nameParts, "-"));
+ }
+
+ @Nullable
+ private static String normalizeVersionParts(@NotNull List versionParts) {
+ return versionParts.isEmpty() ? null : normalizeVersion(StringUtil.join(versionParts, "-"));
}
@NotNull
@@ -332,6 +385,6 @@ public class PyRequirement {
@NotNull
private static String normalizeVersion(@NotNull String s) {
- return s.replace("_", "-").replaceAll("-?py[0-9\\.]+", "");
+ return s.replace("_", "-").replaceAll("-?py[\\d\\.]+", "");
}
}
diff --git a/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java b/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java
index 0de50fe83f5d..dcceb1ad8342 100644
--- a/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java
+++ b/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java
@@ -207,7 +207,7 @@ public class PyPackageManagerImpl extends PyPackageManager {
}
args.addAll(extraArgs);
for (PyRequirement req : requirements) {
- args.addAll(req.toOptions());
+ args.add(req.toOptions());
}
try {
getHelperResult(PACKAGING_TOOL, args, !useUserSite, true, null);
@@ -221,7 +221,7 @@ public class PyPackageManagerImpl extends PyPackageManager {
}
simplifiedArgs.addAll(extraArgs);
for (PyRequirement req : requirements) {
- simplifiedArgs.addAll(req.toOptions());
+ simplifiedArgs.add(req.toOptions());
}
throw new PyExecutionException(e.getMessage(), "pip", simplifiedArgs, e.getStdout(), e.getStderr(), e.getExitCode(), e.getFixes());
}