PyRequirement: implement parsing of vcs urls

This commit is contained in:
Semyon Proshev
2016-05-24 17:29:56 +03:00
parent ebea6721aa
commit 47b7d1b439
2 changed files with 239 additions and 186 deletions
@@ -17,38 +17,91 @@ package com.jetbrains.python.packaging;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.LocalFileSystem;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Function;
import com.intellij.webcore.packaging.PackageVersionComparator;
import com.jetbrains.python.packaging.requirement.PyRequirementRelation;
import com.jetbrains.python.packaging.requirement.PyRequirementVersionSpec;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* @author vlan
* @see <a href="http://legacy.python.org/dev/peps/pep-0386/">[PEP-0386]</a>
*/
public class PyRequirement {
private static final Pattern NAME = Pattern.compile("\\s*(\\w(\\w|[-.])*)\\s*(.*)");
private static final Pattern VERSION_SPEC = Pattern.compile("\\s*(<=?|>=?|==|!=)\\s*((\\w|[-.])+)");
private static final Pattern EDITABLE_EGG = Pattern.compile("\\s*(-e)?\\s*([^#]*)(#egg=(.*))?");
private static final Pattern RECURSIVE_REQUIREMENT = Pattern.compile("^-r\\s*(.*)");
private static final Pattern VCS_PATH = Pattern.compile(".*/([^/]+)/?");
@NotNull private final String myName;
@NotNull private final List<PyRequirementVersionSpec> myVersionSpecs;
@Nullable private final String myURL;
private final boolean myEditable;
@NotNull
private static final String EDITABLE_GROUP = "editable";
@NotNull
private static final String EDITABLE_REGEXP = "((?<" + EDITABLE_GROUP + ">-e|--editable)[ \t]+)?";
@NotNull
private static final String USER_AT_REGEXP = "[\\w-]+@";
@NotNull
private static final String PATH_GROUP = "path";
@NotNull
private static final String PATH_REGEXP = "(?<" + PATH_GROUP + ">[^@#\\s]+)";
@NotNull
private static final String REVISION_REGEXP = "(@[^#\\s]+)?";
@NotNull
private static final String EGG_GROUP = "egg";
@NotNull
private static final String EGG_REGEXP = "(#egg=(?<" + EGG_GROUP + ">\\S+))?";
@NotNull
private static final String COMMENT_REGEXP = "([ \t]+#.*)?";
@NotNull
private static final String ARCHIVE_NAME_GROUP = "name";
@NotNull
private static final Pattern GITHUB_ARCHIVE_URL =
Pattern.compile("https?://github\\.com/[^/\\s]+/(?<" + ARCHIVE_NAME_GROUP + ">[^/\\s]+)/archive/.+");
@NotNull
private static final Pattern ARCHIVE_URL =
Pattern.compile("https?://\\S+/" +
"(?<" + ARCHIVE_NAME_GROUP + ">\\S+)" +
"(\\.tar\\.gz|\\.zip)(#(sha1|sha224|sha256|sha384|sha512|md5)=\\w+)?" + COMMENT_REGEXP);
// supports: git+user@...
@NotNull
private static final Pattern GIT_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP +
"git\\+" + USER_AT_REGEXP + "[^:\\s]+:" +
PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP);
@NotNull
private static final Pattern BZR_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP +
"bzr\\+lp:" +
PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP);
// supports: (bzr|git|hg|svn)(+smth)?://...
@NotNull
private static final Pattern VCS_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP +
"(bzr|git|hg|svn)(\\+[A-Za-z]+)?://?[^/]+/" +
PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP);
@NotNull
private final String myName;
@NotNull
private final String myOptions;
@NotNull
private final List<PyRequirementVersionSpec> myVersionSpecs;
public PyRequirement(@NotNull String name) {
this(name, Collections.<PyRequirementVersionSpec>emptyList());
@@ -61,8 +114,7 @@ public class PyRequirement {
public PyRequirement(@NotNull String name, @NotNull List<PyRequirementVersionSpec> versionSpecs) {
myName = name;
myVersionSpecs = versionSpecs;
myURL = null;
myEditable = false;
myOptions = toString();
}
public PyRequirement(@NotNull String name, @Nullable String version, @NotNull String url, boolean editable) {
@@ -73,61 +125,34 @@ public class PyRequirement {
else {
myVersionSpecs = Collections.emptyList();
}
myURL = url;
myEditable = editable;
myOptions = url;
}
@NotNull
public String getName() {
return myName;
}
@NotNull
public String toOptions() {
return myOptions;
}
@Override
public String toString() {
return myName + StringUtil.join(myVersionSpecs,
new Function<PyRequirementVersionSpec, String>() {
@Override
public String fun(PyRequirementVersionSpec spec) {
return spec.toString();
}
},
","
);
}
@NotNull
public List<String> toOptions() {
final List<String> results = new ArrayList<String>();
if (myEditable) {
results.add("-e");
}
if (myURL != null) {
final int size = myVersionSpecs.size();
assert size <= 1;
final String urlAndName = myURL + "#egg=" + myName;
if (size == 0) {
results.add(urlAndName);
}
else {
final PyRequirementVersionSpec versionSpec = myVersionSpecs.get(0);
assert versionSpec.getRelation() == PyRequirementRelation.EQ;
results.add(urlAndName + "-" + versionSpec.getVersion());
}
return results;
}
else {
results.add(toString());
}
return results;
return myName + StringUtil.join(myVersionSpecs, ",");
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == this) return true;
if (o == null || getClass() != o.getClass()) return false;
PyRequirement that = (PyRequirement)o;
if (myEditable != that.myEditable) return false;
if (!myName.equals(that.myName)) return false;
if (myURL != null ? !myURL.equals(that.myURL) : that.myURL != null) return false;
if (!myVersionSpecs.equals(that.myVersionSpecs)) return false;
if (!myOptions.equals(that.myOptions)) return false;
return true;
}
@@ -136,8 +161,7 @@ public class PyRequirement {
public int hashCode() {
int result = myName.hashCode();
result = 31 * result + myVersionSpecs.hashCode();
result = 31 * result + (myURL != null ? myURL.hashCode() : 0);
result = 31 * result + (myEditable ? 1 : 0);
result = 31 * result + myOptions.hashCode();
return result;
}
@@ -147,27 +171,20 @@ public class PyRequirement {
if (normalizeName(myName).equalsIgnoreCase(pkg.getName())) {
for (PyRequirementVersionSpec spec : myVersionSpecs) {
final int cmp = PackageVersionComparator.VERSION_COMPARATOR.compare(pkg.getVersion(), spec.getVersion());
final PyRequirementRelation relation = spec.getRelation();
if (!relation.isSuccessful(cmp)) {
if (!spec.getRelation().isSuccessful(cmp)) {
return null;
}
}
return pkg;
}
}
return null;
}
/**
* Parses requirement string as described in [pep-0386].
* For example: "myPackage&lt;=10.6a3"
*
* @param line requirement to parse
* @return requirement
* @throws IllegalArgumentException if line can't be parsed
*/
@NotNull
public static PyRequirement fromStringGuaranteed(@NotNull final String line) {
public static PyRequirement fromStringGuaranteed(@NotNull String line) {
final PyRequirement requirement = fromString(line);
if (requirement == null) {
throw new IllegalArgumentException("Failed to parse " + line);
@@ -177,142 +194,169 @@ public class PyRequirement {
@Nullable
public static PyRequirement fromString(@NotNull String line) {
// TODO: Extras, multi-line requirements '\'
final PyRequirement editableEgg = parseEditableEgg(line);
if (editableEgg != null) {
return editableEgg;
final PyRequirement githubArchiveUrl = parseGithubArchiveUrl(line);
if (githubArchiveUrl != null) {
return githubArchiveUrl;
}
final Matcher nameMatcher = NAME.matcher(line);
if (!nameMatcher.matches()) {
return null;
final PyRequirement archiveUrl = parseArchiveUrl(line);
if (archiveUrl != null) {
return archiveUrl;
}
final String name = nameMatcher.group(1);
final String rest = nameMatcher.group(3);
final List<PyRequirementVersionSpec> versionSpecs = new ArrayList<PyRequirementVersionSpec>();
if (!rest.trim().isEmpty()) {
final Matcher versionSpecMatcher = VERSION_SPEC.matcher(rest);
while (versionSpecMatcher.find()) {
final String rel = versionSpecMatcher.group(1);
final String version = versionSpecMatcher.group(2);
final PyRequirementRelation relation = PyRequirementRelation.fromString(rel);
if (relation == null) {
return null;
}
versionSpecs.add(new PyRequirementVersionSpec(relation, version));
}
final PyRequirement vcsProjectUrl = parseVcsProjectUrl(line);
if (vcsProjectUrl != null) {
return vcsProjectUrl;
}
return new PyRequirement(name, versionSpecs);
return null; // TODO
}
@NotNull
public static List<PyRequirement> parse(@NotNull String s) {
return parse(s, null, new HashSet<VirtualFile>());
public static List<PyRequirement> parse(@NotNull String text) {
return parse(text, null, new HashSet<>());
}
@NotNull
public static List<PyRequirement> parse(@NotNull VirtualFile file) {
return parse(file, new HashSet<VirtualFile>());
return parse(loadText(file), file, new HashSet<>());
}
@NotNull
private static List<PyRequirement> parse(@NotNull VirtualFile file, @NotNull Set<VirtualFile> visited) {
if (!visited.contains(file)) {
visited.add(file);
final Document document = FileDocumentManager.getInstance().getDocument(file);
if (document != null) {
return parse(document.getText(), file, visited);
}
}
return Collections.emptyList();
}
@NotNull
public String getName() {
return myName;
}
@NotNull
private static List<PyRequirement> parse(@NotNull String s, @Nullable VirtualFile anchor, @NotNull Set<VirtualFile> visited) {
final Set<PyRequirement> result = new LinkedHashSet<PyRequirement>();
for (String line : StringUtil.splitByLines(s)) {
final String trimmed = line.trim();
if (!trimmed.isEmpty()) {
final PyRequirement req = fromString(line);
if (req != null) {
result.add(req);
}
else if (anchor != null) {
result.addAll(parseRecursiveRequirement(trimmed, anchor, visited));
}
}
}
return new ArrayList<PyRequirement>(result);
}
@NotNull
private static List<PyRequirement> parseRecursiveRequirement(@NotNull String trimmedLine, @NotNull VirtualFile anchor,
@NotNull Set<VirtualFile> visited) {
final Matcher matcher = RECURSIVE_REQUIREMENT.matcher(trimmedLine);
if (matcher.matches()) {
final String fileName = FileUtil.toSystemIndependentName(matcher.group(1));
final VirtualFile dir = anchor.getParent();
if (dir != null) {
VirtualFile file = dir.findFileByRelativePath(fileName);
if (file == null) {
file = LocalFileSystem.getInstance().findFileByPath(fileName);
}
if (file != null) {
return parse(file, visited);
}
}
}
return Collections.emptyList();
private static List<PyRequirement> parse(@NotNull String text,
@Nullable VirtualFile containingFile,
@NotNull Set<VirtualFile> visitedFiles) {
return Arrays
.stream(StringUtil.splitByLines(text))
.map(String::trim)
.filter(line -> !line.isEmpty())
.map(PyRequirement::fromString)
.filter(req -> req != null)
.collect(Collectors.toCollection(LinkedHashSet::new))
.stream()
.collect(Collectors.toList());
}
@Nullable
private static PyRequirement parseEditableEgg(@NotNull String line) {
final Matcher editableEggMatcher = EDITABLE_EGG.matcher(line);
if (!editableEggMatcher.matches()) {
return null;
private static PyRequirement parseGithubArchiveUrl(@NotNull String line) {
final Matcher matcher = GITHUB_ARCHIVE_URL.matcher(line);
if (matcher.matches()) {
return new PyRequirement(matcher.group(ARCHIVE_NAME_GROUP), null, line, false);
}
final boolean editable = editableEggMatcher.group(1) != null;
final String url = editableEggMatcher.group(2);
String egg = editableEggMatcher.group(4);
if (url == null) {
return null;
return null;
}
@Nullable
private static PyRequirement parseArchiveUrl(@NotNull String line) {
final Matcher matcher = ARCHIVE_URL.matcher(line);
if (matcher.matches()) {
final Pair<String, String> nameAndVersion = parseNameAndVersion(matcher.group(ARCHIVE_NAME_GROUP));
return new PyRequirement(nameAndVersion.getFirst(), nameAndVersion.getSecond(), line, false);
}
if (egg == null) {
try {
final URI uri = new URI(url);
if (uri.getScheme() != null) {
String path = uri.getPath();
if (path != null) {
final String[] split = path.split("@", 2);
path = split[0];
final Matcher vcsPathMatcher = VCS_PATH.matcher(path);
if (!vcsPathMatcher.matches()) {
return null;
}
egg = vcsPathMatcher.group(1);
final String gitSuffix = ".git";
egg = StringUtil.trimEnd(egg, gitSuffix);
}
}
}
catch (URISyntaxException e) {
return null;
}
return null;
}
@Nullable
private static PyRequirement parseVcsProjectUrl(@NotNull String line) {
final Matcher vcsMatcher = VCS_PROJECT_URL.matcher(line);
if (vcsMatcher.matches()) {
return createVcsRequirement(line, vcsMatcher);
}
if (egg == null) {
return null;
final Matcher gitMatcher = GIT_PROJECT_URL.matcher(line);
if (gitMatcher.matches()) {
return createVcsRequirement(line, gitMatcher);
}
final Matcher bzrMatcher = BZR_PROJECT_URL.matcher(line);
if (bzrMatcher.matches()) {
return createVcsRequirement(line, bzrMatcher);
}
return null;
}
@NotNull
private static String loadText(@NotNull VirtualFile file) {
final Document document = FileDocumentManager.getInstance().getDocument(file);
return document == null ? "" : document.getText();
}
@NotNull
private static PyRequirement createVcsRequirement(@NotNull String line, @NotNull Matcher matcher) {
final boolean editable = matcher.group(EDITABLE_GROUP) != null;
final String path = matcher.group(PATH_GROUP);
final String egg = matcher.group(EGG_GROUP);
final String project = extractProject(dropTrunk(dropRevision(path)));
final Pair<String, String> nameAndVersion = parseNameAndVersion(egg == null ? StringUtil.trimEnd(project, ".git") : egg);
return new PyRequirement(nameAndVersion.getFirst(), nameAndVersion.getSecond(), line, editable);
}
@NotNull
private static String dropRevision(@NotNull String path) {
final int atIndex = path.lastIndexOf("@");
if (atIndex != -1) {
return path.substring(0, atIndex);
}
return path;
}
@NotNull
private static String dropTrunk(@NotNull String path) {
final String slashTrunk = "/trunk";
if (path.endsWith(slashTrunk)) {
return path.substring(0, path.length() - slashTrunk.length());
}
final String slashTrunkSlash = "/trunk/";
if (path.endsWith(slashTrunkSlash)) {
return path.substring(0, path.length() - slashTrunkSlash.length());
}
return path;
}
@NotNull
private static String extractProject(@NotNull String path) {
final int end = path.endsWith("/") ? path.length() - 1 : path.length();
final int slashIndex = path.lastIndexOf("/", end - 1);
if (slashIndex != -1) {
return path.substring(slashIndex + 1, end);
}
if (end != path.length()) {
return path.substring(0, end);
}
return path;
}
@NotNull
private static Pair<String, String> parseNameAndVersion(@NotNull String name) {
boolean isName = true;
final List<String> nameParts = new ArrayList<String>();
final List<String> versionParts = new ArrayList<String>();
for (String part : StringUtil.split(egg, "-")) {
if (part.matches("[0-9].*") || "dev".equals(part)) {
for (String part : StringUtil.split(name, "-")) {
final boolean partStartsWithDigit = !part.isEmpty() && Character.isDigit(part.charAt(0));
if (partStartsWithDigit || "dev".equals(part)) {
isName = false;
}
if (isName) {
nameParts.add(part);
}
@@ -320,9 +364,18 @@ public class PyRequirement {
versionParts.add(part);
}
}
final String name = normalizeName(StringUtil.join(nameParts, "-"));
final String version = !versionParts.isEmpty() ? normalizeVersion(StringUtil.join(versionParts, "-")) : null;
return new PyRequirement(name, version, url, editable);
return Pair.create(normalizeNameParts(nameParts), normalizeVersionParts(versionParts));
}
@NotNull
private static String normalizeNameParts(@NotNull List<String> nameParts) {
return normalizeName(StringUtil.join(nameParts, "-"));
}
@Nullable
private static String normalizeVersionParts(@NotNull List<String> versionParts) {
return versionParts.isEmpty() ? null : normalizeVersion(StringUtil.join(versionParts, "-"));
}
@NotNull
@@ -332,6 +385,6 @@ public class PyRequirement {
@NotNull
private static String normalizeVersion(@NotNull String s) {
return s.replace("_", "-").replaceAll("-?py[0-9\\.]+", "");
return s.replace("_", "-").replaceAll("-?py[\\d\\.]+", "");
}
}
@@ -207,7 +207,7 @@ public class PyPackageManagerImpl extends PyPackageManager {
}
args.addAll(extraArgs);
for (PyRequirement req : requirements) {
args.addAll(req.toOptions());
args.add(req.toOptions());
}
try {
getHelperResult(PACKAGING_TOOL, args, !useUserSite, true, null);
@@ -221,7 +221,7 @@ public class PyPackageManagerImpl extends PyPackageManager {
}
simplifiedArgs.addAll(extraArgs);
for (PyRequirement req : requirements) {
simplifiedArgs.addAll(req.toOptions());
simplifiedArgs.add(req.toOptions());
}
throw new PyExecutionException(e.getMessage(), "pip", simplifiedArgs, e.getStdout(), e.getStderr(), e.getExitCode(), e.getFixes());
}