diff --git a/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java b/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java index f05a63174dcd..69fc837a5912 100644 --- a/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java +++ b/python/openapi/src/com/jetbrains/python/packaging/PyRequirement.java @@ -17,38 +17,91 @@ package com.jetbrains.python.packaging; import com.intellij.openapi.editor.Document; import com.intellij.openapi.fileEditor.FileDocumentManager; -import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.util.Function; import com.intellij.webcore.packaging.PackageVersionComparator; import com.jetbrains.python.packaging.requirement.PyRequirementRelation; import com.jetbrains.python.packaging.requirement.PyRequirementVersionSpec; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.net.URI; -import java.net.URISyntaxException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * @author vlan * @see [PEP-0386] */ public class PyRequirement { - private static final Pattern NAME = Pattern.compile("\\s*(\\w(\\w|[-.])*)\\s*(.*)"); - private static final Pattern VERSION_SPEC = Pattern.compile("\\s*(<=?|>=?|==|!=)\\s*((\\w|[-.])+)"); - private static final Pattern EDITABLE_EGG = Pattern.compile("\\s*(-e)?\\s*([^#]*)(#egg=(.*))?"); - private static final Pattern RECURSIVE_REQUIREMENT = Pattern.compile("^-r\\s*(.*)"); - private static final Pattern VCS_PATH = Pattern.compile(".*/([^/]+)/?"); - @NotNull private final String myName; - @NotNull private final List myVersionSpecs; - @Nullable private final String myURL; - private final boolean myEditable; + @NotNull + private static final String EDITABLE_GROUP = "editable"; + + @NotNull + private static final String EDITABLE_REGEXP = "((?<" + EDITABLE_GROUP + ">-e|--editable)[ \t]+)?"; + + @NotNull + private static final String USER_AT_REGEXP = "[\\w-]+@"; + + @NotNull + private static final String PATH_GROUP = "path"; + + @NotNull + private static final String PATH_REGEXP = "(?<" + PATH_GROUP + ">[^@#\\s]+)"; + + @NotNull + private static final String REVISION_REGEXP = "(@[^#\\s]+)?"; + + @NotNull + private static final String EGG_GROUP = "egg"; + + @NotNull + private static final String EGG_REGEXP = "(#egg=(?<" + EGG_GROUP + ">\\S+))?"; + + @NotNull + private static final String COMMENT_REGEXP = "([ \t]+#.*)?"; + + @NotNull + private static final String ARCHIVE_NAME_GROUP = "name"; + + @NotNull + private static final Pattern GITHUB_ARCHIVE_URL = + Pattern.compile("https?://github\\.com/[^/\\s]+/(?<" + ARCHIVE_NAME_GROUP + ">[^/\\s]+)/archive/.+"); + + @NotNull + private static final Pattern ARCHIVE_URL = + Pattern.compile("https?://\\S+/" + + "(?<" + ARCHIVE_NAME_GROUP + ">\\S+)" + + "(\\.tar\\.gz|\\.zip)(#(sha1|sha224|sha256|sha384|sha512|md5)=\\w+)?" + COMMENT_REGEXP); + + // supports: git+user@... + @NotNull + private static final Pattern GIT_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP + + "git\\+" + USER_AT_REGEXP + "[^:\\s]+:" + + PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP); + + @NotNull + private static final Pattern BZR_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP + + "bzr\\+lp:" + + PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP); + + // supports: (bzr|git|hg|svn)(+smth)?://... + @NotNull + private static final Pattern VCS_PROJECT_URL = Pattern.compile(EDITABLE_REGEXP + + "(bzr|git|hg|svn)(\\+[A-Za-z]+)?://?[^/]+/" + + PATH_REGEXP + REVISION_REGEXP + EGG_REGEXP + COMMENT_REGEXP); + + @NotNull + private final String myName; + + @NotNull + private final String myOptions; + + @NotNull + private final List myVersionSpecs; public PyRequirement(@NotNull String name) { this(name, Collections.emptyList()); @@ -61,8 +114,7 @@ public class PyRequirement { public PyRequirement(@NotNull String name, @NotNull List versionSpecs) { myName = name; myVersionSpecs = versionSpecs; - myURL = null; - myEditable = false; + myOptions = toString(); } public PyRequirement(@NotNull String name, @Nullable String version, @NotNull String url, boolean editable) { @@ -73,61 +125,34 @@ public class PyRequirement { else { myVersionSpecs = Collections.emptyList(); } - myURL = url; - myEditable = editable; + myOptions = url; } @NotNull + public String getName() { + return myName; + } + + @NotNull + public String toOptions() { + return myOptions; + } + @Override public String toString() { - return myName + StringUtil.join(myVersionSpecs, - new Function() { - @Override - public String fun(PyRequirementVersionSpec spec) { - return spec.toString(); - } - }, - "," - ); - } - - @NotNull - public List toOptions() { - final List results = new ArrayList(); - if (myEditable) { - results.add("-e"); - } - if (myURL != null) { - final int size = myVersionSpecs.size(); - assert size <= 1; - final String urlAndName = myURL + "#egg=" + myName; - if (size == 0) { - results.add(urlAndName); - } - else { - final PyRequirementVersionSpec versionSpec = myVersionSpecs.get(0); - assert versionSpec.getRelation() == PyRequirementRelation.EQ; - results.add(urlAndName + "-" + versionSpec.getVersion()); - } - return results; - } - else { - results.add(toString()); - } - return results; + return myName + StringUtil.join(myVersionSpecs, ","); } @Override public boolean equals(Object o) { - if (this == o) return true; + if (o == this) return true; if (o == null || getClass() != o.getClass()) return false; PyRequirement that = (PyRequirement)o; - if (myEditable != that.myEditable) return false; if (!myName.equals(that.myName)) return false; - if (myURL != null ? !myURL.equals(that.myURL) : that.myURL != null) return false; if (!myVersionSpecs.equals(that.myVersionSpecs)) return false; + if (!myOptions.equals(that.myOptions)) return false; return true; } @@ -136,8 +161,7 @@ public class PyRequirement { public int hashCode() { int result = myName.hashCode(); result = 31 * result + myVersionSpecs.hashCode(); - result = 31 * result + (myURL != null ? myURL.hashCode() : 0); - result = 31 * result + (myEditable ? 1 : 0); + result = 31 * result + myOptions.hashCode(); return result; } @@ -147,27 +171,20 @@ public class PyRequirement { if (normalizeName(myName).equalsIgnoreCase(pkg.getName())) { for (PyRequirementVersionSpec spec : myVersionSpecs) { final int cmp = PackageVersionComparator.VERSION_COMPARATOR.compare(pkg.getVersion(), spec.getVersion()); - final PyRequirementRelation relation = spec.getRelation(); - if (!relation.isSuccessful(cmp)) { + + if (!spec.getRelation().isSuccessful(cmp)) { return null; } } return pkg; } } + return null; } - /** - * Parses requirement string as described in [pep-0386]. - * For example: "myPackage<=10.6a3" - * - * @param line requirement to parse - * @return requirement - * @throws IllegalArgumentException if line can't be parsed - */ @NotNull - public static PyRequirement fromStringGuaranteed(@NotNull final String line) { + public static PyRequirement fromStringGuaranteed(@NotNull String line) { final PyRequirement requirement = fromString(line); if (requirement == null) { throw new IllegalArgumentException("Failed to parse " + line); @@ -177,142 +194,169 @@ public class PyRequirement { @Nullable public static PyRequirement fromString(@NotNull String line) { - // TODO: Extras, multi-line requirements '\' - final PyRequirement editableEgg = parseEditableEgg(line); - if (editableEgg != null) { - return editableEgg; + final PyRequirement githubArchiveUrl = parseGithubArchiveUrl(line); + if (githubArchiveUrl != null) { + return githubArchiveUrl; } - final Matcher nameMatcher = NAME.matcher(line); - if (!nameMatcher.matches()) { - return null; + + final PyRequirement archiveUrl = parseArchiveUrl(line); + if (archiveUrl != null) { + return archiveUrl; } - final String name = nameMatcher.group(1); - final String rest = nameMatcher.group(3); - final List versionSpecs = new ArrayList(); - if (!rest.trim().isEmpty()) { - final Matcher versionSpecMatcher = VERSION_SPEC.matcher(rest); - while (versionSpecMatcher.find()) { - final String rel = versionSpecMatcher.group(1); - final String version = versionSpecMatcher.group(2); - final PyRequirementRelation relation = PyRequirementRelation.fromString(rel); - if (relation == null) { - return null; - } - versionSpecs.add(new PyRequirementVersionSpec(relation, version)); - } + + final PyRequirement vcsProjectUrl = parseVcsProjectUrl(line); + if (vcsProjectUrl != null) { + return vcsProjectUrl; } - return new PyRequirement(name, versionSpecs); + + return null; // TODO } @NotNull - public static List parse(@NotNull String s) { - return parse(s, null, new HashSet()); + public static List parse(@NotNull String text) { + return parse(text, null, new HashSet<>()); } @NotNull public static List parse(@NotNull VirtualFile file) { - return parse(file, new HashSet()); + return parse(loadText(file), file, new HashSet<>()); } @NotNull - private static List parse(@NotNull VirtualFile file, @NotNull Set visited) { - if (!visited.contains(file)) { - visited.add(file); - final Document document = FileDocumentManager.getInstance().getDocument(file); - if (document != null) { - return parse(document.getText(), file, visited); - } - } - return Collections.emptyList(); - } - - @NotNull - public String getName() { - return myName; - } - - @NotNull - private static List parse(@NotNull String s, @Nullable VirtualFile anchor, @NotNull Set visited) { - final Set result = new LinkedHashSet(); - for (String line : StringUtil.splitByLines(s)) { - final String trimmed = line.trim(); - if (!trimmed.isEmpty()) { - final PyRequirement req = fromString(line); - if (req != null) { - result.add(req); - } - else if (anchor != null) { - result.addAll(parseRecursiveRequirement(trimmed, anchor, visited)); - } - } - } - return new ArrayList(result); - } - - @NotNull - private static List parseRecursiveRequirement(@NotNull String trimmedLine, @NotNull VirtualFile anchor, - @NotNull Set visited) { - final Matcher matcher = RECURSIVE_REQUIREMENT.matcher(trimmedLine); - if (matcher.matches()) { - final String fileName = FileUtil.toSystemIndependentName(matcher.group(1)); - final VirtualFile dir = anchor.getParent(); - if (dir != null) { - VirtualFile file = dir.findFileByRelativePath(fileName); - if (file == null) { - file = LocalFileSystem.getInstance().findFileByPath(fileName); - } - if (file != null) { - return parse(file, visited); - } - } - } - return Collections.emptyList(); + private static List parse(@NotNull String text, + @Nullable VirtualFile containingFile, + @NotNull Set visitedFiles) { + return Arrays + .stream(StringUtil.splitByLines(text)) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .map(PyRequirement::fromString) + .filter(req -> req != null) + .collect(Collectors.toCollection(LinkedHashSet::new)) + .stream() + .collect(Collectors.toList()); } @Nullable - private static PyRequirement parseEditableEgg(@NotNull String line) { - final Matcher editableEggMatcher = EDITABLE_EGG.matcher(line); - if (!editableEggMatcher.matches()) { - return null; + private static PyRequirement parseGithubArchiveUrl(@NotNull String line) { + final Matcher matcher = GITHUB_ARCHIVE_URL.matcher(line); + + if (matcher.matches()) { + return new PyRequirement(matcher.group(ARCHIVE_NAME_GROUP), null, line, false); } - final boolean editable = editableEggMatcher.group(1) != null; - final String url = editableEggMatcher.group(2); - String egg = editableEggMatcher.group(4); - if (url == null) { - return null; + + return null; + } + + @Nullable + private static PyRequirement parseArchiveUrl(@NotNull String line) { + final Matcher matcher = ARCHIVE_URL.matcher(line); + + if (matcher.matches()) { + final Pair nameAndVersion = parseNameAndVersion(matcher.group(ARCHIVE_NAME_GROUP)); + + return new PyRequirement(nameAndVersion.getFirst(), nameAndVersion.getSecond(), line, false); } - if (egg == null) { - try { - final URI uri = new URI(url); - if (uri.getScheme() != null) { - String path = uri.getPath(); - if (path != null) { - final String[] split = path.split("@", 2); - path = split[0]; - final Matcher vcsPathMatcher = VCS_PATH.matcher(path); - if (!vcsPathMatcher.matches()) { - return null; - } - egg = vcsPathMatcher.group(1); - final String gitSuffix = ".git"; - egg = StringUtil.trimEnd(egg, gitSuffix); - } - } - } - catch (URISyntaxException e) { - return null; - } + + return null; + } + + @Nullable + private static PyRequirement parseVcsProjectUrl(@NotNull String line) { + final Matcher vcsMatcher = VCS_PROJECT_URL.matcher(line); + if (vcsMatcher.matches()) { + return createVcsRequirement(line, vcsMatcher); } - if (egg == null) { - return null; + + final Matcher gitMatcher = GIT_PROJECT_URL.matcher(line); + if (gitMatcher.matches()) { + return createVcsRequirement(line, gitMatcher); } + + final Matcher bzrMatcher = BZR_PROJECT_URL.matcher(line); + if (bzrMatcher.matches()) { + return createVcsRequirement(line, bzrMatcher); + } + + return null; + } + + @NotNull + private static String loadText(@NotNull VirtualFile file) { + final Document document = FileDocumentManager.getInstance().getDocument(file); + + return document == null ? "" : document.getText(); + } + + @NotNull + private static PyRequirement createVcsRequirement(@NotNull String line, @NotNull Matcher matcher) { + final boolean editable = matcher.group(EDITABLE_GROUP) != null; + final String path = matcher.group(PATH_GROUP); + final String egg = matcher.group(EGG_GROUP); + + final String project = extractProject(dropTrunk(dropRevision(path))); + final Pair nameAndVersion = parseNameAndVersion(egg == null ? StringUtil.trimEnd(project, ".git") : egg); + + return new PyRequirement(nameAndVersion.getFirst(), nameAndVersion.getSecond(), line, editable); + } + + @NotNull + private static String dropRevision(@NotNull String path) { + final int atIndex = path.lastIndexOf("@"); + + if (atIndex != -1) { + return path.substring(0, atIndex); + } + + return path; + } + + @NotNull + private static String dropTrunk(@NotNull String path) { + final String slashTrunk = "/trunk"; + + if (path.endsWith(slashTrunk)) { + return path.substring(0, path.length() - slashTrunk.length()); + } + + final String slashTrunkSlash = "/trunk/"; + + if (path.endsWith(slashTrunkSlash)) { + return path.substring(0, path.length() - slashTrunkSlash.length()); + } + + return path; + } + + @NotNull + private static String extractProject(@NotNull String path) { + final int end = path.endsWith("/") ? path.length() - 1 : path.length(); + final int slashIndex = path.lastIndexOf("/", end - 1); + + if (slashIndex != -1) { + return path.substring(slashIndex + 1, end); + } + + if (end != path.length()) { + return path.substring(0, end); + } + + return path; + } + + @NotNull + private static Pair parseNameAndVersion(@NotNull String name) { boolean isName = true; final List nameParts = new ArrayList(); final List versionParts = new ArrayList(); - for (String part : StringUtil.split(egg, "-")) { - if (part.matches("[0-9].*") || "dev".equals(part)) { + + for (String part : StringUtil.split(name, "-")) { + final boolean partStartsWithDigit = !part.isEmpty() && Character.isDigit(part.charAt(0)); + + if (partStartsWithDigit || "dev".equals(part)) { isName = false; } + if (isName) { nameParts.add(part); } @@ -320,9 +364,18 @@ public class PyRequirement { versionParts.add(part); } } - final String name = normalizeName(StringUtil.join(nameParts, "-")); - final String version = !versionParts.isEmpty() ? normalizeVersion(StringUtil.join(versionParts, "-")) : null; - return new PyRequirement(name, version, url, editable); + + return Pair.create(normalizeNameParts(nameParts), normalizeVersionParts(versionParts)); + } + + @NotNull + private static String normalizeNameParts(@NotNull List nameParts) { + return normalizeName(StringUtil.join(nameParts, "-")); + } + + @Nullable + private static String normalizeVersionParts(@NotNull List versionParts) { + return versionParts.isEmpty() ? null : normalizeVersion(StringUtil.join(versionParts, "-")); } @NotNull @@ -332,6 +385,6 @@ public class PyRequirement { @NotNull private static String normalizeVersion(@NotNull String s) { - return s.replace("_", "-").replaceAll("-?py[0-9\\.]+", ""); + return s.replace("_", "-").replaceAll("-?py[\\d\\.]+", ""); } } diff --git a/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java b/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java index 0de50fe83f5d..dcceb1ad8342 100644 --- a/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java +++ b/python/src/com/jetbrains/python/packaging/PyPackageManagerImpl.java @@ -207,7 +207,7 @@ public class PyPackageManagerImpl extends PyPackageManager { } args.addAll(extraArgs); for (PyRequirement req : requirements) { - args.addAll(req.toOptions()); + args.add(req.toOptions()); } try { getHelperResult(PACKAGING_TOOL, args, !useUserSite, true, null); @@ -221,7 +221,7 @@ public class PyPackageManagerImpl extends PyPackageManager { } simplifiedArgs.addAll(extraArgs); for (PyRequirement req : requirements) { - simplifiedArgs.addAll(req.toOptions()); + simplifiedArgs.add(req.toOptions()); } throw new PyExecutionException(e.getMessage(), "pip", simplifiedArgs, e.getStdout(), e.getStderr(), e.getExitCode(), e.getFixes()); }