From 8e3b1cb107bbdec7b234e7ebf0a7d185bbe32219 Mon Sep 17 00:00:00 2001 From: Semyon Proshev Date: Thu, 22 Jun 2017 20:11:59 +0300 Subject: [PATCH] Many pypi packages could install same modules. Support this in PyPIPackageUtil.PACKAGES_TOPLEVEL and its usages. --- .../python/codeInsight/typing/PyTypeShed.kt | 5 +-- .../PyPackageRequirementsInspection.java | 18 ++++++----- .../PyUnresolvedReferencesInspection.java | 14 +++------ .../python/packaging/PyPIPackageUtil.java | 31 ++++++++++--------- .../requirements.txt | 2 ++ .../test1.py | 2 ++ .../PyPackageRequirementsInspectionTest.java | 4 +++ 7 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/requirements.txt create mode 100644 python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/test1.py diff --git a/python/src/com/jetbrains/python/codeInsight/typing/PyTypeShed.kt b/python/src/com/jetbrains/python/codeInsight/typing/PyTypeShed.kt index 6a9d7198b33b..51b475f3aeaa 100644 --- a/python/src/com/jetbrains/python/codeInsight/typing/PyTypeShed.kt +++ b/python/src/com/jetbrains/python/codeInsight/typing/PyTypeShed.kt @@ -62,9 +62,10 @@ object PyTypeShed { if (ApplicationManager.getApplication().isUnitTestMode) { return true } - val pyPIPackage = PyPIPackageUtil.PACKAGES_TOPLEVEL[topLevelPackage] ?: topLevelPackage + val pyPIPackages = PyPIPackageUtil.PACKAGES_TOPLEVEL[topLevelPackage] ?: emptyList() val packages = PyPackageManagers.getInstance().forSdk(sdk).packages ?: return true - return PyPackageUtil.findPackage(packages, pyPIPackage) != null + return PyPackageUtil.findPackage(packages, topLevelPackage) != null || + pyPIPackages.any { PyPackageUtil.findPackage(packages, it) != null } } return false } diff --git a/python/src/com/jetbrains/python/inspections/PyPackageRequirementsInspection.java b/python/src/com/jetbrains/python/inspections/PyPackageRequirementsInspection.java index 5f33dd380f25..1490c5817fb1 100644 --- a/python/src/com/jetbrains/python/inspections/PyPackageRequirementsInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyPackageRequirementsInspection.java @@ -32,6 +32,7 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.profile.codeInspection.InspectionProjectProfileManager; import com.intellij.profile.codeInspection.ProjectInspectionProfileManager; import com.intellij.psi.*; +import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.PyBundle; import com.jetbrains.python.codeInsight.imports.AddImportHelper; import com.jetbrains.python.codeInsight.stdlib.PyStdlibUtil; @@ -159,10 +160,11 @@ public class PyPackageRequirementsInspection extends PyInspection { final String packageName = packageReferenceExpression.getName(); if (packageName != null && !myIgnoredPackages.contains(packageName)) { - final String possiblePyPIPackageName = PyPIPackageUtil.PACKAGES_TOPLEVEL.get(packageName); + final List possiblePyPIPackageNames = PyPIPackageUtil.PACKAGES_TOPLEVEL.getOrDefault(packageName, Collections.emptyList()); if (!ApplicationManager.getApplication().isUnitTestMode() && - StreamEx.of(packageName, possiblePyPIPackageName).nonNull().noneMatch(PyPIPackageUtil.INSTANCE::isInPyPI)) return; + !PyPIPackageUtil.INSTANCE.isInPyPI(packageName) && + !ContainerUtil.exists(possiblePyPIPackageNames, PyPIPackageUtil.INSTANCE::isInPyPI)) return; if (PyPackageUtil.SETUPTOOLS.equals(packageName)) return; @@ -177,16 +179,16 @@ public class PyPackageRequirementsInspection extends PyInspection { for (PyRequirement req : requirements) { final String name = req.getName(); - if (packageName.equalsIgnoreCase(name) || possiblePyPIPackageName != null && possiblePyPIPackageName.equalsIgnoreCase(name)) { + if (name.equalsIgnoreCase(packageName) || ContainerUtil.exists(possiblePyPIPackageNames, name::equalsIgnoreCase)) { return; } final String nameWhereUnderscoreReplacedWithHyphen = name.replaceAll("_", "-"); - if (possiblePyPIPackageName != null && possiblePyPIPackageName.equalsIgnoreCase(nameWhereUnderscoreReplacedWithHyphen)) { + if (ContainerUtil.exists(possiblePyPIPackageNames, nameWhereUnderscoreReplacedWithHyphen::equalsIgnoreCase)) { return; } final String nameWhereHyphenReplacedWithUnderscore = name.replaceAll("-", "_"); - if (packageName.equalsIgnoreCase(nameWhereHyphenReplacedWithUnderscore) || - possiblePyPIPackageName != null && possiblePyPIPackageName.equalsIgnoreCase(nameWhereHyphenReplacedWithUnderscore)) { + if (nameWhereHyphenReplacedWithUnderscore.equalsIgnoreCase(packageName) || + ContainerUtil.exists(possiblePyPIPackageNames, nameWhereHyphenReplacedWithUnderscore::equalsIgnoreCase)) { return; } } @@ -208,8 +210,8 @@ public class PyPackageRequirementsInspection extends PyInspection { } final String suggestedPackageName = StreamEx - .of(packageName, possiblePyPIPackageName) - .nonNull() + .of(packageName) + .append(possiblePyPIPackageNames) .findFirst(PyPIPackageUtil.INSTANCE::isInPyPI) .orElse(packageName); diff --git a/python/src/com/jetbrains/python/inspections/unresolvedReference/PyUnresolvedReferencesInspection.java b/python/src/com/jetbrains/python/inspections/unresolvedReference/PyUnresolvedReferencesInspection.java index 66ec1f247c1b..0e717b490ba2 100644 --- a/python/src/com/jetbrains/python/inspections/unresolvedReference/PyUnresolvedReferencesInspection.java +++ b/python/src/com/jetbrains/python/inspections/unresolvedReference/PyUnresolvedReferencesInspection.java @@ -635,15 +635,11 @@ public class PyUnresolvedReferencesInspection extends PyInspection { final Module module = ModuleUtilCore.findModuleForPsiElement(node); final Sdk sdk = PythonSdkType.findPythonSdk(module); if (module != null && sdk != null && PyPackageUtil.packageManagementEnabled(sdk)) { - if (PyPIPackageUtil.INSTANCE.isInPyPI(packageName)) { - addInstallPackageAction(actions, packageName, module, sdk); - } - else { - if (PyPIPackageUtil.PACKAGES_TOPLEVEL.containsKey(packageName)) { - final String suggestedPackage = PyPIPackageUtil.PACKAGES_TOPLEVEL.get(packageName); - addInstallPackageAction(actions, suggestedPackage, module, sdk); - } - } + StreamEx + .of(packageName) + .append(PyPIPackageUtil.PACKAGES_TOPLEVEL.getOrDefault(packageName, Collections.emptyList())) + .filter(PyPIPackageUtil.INSTANCE::isInPyPI) + .forEach(pkg -> addInstallPackageAction(actions, pkg, module, sdk)); } } } diff --git a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java index f278267989a5..eed7e3129dd6 100644 --- a/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java +++ b/python/src/com/jetbrains/python/packaging/PyPIPackageUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2017 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,9 +28,9 @@ import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.ApplicationNamesInfo; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.CatchingConsumer; +import com.intellij.util.SmartList; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.io.HttpRequests; import com.intellij.webcore.packaging.PackageVersionComparator; @@ -44,11 +44,12 @@ import javax.swing.text.MutableAttributeSet; import javax.swing.text.html.HTML; import javax.swing.text.html.HTMLEditorKit; import javax.swing.text.html.parser.ParserDelegator; -import java.io.FileReader; import java.io.IOException; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; @@ -64,9 +65,9 @@ public class PyPIPackageUtil { public static final String PYPI_LIST_URL = PYPI_HOST + "/simple"; /** - * Contains mapping "importable top-level package" -> "package name on PyPI". + * Contains mapping "importable top-level package" -> "package names on PyPI". */ - public static final ImmutableMap PACKAGES_TOPLEVEL = loadPackageAliases(); + public static final ImmutableMap> PACKAGES_TOPLEVEL = loadPackageAliases(); public static final PyPIPackageUtil INSTANCE = new PyPIPackageUtil(); @@ -138,15 +139,17 @@ public class PyPIPackageUtil { } @NotNull - private static ImmutableMap loadPackageAliases() { - final ImmutableMap.Builder builder = ImmutableMap.builder(); - try (FileReader reader = new FileReader(PythonHelpersLocator.getHelperPath("/tools/packages"))) { - final String text = FileUtil.loadTextAndClose(reader); - final List lines = StringUtil.split(text, "\n"); - for (String line : lines) { - final List split = StringUtil.split(line, " "); - builder.put(split.get(0), split.get(1)); - } + private static ImmutableMap> loadPackageAliases() { + final ImmutableMap.Builder> builder = ImmutableMap.builder(); + try { + Files + .lines(Paths.get(PythonHelpersLocator.getHelperPath("/tools/packages"))) + .forEach( + line -> { + final List split = StringUtil.split(line, " "); + builder.put(split.get(0), new SmartList<>(ContainerUtil.subList(split, 1))); + } + ); } catch (IOException e) { LOG.error("Cannot find \"packages\". " + e.getMessage()); diff --git a/python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/requirements.txt b/python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/requirements.txt new file mode 100644 index 000000000000..912314bcbd25 --- /dev/null +++ b/python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/requirements.txt @@ -0,0 +1,2 @@ +django-recaptcha-mozilla +pyzmq-ctypes \ No newline at end of file diff --git a/python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/test1.py b/python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/test1.py new file mode 100644 index 000000000000..262ad65a48d9 --- /dev/null +++ b/python/testData/inspections/PyPackageRequirementsInspection/OnePackageManyPossibleRequirements/test1.py @@ -0,0 +1,2 @@ +import captcha +import zmq \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java b/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java index 78cfcd4c6b12..27030486eb70 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyPackageRequirementsInspectionTest.java @@ -65,6 +65,10 @@ public class PyPackageRequirementsInspectionTest extends PyTestCase { doTest("test1.py"); } + public void testOnePackageManyPossibleRequirements() { + doTest("test1.py"); + } + private void doTest(@NotNull final String filename) { final String testName = getTestName(false); myFixture.copyDirectoryToProject("inspections/PyPackageRequirementsInspection/" + testName, "");