From 0d5f27c47dfa9e45ffa957bf138ca56b2b9b2ef1 Mon Sep 17 00:00:00 2001 From: Aleksei Kniazev Date: Wed, 11 Sep 2019 15:43:23 +0300 Subject: [PATCH] imports are inserted after module-level dunder names to comply with PEP-8 (PY-23475) GitOrigin-RevId: 9cad837e708f3c9e52abea59d9e239371cb515bc --- .../codeInsight/imports/AddImportHelper.java | 23 ++++++++++-- .../jetbrains/python/psi/impl/PyFileImpl.java | 37 ++++++++++++++----- .../jetbrains/python/formatter/PyBlock.java | 6 +++ .../addImport/moduleLevelDunder.after.py | 3 ++ .../testData/addImport/moduleLevelDunder.py | 1 + .../moduleLevelDunderAndDocstring.after.py | 5 +++ .../moduleLevelDunderAndDocstring.py | 3 ++ ...oduleLevelDunderAndExistingImport.after.py | 4 ++ .../moduleLevelDunderAndExistingImport.py | 3 ++ ...uleLevelDunderAndImportFromFuture.after.py | 5 +++ .../moduleLevelDunderAndImportFromFuture.py | 3 ++ .../formatter/moduleLevelDunderWithImports.py | 6 +++ .../moduleLevelDunderWithImports_after.py | 8 ++++ ...mportFromFutureWithRegularImports.after.py | 12 ++++++ .../importFromFutureWithRegularImports.py | 12 ++++++ .../moduleLevelDunder.after.py | 11 ++++++ .../optimizeImports/moduleLevelDunder.py | 13 +++++++ ...elDunderWithImportFromFutureAbove.after.py | 13 +++++++ ...uleLevelDunderWithImportFromFutureAbove.py | 15 ++++++++ ...elDunderWithImportFromFutureBelow.after.py | 14 +++++++ ...uleLevelDunderWithImportFromFutureBelow.py | 14 +++++++ .../com/jetbrains/python/PyAddImportTest.java | 20 ++++++++++ .../com/jetbrains/python/PyFormatterTest.java | 5 +++ .../python/PyOptimizeImportsTest.java | 22 +++++++++++ 24 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 python/testData/addImport/moduleLevelDunder.after.py create mode 100644 python/testData/addImport/moduleLevelDunder.py create mode 100644 python/testData/addImport/moduleLevelDunderAndDocstring.after.py create mode 100644 python/testData/addImport/moduleLevelDunderAndDocstring.py create mode 100644 python/testData/addImport/moduleLevelDunderAndExistingImport.after.py create mode 100644 python/testData/addImport/moduleLevelDunderAndExistingImport.py create mode 100644 python/testData/addImport/moduleLevelDunderAndImportFromFuture.after.py create mode 100644 python/testData/addImport/moduleLevelDunderAndImportFromFuture.py create mode 100644 python/testData/formatter/moduleLevelDunderWithImports.py create mode 100644 python/testData/formatter/moduleLevelDunderWithImports_after.py create mode 100644 python/testData/optimizeImports/importFromFutureWithRegularImports.after.py create mode 100644 python/testData/optimizeImports/importFromFutureWithRegularImports.py create mode 100644 python/testData/optimizeImports/moduleLevelDunder.after.py create mode 100644 python/testData/optimizeImports/moduleLevelDunder.py create mode 100644 python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.after.py create mode 100644 python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.py create mode 100644 python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.after.py create mode 100644 python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.py diff --git a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/AddImportHelper.java b/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/AddImportHelper.java index a16e570c9de1..2e4f2f768626 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/AddImportHelper.java +++ b/python/python-psi-impl/src/com/jetbrains/python/codeInsight/imports/AddImportHelper.java @@ -187,7 +187,7 @@ public class AddImportHelper { PsiElement feeler = insertParent.getFirstChild(); if (feeler == null) return null; // skip initial comments and whitespace and try to get just below the last import stmt - boolean skippedOverImports = false; + boolean skippedOverStatements = false; boolean skippedOverDoc = false; PsiElement seeker = feeler; final boolean isInjected = InjectedLanguageManager.getInstance(feeler.getProject()).isInjectedFragment(feeler.getContainingFile()); @@ -206,7 +206,7 @@ public class AddImportHelper { } seeker = feeler; feeler = feeler.getNextSibling(); - skippedOverImports = true; + skippedOverStatements = true; } else if (PsiTreeUtil.instanceOf(feeler, PsiWhiteSpace.class, PsiComment.class)) { seeker = feeler; @@ -216,8 +216,13 @@ public class AddImportHelper { feeler = feeler.getNextSibling(); seeker = feeler; } + else if (isAssignmentToModuleLevelDunderName(feeler)) { + feeler = feeler.getNextSibling(); + seeker = feeler; + skippedOverStatements = true; + } // maybe we arrived at the doc comment stmt; skip over it, too - else if (!skippedOverImports && !skippedOverDoc && insertParent instanceof PyFile) { + else if (!skippedOverStatements && !skippedOverDoc && insertParent instanceof PyFile) { // this gives the literal; its parent is the expr seeker may have encountered final PsiElement docElem = DocStringUtil.findDocStringExpression((PyElement)insertParent); if (docElem != null && docElem.getParent() == feeler) { @@ -572,6 +577,18 @@ public class AddImportHelper { } } + public static boolean isAssignmentToModuleLevelDunderName(@Nullable PsiElement element) { + if (element instanceof PyAssignmentStatement && PyUtil.isTopLevel(element)) { + PyAssignmentStatement statement = (PyAssignmentStatement)element; + PyExpression[] targets = statement.getTargets(); + if (targets.length == 1) { + String name = targets[0].getName(); + return name != null && PyUtil.isSpecialName(name); + } + } + return false; + } + private static void addFileSystemItemImport(@NotNull PsiFileSystemItem target, @NotNull PsiFile file, @NotNull PyElement element) { final PsiFileSystemItem toImport = target.getParent(); if (toImport == null) return; diff --git a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFileImpl.java b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFileImpl.java index 6b455b5f0704..9c29cba6b40f 100644 --- a/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFileImpl.java +++ b/python/python-psi-impl/src/com/jetbrains/python/psi/impl/PyFileImpl.java @@ -31,6 +31,7 @@ import com.jetbrains.python.PyNames; import com.jetbrains.python.PythonFileType; import com.jetbrains.python.PythonLanguage; import com.jetbrains.python.codeInsight.controlflow.ControlFlowCache; +import com.jetbrains.python.codeInsight.imports.AddImportHelper; import com.jetbrains.python.documentation.docstrings.DocStringUtil; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.references.PyReferenceImpl; @@ -649,19 +650,37 @@ public class PyFileImpl extends PsiFileBase implements PyFile, PyExpression { public List getImportBlock() { final List result = new ArrayList<>(); final PsiElement firstChild = getFirstChild(); - final PyImportStatementBase firstImport; + PsiElement currentStatement; if (firstChild instanceof PyImportStatementBase) { - firstImport = (PyImportStatementBase)firstChild; + currentStatement = firstChild; } else { - firstImport = PsiTreeUtil.getNextSiblingOfType(firstChild, PyImportStatementBase.class); + currentStatement = PsiTreeUtil.getNextSiblingOfType(firstChild, PyImportStatementBase.class); } - if (firstImport != null) { - result.add(firstImport); - PsiElement nextImport = PyPsiUtils.getNextNonCommentSibling(firstImport, true); - while (nextImport instanceof PyImportStatementBase) { - result.add((PyImportStatementBase)nextImport); - nextImport = PyPsiUtils.getNextNonCommentSibling(nextImport, true); + if (currentStatement != null) { + // skip imports from future before module level dunders + final List fromFuture = new ArrayList<>(); + while (currentStatement instanceof PyFromImportStatement && ((PyFromImportStatement)currentStatement).isFromFuture()) { + fromFuture.add((PyImportStatementBase)currentStatement); + currentStatement = PyPsiUtils.getNextNonCommentSibling(currentStatement, true); + } + + // skip module level dunders + boolean hasModuleLevelDunders = false; + while (AddImportHelper.isAssignmentToModuleLevelDunderName(currentStatement)) { + hasModuleLevelDunders = true; + currentStatement = PyPsiUtils.getNextNonCommentSibling(currentStatement, true); + } + + // if there is an import from future and a module level-dunder between it and other imports, + // this import is not considered a part of the import block to avoid problems with "Optimize imports" and foldings + if (!hasModuleLevelDunders) { + result.addAll(fromFuture); + } + // collect imports + while (currentStatement instanceof PyImportStatementBase) { + result.add((PyImportStatementBase)currentStatement); + currentStatement = PyPsiUtils.getNextNonCommentSibling(currentStatement, true); } } return result; diff --git a/python/src/com/jetbrains/python/formatter/PyBlock.java b/python/src/com/jetbrains/python/formatter/PyBlock.java index 681387f5e383..a5ff654ada07 100644 --- a/python/src/com/jetbrains/python/formatter/PyBlock.java +++ b/python/src/com/jetbrains/python/formatter/PyBlock.java @@ -15,6 +15,7 @@ import com.jetbrains.python.PyElementTypes; import com.jetbrains.python.PyTokenTypes; import com.jetbrains.python.PythonCodeStyleService; import com.jetbrains.python.PythonDialectsTokenSetProvider; +import com.jetbrains.python.codeInsight.imports.AddImportHelper; import com.jetbrains.python.psi.*; import com.jetbrains.python.psi.impl.PyPsiUtils; import org.jetbrains.annotations.NotNull; @@ -862,6 +863,11 @@ public class PyBlock implements ASTBlock { } } + if (psi2 instanceof PyImportStatementBase && AddImportHelper.isAssignmentToModuleLevelDunderName(psi1)) { + // blank line between module-level dunder name and import statement + return Spacing.createSpacing(0, 0, 2, settings.KEEP_LINE_BREAKS, settings.KEEP_BLANK_LINES_IN_DECLARATIONS); + } + if ((PyElementTypes.CLASS_OR_FUNCTION.contains(childType1) && STATEMENT_OR_DECLARATION.contains(childType2)) || STATEMENT_OR_DECLARATION.contains(childType1) && PyElementTypes.CLASS_OR_FUNCTION.contains(childType2)) { if (PyUtil.isTopLevel(psi1)) { diff --git a/python/testData/addImport/moduleLevelDunder.after.py b/python/testData/addImport/moduleLevelDunder.after.py new file mode 100644 index 000000000000..e12f0a9ece29 --- /dev/null +++ b/python/testData/addImport/moduleLevelDunder.after.py @@ -0,0 +1,3 @@ +__author__ = "akniazev" + +from collections import OrderedDict \ No newline at end of file diff --git a/python/testData/addImport/moduleLevelDunder.py b/python/testData/addImport/moduleLevelDunder.py new file mode 100644 index 000000000000..8de996956ba2 --- /dev/null +++ b/python/testData/addImport/moduleLevelDunder.py @@ -0,0 +1 @@ +__author__ = "akniazev" \ No newline at end of file diff --git a/python/testData/addImport/moduleLevelDunderAndDocstring.after.py b/python/testData/addImport/moduleLevelDunderAndDocstring.after.py new file mode 100644 index 000000000000..01af151ba3aa --- /dev/null +++ b/python/testData/addImport/moduleLevelDunderAndDocstring.after.py @@ -0,0 +1,5 @@ +"""Top level docstring""" + +__author__ = "akniazev" + +from collections import OrderedDict \ No newline at end of file diff --git a/python/testData/addImport/moduleLevelDunderAndDocstring.py b/python/testData/addImport/moduleLevelDunderAndDocstring.py new file mode 100644 index 000000000000..12f82d9fa033 --- /dev/null +++ b/python/testData/addImport/moduleLevelDunderAndDocstring.py @@ -0,0 +1,3 @@ +"""Top level docstring""" + +__author__ = "akniazev" \ No newline at end of file diff --git a/python/testData/addImport/moduleLevelDunderAndExistingImport.after.py b/python/testData/addImport/moduleLevelDunderAndExistingImport.after.py new file mode 100644 index 000000000000..356581d027d4 --- /dev/null +++ b/python/testData/addImport/moduleLevelDunderAndExistingImport.after.py @@ -0,0 +1,4 @@ +__author__ = "akniazev" + +from collections import OrderedDict +from sys import path \ No newline at end of file diff --git a/python/testData/addImport/moduleLevelDunderAndExistingImport.py b/python/testData/addImport/moduleLevelDunderAndExistingImport.py new file mode 100644 index 000000000000..0c657ae1b017 --- /dev/null +++ b/python/testData/addImport/moduleLevelDunderAndExistingImport.py @@ -0,0 +1,3 @@ +__author__ = "akniazev" + +from sys import path \ No newline at end of file diff --git a/python/testData/addImport/moduleLevelDunderAndImportFromFuture.after.py b/python/testData/addImport/moduleLevelDunderAndImportFromFuture.after.py new file mode 100644 index 000000000000..e5b4cb09f226 --- /dev/null +++ b/python/testData/addImport/moduleLevelDunderAndImportFromFuture.after.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import + +__author__ = "akniazev" + +from collections import OrderedDict \ No newline at end of file diff --git a/python/testData/addImport/moduleLevelDunderAndImportFromFuture.py b/python/testData/addImport/moduleLevelDunderAndImportFromFuture.py new file mode 100644 index 000000000000..87e9851d031b --- /dev/null +++ b/python/testData/addImport/moduleLevelDunderAndImportFromFuture.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +__author__ = "akniazev" \ No newline at end of file diff --git a/python/testData/formatter/moduleLevelDunderWithImports.py b/python/testData/formatter/moduleLevelDunderWithImports.py new file mode 100644 index 000000000000..6cda884327e1 --- /dev/null +++ b/python/testData/formatter/moduleLevelDunderWithImports.py @@ -0,0 +1,6 @@ +""" +Docstring for file +""" +from __future__ import print_function +__author__ = "akniazev" +from collections import OrderedDict \ No newline at end of file diff --git a/python/testData/formatter/moduleLevelDunderWithImports_after.py b/python/testData/formatter/moduleLevelDunderWithImports_after.py new file mode 100644 index 000000000000..67bda18956d4 --- /dev/null +++ b/python/testData/formatter/moduleLevelDunderWithImports_after.py @@ -0,0 +1,8 @@ +""" +Docstring for file +""" +from __future__ import print_function + +__author__ = "akniazev" + +from collections import OrderedDict diff --git a/python/testData/optimizeImports/importFromFutureWithRegularImports.after.py b/python/testData/optimizeImports/importFromFutureWithRegularImports.after.py new file mode 100644 index 000000000000..46d668ae97aa --- /dev/null +++ b/python/testData/optimizeImports/importFromFutureWithRegularImports.after.py @@ -0,0 +1,12 @@ +from __future__ import print_function + +from collections import OrderedDict +from datetime import date +from datetime import time + +from foo import bar + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testData/optimizeImports/importFromFutureWithRegularImports.py b/python/testData/optimizeImports/importFromFutureWithRegularImports.py new file mode 100644 index 000000000000..d5f57fedbba9 --- /dev/null +++ b/python/testData/optimizeImports/importFromFutureWithRegularImports.py @@ -0,0 +1,12 @@ +from datetime import date +from sys import path +from foo import bar +from __future__ import print_function +from collections import OrderedDict +from datetime import time + + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testData/optimizeImports/moduleLevelDunder.after.py b/python/testData/optimizeImports/moduleLevelDunder.after.py new file mode 100644 index 000000000000..8755c086d5e5 --- /dev/null +++ b/python/testData/optimizeImports/moduleLevelDunder.after.py @@ -0,0 +1,11 @@ +__author__ = "akniazev" + +from collections import OrderedDict +from datetime import date, time + +from foo import bar + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testData/optimizeImports/moduleLevelDunder.py b/python/testData/optimizeImports/moduleLevelDunder.py new file mode 100644 index 000000000000..56f3b0adec1d --- /dev/null +++ b/python/testData/optimizeImports/moduleLevelDunder.py @@ -0,0 +1,13 @@ +__author__ = "akniazev" + +from datetime import date +from sys import path +from foo import bar +from collections import OrderedDict +from datetime import time + + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.after.py b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.after.py new file mode 100644 index 000000000000..6e76e24d95a4 --- /dev/null +++ b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.after.py @@ -0,0 +1,13 @@ +from __future__ import print_function + +__author__ = "akniazev" + +from collections import OrderedDict +from datetime import date, time + +from foo import bar + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.py b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.py new file mode 100644 index 000000000000..70e3678d2137 --- /dev/null +++ b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureAbove.py @@ -0,0 +1,15 @@ +from __future__ import print_function + +__author__ = "akniazev" + +from datetime import date +from sys import path +from foo import bar +from collections import OrderedDict +from datetime import time + + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.after.py b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.after.py new file mode 100644 index 000000000000..eaeaaaf8f1a9 --- /dev/null +++ b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.after.py @@ -0,0 +1,14 @@ +__author__ = "akniazev" + +from __future__ import print_function + +from collections import OrderedDict +from datetime import date +from datetime import time + +from foo import bar + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.py b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.py new file mode 100644 index 000000000000..d9f18f26c596 --- /dev/null +++ b/python/testData/optimizeImports/moduleLevelDunderWithImportFromFutureBelow.py @@ -0,0 +1,14 @@ +__author__ = "akniazev" + +from __future__ import print_function +from datetime import date +from sys import path +from foo import bar +from collections import OrderedDict +from datetime import time + + +date(1, 1, 1) +time(1) +OrderedDict() +bar() \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PyAddImportTest.java b/python/testSrc/com/jetbrains/python/PyAddImportTest.java index afdf4cd102fb..84cc3e313d35 100644 --- a/python/testSrc/com/jetbrains/python/PyAddImportTest.java +++ b/python/testSrc/com/jetbrains/python/PyAddImportTest.java @@ -136,6 +136,26 @@ public class PyAddImportTest extends PyTestCase { ); } + // PY-23475 + public void testModuleLevelDunder() { + doAddFromImport("collections", "OrderedDict", BUILTIN); + } + + // PY-23475 + public void testModuleLevelDunderAndImportFromFuture() { + doAddFromImport("collections", "OrderedDict", BUILTIN); + } + + // PY-23475 + public void testModuleLevelDunderAndExistingImport(){ + doAddFromImport("collections", "OrderedDict", BUILTIN); + } + + // PY-23475 + public void testModuleLevelDunderAndDocstring(){ + doAddFromImport("collections", "OrderedDict", BUILTIN); + } + private void doAddOrUpdateFromImport(final String path, final String name, final ImportPriority priority) { myFixture.configureByFile(getTestName(true) + ".py"); WriteCommandAction.runWriteCommandAction(myFixture.getProject(), () -> { diff --git a/python/testSrc/com/jetbrains/python/PyFormatterTest.java b/python/testSrc/com/jetbrains/python/PyFormatterTest.java index dfaaccf31bb8..c432a6d7b5fc 100644 --- a/python/testSrc/com/jetbrains/python/PyFormatterTest.java +++ b/python/testSrc/com/jetbrains/python/PyFormatterTest.java @@ -964,4 +964,9 @@ public class PyFormatterTest extends PyTestCase { public void testSpacesAroundColonEqInAssignmentExpression() { runWithLanguageLevel(LanguageLevel.PYTHON38, this::doTest); } + + // PY-23475 + public void testModuleLevelDunderWithImports() { + doTest(); + } } diff --git a/python/testSrc/com/jetbrains/python/PyOptimizeImportsTest.java b/python/testSrc/com/jetbrains/python/PyOptimizeImportsTest.java index 11b102f4bbce..b25d0eecba5c 100644 --- a/python/testSrc/com/jetbrains/python/PyOptimizeImportsTest.java +++ b/python/testSrc/com/jetbrains/python/PyOptimizeImportsTest.java @@ -354,6 +354,28 @@ public class PyOptimizeImportsTest extends PyTestCase { doTest(); } + // PY-23475 + public void testModuleLevelDunder() { + getPythonCodeStyleSettings().OPTIMIZE_IMPORTS_JOIN_FROM_IMPORTS_WITH_SAME_SOURCE = true; + doTest(); + } + + // PY-23475 + public void testModuleLevelDunderWithImportFromFutureAbove() { + getPythonCodeStyleSettings().OPTIMIZE_IMPORTS_JOIN_FROM_IMPORTS_WITH_SAME_SOURCE = true; + doTest(); + } + + // PY-23475 + public void testModuleLevelDunderWithImportFromFutureBelow() { + doTest(); + } + + // PY-23475 + public void testImportFromFutureWithRegularImports() { + doTest(); + } + private void doMultiFileTest() { final String testName = getTestName(true); myFixture.copyDirectoryToProject(testName, "");