From d5aac1eff87d45cf0ea2694581dff992ea069ef6 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 28 Feb 2013 13:59:54 +0100 Subject: [PATCH 01/14] include resources from python-psi-api module in Python plugin build (PY-9015) --- python/build/python_plugin_build.gant | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/build/python_plugin_build.gant b/python/build/python_plugin_build.gant index 22cd2fa930a4..a364483f6720 100644 --- a/python/build/python_plugin_build.gant +++ b/python/build/python_plugin_build.gant @@ -306,6 +306,10 @@ target(name: "compile", description: "Compile module python") { patternset(refid: "resources.pt") type(type: "file") } + fileset(dir: "${pluginHome}/psi-api/resources") { + patternset(refid: "resources.pt") + type(type: "file") + } connectorPlugins.each { fileset(dir: "${pluginHome}/${it}/resources") { patternset(refid: "resources.pt") From 53abb4e61adb0973d19fcc5121bb8aaa1be2030f Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 28 Feb 2013 14:02:58 +0100 Subject: [PATCH 02/14] temp revert usage of new API to make sure the Python plugin compiles against EAP 128.18 --- .../quickfix/AddIgnoredIdentifierQuickFix.java | 8 +++++--- .../python/validation/Pep8ExternalAnnotator.java | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/python/src/com/jetbrains/python/inspections/quickfix/AddIgnoredIdentifierQuickFix.java b/python/src/com/jetbrains/python/inspections/quickfix/AddIgnoredIdentifierQuickFix.java index 9a3e081e3dd9..3ae2e3c076aa 100644 --- a/python/src/com/jetbrains/python/inspections/quickfix/AddIgnoredIdentifierQuickFix.java +++ b/python/src/com/jetbrains/python/inspections/quickfix/AddIgnoredIdentifierQuickFix.java @@ -44,10 +44,12 @@ public class AddIgnoredIdentifierQuickFix implements LocalQuickFix, LowPriorityA @Override public void applyFix(@NotNull Project project, @NotNull ProblemDescriptor descriptor) { final PsiElement context = descriptor.getPsiElement(); - InspectionProfile profile = InspectionProjectProfileManager.getInstance(project).getInspectionProfile(); - profile.modifyToolSettings(PyUnresolvedReferencesInspection.SHORT_NAME_KEY, context, new Consumer() { + InspectionProfile profile = InspectionProjectProfileManager.getInstance(project).getInspectionProfile(context); + profile.modifyProfile(new Consumer() { @Override - public void consume(PyUnresolvedReferencesInspection inspection) { + public void consume(ModifiableModel model) { + PyUnresolvedReferencesInspection inspection = + (PyUnresolvedReferencesInspection)model.getUnwrappedTool(PyUnresolvedReferencesInspection.class.getSimpleName(), context); String name = myIdentifier.toString(); if (myIgnoreAllAttributes) { name += END_WILDCARD; diff --git a/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java b/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java index 40ce37a5ab2b..01a807d3d9fd 100644 --- a/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java +++ b/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java @@ -4,6 +4,7 @@ import com.intellij.codeHighlighting.HighlightDisplayLevel; import com.intellij.codeInsight.daemon.HighlightDisplayKey; import com.intellij.codeInsight.intention.IntentionAction; import com.intellij.codeInspection.InspectionProfile; +import com.intellij.codeInspection.ModifiableModel; import com.intellij.codeInspection.ex.CustomEditInspectionToolsSettingsAction; import com.intellij.execution.process.ProcessOutput; import com.intellij.lang.annotation.Annotation; @@ -233,10 +234,10 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator() { + InspectionProjectProfileManager.getInstance(project).getInspectionProfile(file).modifyProfile(new Consumer() { @Override - public void consume(PyPep8Inspection tool) { + public void consume(ModifiableModel model) { + PyPep8Inspection tool = (PyPep8Inspection)model.getUnwrappedTool(PyPep8Inspection.INSPECTION_SHORT_NAME, file); tool.ignoredErrors.add(myCode); } }); From 95c61f72da3cf542d6959211d97273973fe41758 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 28 Feb 2013 14:41:14 +0100 Subject: [PATCH 03/14] revert more usages of new API to fix compilation --- .../python/inspections/PyUnresolvedReferencesInspection.java | 2 +- .../com/jetbrains/python/validation/Pep8ExternalAnnotator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java b/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java index f6da89696114..e9e2b2a42e3c 100644 --- a/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyUnresolvedReferencesInspection.java @@ -70,7 +70,7 @@ public class PyUnresolvedReferencesInspection extends PyInspection { public static PyUnresolvedReferencesInspection getInstance(PsiElement element) { final InspectionProfile inspectionProfile = InspectionProjectProfileManager.getInstance(element.getProject()).getInspectionProfile(); - return inspectionProfile.getUnwrappedTool(SHORT_NAME_KEY, element); + return (PyUnresolvedReferencesInspection)inspectionProfile.getUnwrappedTool(SHORT_NAME_KEY.toString(), element); } @Nls diff --git a/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java b/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java index 01a807d3d9fd..5b0124fb15c2 100644 --- a/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java +++ b/python/src/com/jetbrains/python/validation/Pep8ExternalAnnotator.java @@ -100,7 +100,7 @@ public class Pep8ExternalAnnotator extends ExternalAnnotator ignoredErrors = inspection.ignoredErrors; final int margin = CodeStyleSettingsManager.getInstance(file.getProject()).getCurrentSettings().RIGHT_MARGIN; return new State(homePath, file.getText(), profile.getErrorLevel(key, file), ignoredErrors, margin); From 0384e217d69ab52539956c85e2a880ffadb4be36 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 28 Feb 2013 14:55:36 +0100 Subject: [PATCH 04/14] fix confusion between SDK paths when "use module SDK" option is selected and a Python SDK is configured (PY-9021) --- .../run/AbstractPythonRunConfiguration.java | 23 +++++++++++++------ .../python/run/PythonCommandLineState.java | 4 ++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java b/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java index b10b80f72648..979decedb251 100644 --- a/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java +++ b/python/src/com/jetbrains/python/run/AbstractPythonRunConfiguration.java @@ -183,6 +183,15 @@ public abstract class AbstractPythonRunConfiguration Date: Fri, 1 Mar 2013 17:19:29 +0400 Subject: [PATCH 05/14] take search scope obtained from run configuration into account to navigate to correct class when user clicks on stacktrace printed in console or log (IDEA-63362) --- python/src/com/jetbrains/python/run/PythonRunner.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/src/com/jetbrains/python/run/PythonRunner.java b/python/src/com/jetbrains/python/run/PythonRunner.java index e21d977d968e..07bc8cf59ee3 100644 --- a/python/src/com/jetbrains/python/run/PythonRunner.java +++ b/python/src/com/jetbrains/python/run/PythonRunner.java @@ -48,9 +48,7 @@ public class PythonRunner extends DefaultProgramRunner { } if (executionResult == null) return null; - final RunContentBuilder contentBuilder = new RunContentBuilder(project, this, executor); - contentBuilder.setExecutionResult(executionResult); - contentBuilder.setEnvironment(env); + final RunContentBuilder contentBuilder = new RunContentBuilder(project, this, executor, executionResult, env); return contentBuilder.showRunContent(contentToReuse); } } From e22e74047fc35386c72f635f548963626fbbc6b6 Mon Sep 17 00:00:00 2001 From: Rustam Vishnyakov Date: Fri, 1 Mar 2013 17:19:58 +0400 Subject: [PATCH 06/14] Make some ReST highlighting attributes depend on Language Defaults (new API) --- .../com/jetbrains/rest/RestSyntaxHighlighter.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java b/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java index a87463df3424..5d84b70b17f1 100644 --- a/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java +++ b/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java @@ -18,6 +18,7 @@ package com.jetbrains.rest; import com.google.common.collect.Maps; import com.intellij.lexer.Lexer; +import com.intellij.openapi.editor.DefaultLanguageHighlighterColors; import com.intellij.openapi.editor.SyntaxHighlighterColors; import com.intellij.openapi.editor.colors.TextAttributesKey; import com.intellij.openapi.editor.markup.TextAttributes; @@ -35,12 +36,12 @@ import java.util.Map; public class RestSyntaxHighlighter extends SyntaxHighlighterBase implements RestTokenTypes { public static final TextAttributesKey REST_COMMENT = TextAttributesKey.createTextAttributesKey( "REST.LINE_COMMENT", - SyntaxHighlighterColors.LINE_COMMENT.getDefaultAttributes() + DefaultLanguageHighlighterColors.LINE_COMMENT ); public static final TextAttributesKey REST_SECTION_HEADER = TextAttributesKey.createTextAttributesKey( "REST.SECTION.HEADER", - SyntaxHighlighterColors.NUMBER.getDefaultAttributes() + DefaultLanguageHighlighterColors.NUMBER ); public static final TextAttributesKey REST_BOLD = TextAttributesKey.createTextAttributesKey( @@ -58,21 +59,21 @@ public class RestSyntaxHighlighter extends SyntaxHighlighterBase implements Rest public static final TextAttributesKey REST_INTERPRETED = TextAttributesKey.createTextAttributesKey( "REST.INTERPRETED", - new TextAttributes(Color.black, new Color(202, 218, 186), null, null, Font.PLAIN) + DefaultLanguageHighlighterColors.TEMPLATE_LANGUAGE_COLOR ); public static final TextAttributesKey REST_REF_NAME = TextAttributesKey.createTextAttributesKey( "REST.REF.NAME", - SyntaxHighlighterColors.STRING.getDefaultAttributes() + DefaultLanguageHighlighterColors.STRING ); public static final TextAttributesKey REST_EXPLICIT= TextAttributesKey.createTextAttributesKey( "REST.EXPLICIT", - SyntaxHighlighterColors.KEYWORD.getDefaultAttributes() + DefaultLanguageHighlighterColors.KEYWORD ); public static final TextAttributesKey REST_FIELD = TextAttributesKey.createTextAttributesKey( "REST.FIELD", - SyntaxHighlighterColors.KEYWORD.getDefaultAttributes() + DefaultLanguageHighlighterColors.INSTANCE_FIELD ); public static final TextAttributesKey REST_INLINE = TextAttributesKey.createTextAttributesKey( From 680a841d5a01935392db68faa3b65a0f15076907 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 09:50:47 +0100 Subject: [PATCH 07/14] correctly highlight __nonzero__ and __bool__ depending on language level (PY-9023) --- .../src/com/jetbrains/python/PyNames.java | 18 ++++++++++++++++-- ...pecialMethodNamesCompletionContributor.java | 7 ++++--- .../PySuperMethodCompletionContributor.java | 4 +++- .../python/validation/PyBuiltinAnnotator.java | 3 ++- .../validation/PyDefinitionsAnnotator.java | 8 +++----- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/python/psi-api/src/com/jetbrains/python/PyNames.java b/python/psi-api/src/com/jetbrains/python/PyNames.java index 0e7ef7181afe..5b1eb98bf5d4 100644 --- a/python/psi-api/src/com/jetbrains/python/PyNames.java +++ b/python/psi-api/src/com/jetbrains/python/PyNames.java @@ -2,6 +2,7 @@ package com.jetbrains.python; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.jetbrains.python.psi.LanguageLevel; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -187,7 +188,7 @@ public class PyNames { private static final BuiltinDescription _self_item_descr = new BuiltinDescription("(self, item)"); private static final BuiltinDescription _self_key_descr = new BuiltinDescription("(self, key)"); - public static final ImmutableMap BuiltinMethods = ImmutableMap.builder() + private static final ImmutableMap BuiltinMethods = ImmutableMap.builder() .put("__abs__", _only_self_descr) .put("__add__", _self_other_descr) .put("__and__", _self_other_descr) @@ -260,7 +261,6 @@ public class PyNames { .put("__ne__", _self_other_descr) .put("__neg__", _only_self_descr) .put(NEW, new BuiltinDescription("(cls, *args, **kwargs)")) - .put("__nonzero__", _only_self_descr) .put("__oct__", _only_self_descr) .put("__or__", _self_other_descr) //_BuiltinMethods.put("__path__", _only_self_descr); @@ -297,6 +297,20 @@ public class PyNames { .put("__xor__", _self_other_descr) .build(); + public static ImmutableMap PY2_BUILTIN_METHODS = ImmutableMap.builder() + .putAll(BuiltinMethods) + .put("__nonzero__", _only_self_descr) + .build(); + + public static ImmutableMap PY3_BUILTIN_METHODS = ImmutableMap.builder() + .putAll(BuiltinMethods) + .put("__bool__", _only_self_descr) + .build(); + + public static ImmutableMap getBuiltinMethods(LanguageLevel level) { + return level.isPy3K() ? PY3_BUILTIN_METHODS : PY2_BUILTIN_METHODS; + } + // canonical names, not forced by interpreter public static final String CANONICAL_SELF = "self"; public static final String BASESTRING = "basestring"; diff --git a/python/src/com/jetbrains/python/codeInsight/completion/PySpecialMethodNamesCompletionContributor.java b/python/src/com/jetbrains/python/codeInsight/completion/PySpecialMethodNamesCompletionContributor.java index 130306020fd8..5275c072f9d6 100644 --- a/python/src/com/jetbrains/python/codeInsight/completion/PySpecialMethodNamesCompletionContributor.java +++ b/python/src/com/jetbrains/python/codeInsight/completion/PySpecialMethodNamesCompletionContributor.java @@ -8,6 +8,7 @@ import com.intellij.codeInsight.lookup.TailTypeDecorator; import com.intellij.util.ProcessingContext; import com.jetbrains.python.PyNames; import com.jetbrains.python.PythonLanguage; +import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.psi.PyClass; import com.jetbrains.python.psi.PyFunction; import icons.PythonIcons; @@ -45,9 +46,9 @@ public class PySpecialMethodNamesCompletionContributor extends CompletionContrib protected void addCompletions( @NotNull final CompletionParameters parameters, final ProcessingContext context, @NotNull final CompletionResultSet result ) { - for (Map.Entry entry: PyNames.BuiltinMethods.entrySet()) { - LookupElementBuilder item; - item = LookupElementBuilder + LanguageLevel languageLevel = LanguageLevel.forElement(parameters.getOriginalFile()); + for (Map.Entry entry: PyNames.getBuiltinMethods(languageLevel).entrySet()) { + LookupElementBuilder item = LookupElementBuilder .create(entry.getKey() + entry.getValue().getSignature()) .bold() .withTypeText("predefined") diff --git a/python/src/com/jetbrains/python/codeInsight/completion/PySuperMethodCompletionContributor.java b/python/src/com/jetbrains/python/codeInsight/completion/PySuperMethodCompletionContributor.java index 8a4032b80631..197220f10607 100644 --- a/python/src/com/jetbrains/python/codeInsight/completion/PySuperMethodCompletionContributor.java +++ b/python/src/com/jetbrains/python/codeInsight/completion/PySuperMethodCompletionContributor.java @@ -10,6 +10,7 @@ import com.intellij.psi.util.PsiTreeUtil; import com.intellij.util.ProcessingContext; import com.jetbrains.python.PyNames; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.psi.PyClass; import com.jetbrains.python.psi.PyFunction; import org.jetbrains.annotations.NotNull; @@ -44,7 +45,8 @@ public class PySuperMethodCompletionContributor extends CompletionContributor { for (PyFunction function : containingClass.getMethods()) { seenNames.add(function.getName()); } - seenNames.addAll(PyNames.BuiltinMethods.keySet()); + LanguageLevel languageLevel = LanguageLevel.forElement(parameters.getOriginalFile()); + seenNames.addAll(PyNames.getBuiltinMethods(languageLevel).keySet()); for (PyClass ancestor : containingClass.iterateAncestorClasses()) { for (PyFunction superMethod : ancestor.getMethods()) { if (!seenNames.contains(superMethod.getName())) { diff --git a/python/src/com/jetbrains/python/validation/PyBuiltinAnnotator.java b/python/src/com/jetbrains/python/validation/PyBuiltinAnnotator.java index e7572433420b..b952ef67f01e 100644 --- a/python/src/com/jetbrains/python/validation/PyBuiltinAnnotator.java +++ b/python/src/com/jetbrains/python/validation/PyBuiltinAnnotator.java @@ -52,7 +52,8 @@ public class PyBuiltinAnnotator extends PyAnnotator { * @return true iff the node was highlighted. */ private boolean highlightAsAttribute(PyQualifiedExpression node, String name) { - if (PyNames.UnderscoredAttributes.contains(name) || PyNames.BuiltinMethods.containsKey(name)) { + LanguageLevel languageLevel = LanguageLevel.forElement(node); + if (PyNames.UnderscoredAttributes.contains(name) || PyNames.getBuiltinMethods(languageLevel).containsKey(name)) { // things like __len__ if ( (node.getQualifier() != null) // foo.__len__ diff --git a/python/src/com/jetbrains/python/validation/PyDefinitionsAnnotator.java b/python/src/com/jetbrains/python/validation/PyDefinitionsAnnotator.java index 53fded84244e..7893c8f52c27 100644 --- a/python/src/com/jetbrains/python/validation/PyDefinitionsAnnotator.java +++ b/python/src/com/jetbrains/python/validation/PyDefinitionsAnnotator.java @@ -6,10 +6,7 @@ import com.intellij.openapi.project.IndexNotReadyException; import com.intellij.psi.PsiElement; import com.jetbrains.python.highlighting.PyHighlighter; import com.jetbrains.python.PyNames; -import com.jetbrains.python.psi.PyClass; -import com.jetbrains.python.psi.PyDecorator; -import com.jetbrains.python.psi.PyDecoratorList; -import com.jetbrains.python.psi.PyFunction; +import com.jetbrains.python.psi.*; /** * Highlights class definitions, functrion definitions, and decorators. @@ -33,7 +30,8 @@ public class PyDefinitionsAnnotator extends PyAnnotator { if (name_node != null) { Annotation ann = getHolder().createInfoAnnotation(name_node, null); final String name = node.getName(); - if (PyNames.UnderscoredAttributes.contains(name) || PyNames.BuiltinMethods.containsKey(name)) { + LanguageLevel languageLevel = LanguageLevel.forElement(node); + if (PyNames.UnderscoredAttributes.contains(name) || PyNames.getBuiltinMethods(languageLevel).containsKey(name)) { PyClass cls = node.getContainingClass(); if (PyNames.NEW.equals(name)) { boolean new_style_class = false; From f29a942ca1fcd0e5b2fb8a61c125de7be63c1c25 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 10:20:02 +0100 Subject: [PATCH 08/14] Select Word handles escape sequences in Python string literals (PY-9014) --- .../selectWord/PyLiteralSelectionHandler.java | 15 +++++++++++++-- .../testData/selectWord/escapeSequence/after1.py | 1 + .../testData/selectWord/escapeSequence/before.py | 1 + .../com/jetbrains/python/PySelectWordTest.java | 4 ++++ 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 python/testData/selectWord/escapeSequence/after1.py create mode 100644 python/testData/selectWord/escapeSequence/before.py diff --git a/python/src/com/jetbrains/python/editor/selectWord/PyLiteralSelectionHandler.java b/python/src/com/jetbrains/python/editor/selectWord/PyLiteralSelectionHandler.java index da68837950ae..c71e22f6f2c0 100644 --- a/python/src/com/jetbrains/python/editor/selectWord/PyLiteralSelectionHandler.java +++ b/python/src/com/jetbrains/python/editor/selectWord/PyLiteralSelectionHandler.java @@ -1,14 +1,17 @@ package com.jetbrains.python.editor.selectWord; import com.intellij.codeInsight.editorActions.ExtendWordSelectionHandler; +import com.intellij.codeInsight.editorActions.SelectWordUtil; import com.intellij.lang.ASTNode; import com.intellij.openapi.editor.Editor; import com.intellij.openapi.util.TextRange; import com.intellij.psi.PsiElement; import com.intellij.psi.util.PsiTreeUtil; import com.jetbrains.python.PyTokenTypes; +import com.jetbrains.python.lexer.PyStringLiteralLexer; import com.jetbrains.python.psi.PyStringLiteralExpression; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -26,10 +29,18 @@ public class PyLiteralSelectionHandler implements ExtendWordSelectionHandler { public List select(PsiElement e, CharSequence editorText, int cursorOffset, Editor editor) { final PyStringLiteralExpression literal = PsiTreeUtil.getParentOfType(e, PyStringLiteralExpression.class); if (literal != null) { - for (TextRange stringRange : literal.getStringValueTextRanges()) { + List ranges = literal.getStringValueTextRanges(); + List nodes = literal.getStringNodes(); + for (int i = 0; i < ranges.size(); i++) { + TextRange stringRange = ranges.get(i); TextRange offsetRange = stringRange.shiftRight(literal.getTextRange().getStartOffset()); if (offsetRange.contains(cursorOffset) && offsetRange.getLength() > 1) { - return Collections.singletonList(offsetRange); + List result = new ArrayList(); + SelectWordUtil.addWordHonoringEscapeSequences(editorText, nodes.get(i).getTextRange(), cursorOffset, + new PyStringLiteralLexer(nodes.get(i).getElementType()), + result); + result.add(offsetRange); + return result; } } } diff --git a/python/testData/selectWord/escapeSequence/after1.py b/python/testData/selectWord/escapeSequence/after1.py new file mode 100644 index 000000000000..46db6e0b3918 --- /dev/null +++ b/python/testData/selectWord/escapeSequence/after1.py @@ -0,0 +1 @@ +s = 'Hello\nWorld!' \ No newline at end of file diff --git a/python/testData/selectWord/escapeSequence/before.py b/python/testData/selectWord/escapeSequence/before.py new file mode 100644 index 000000000000..be782b39a712 --- /dev/null +++ b/python/testData/selectWord/escapeSequence/before.py @@ -0,0 +1 @@ +s = 'Hello\nWorld!' \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PySelectWordTest.java b/python/testSrc/com/jetbrains/python/PySelectWordTest.java index 1f7298539ccd..bf34068dfcab 100644 --- a/python/testSrc/com/jetbrains/python/PySelectWordTest.java +++ b/python/testSrc/com/jetbrains/python/PySelectWordTest.java @@ -31,6 +31,10 @@ public class PySelectWordTest extends PyTestCase { doTest(); } + public void testEscapeSequence() { // PY-9014 + doTest(); + } + private void doTest() { @NonNls final String path = "selectWord/" + getTestName(true); myFixture.copyDirectoryToProject(path, path); From 941970979b3d84a300a9ca34028c84c3f98f9768 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 10:57:01 +0100 Subject: [PATCH 09/14] surprisingly enough it looks that up until now we didn't have any mechanism for inserting a backslash when Format wraps code in a position where implicit wrap doesn't happen (PY-9032) --- .../python/editor/PythonEnterHandler.java | 32 +++++++++++-------- .../PyWhiteSpaceFormattingStrategy.java | 18 +++++++++++ .../formatter/wrapInBinaryExpression.py | 1 + .../formatter/wrapInBinaryExpression_after.py | 2 ++ .../com/jetbrains/python/PyFormatterTest.java | 5 +++ 5 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 python/testData/formatter/wrapInBinaryExpression.py create mode 100644 python/testData/formatter/wrapInBinaryExpression_after.py diff --git a/python/src/com/jetbrains/python/editor/PythonEnterHandler.java b/python/src/com/jetbrains/python/editor/PythonEnterHandler.java index 734e11b2e239..058dd7d2e10e 100644 --- a/python/src/com/jetbrains/python/editor/PythonEnterHandler.java +++ b/python/src/com/jetbrains/python/editor/PythonEnterHandler.java @@ -159,34 +159,42 @@ public class PythonEnterHandler extends EnterHandlerDelegateAdapter { DataContext dataContext, int offset, Document doc) { + boolean autoWrapInProgress = DataManager.getInstance().loadFromDataContext(dataContext, + AutoHardWrapHandler.AUTO_WRAP_LINE_IN_PROGRESS_KEY) != null; + if (needInsertBackslash(file, offset, autoWrapInProgress)) { + doc.insertString(offset, "\\"); + caretOffset.set(caretOffset.get() + 1); + } + return Result.Continue; + } + + public static boolean needInsertBackslash(PsiFile file, int offset, boolean autoWrapInProgress) { if (offset > 0) { final PsiElement beforeCaret = file.findElementAt(offset - 1); if (beforeCaret instanceof PsiWhiteSpace && beforeCaret.getText().indexOf('\\') >= 0) { // we've got a backslash at EOL already, don't need another one - return Result.Continue; + return false; } } PsiElement statementBefore = findStatementBeforeCaret(file, offset); PsiElement statementAfter = findStatementAfterCaret(file, offset); if (statementBefore != statementAfter) { // Enter pressed at statement break - return Result.Continue; + return false; } if (statementBefore == null) { // empty file - return Result.Continue; + return false; } if (PsiTreeUtil.hasErrorElements(statementBefore)) { - final Boolean autoWrapping = - DataManager.getInstance().loadFromDataContext(dataContext, AutoHardWrapHandler.AUTO_WRAP_LINE_IN_PROGRESS_KEY); - if (autoWrapping == null) { + if (!autoWrapInProgress) { // code is already bad, don't mess it up even further - return Result.Continue; + return false; } // if we're in middle of typing, it's expected that we will have error elements } if (inFromImportParentheses(statementBefore, offset)) { - return Result.Continue; + return false; } PsiElement wrappableBefore = findWrappable(file, offset, true); @@ -210,13 +218,9 @@ public class PythonEnterHandler extends EnterHandlerDelegateAdapter { } } if (wrappableBefore instanceof PsiComment || wrappableAfter instanceof PsiComment) { - return Result.Continue; + return false; } - if (wrappableAfter == null || wrappableBefore != wrappableAfter) { - doc.insertString(offset, "\\"); - caretOffset.set(caretOffset.get() + 1); - } - return Result.Continue; + return wrappableAfter == null || wrappableBefore != wrappableAfter; } private static void insertDocStringStub(Editor editor, PsiElement element) { diff --git a/python/src/com/jetbrains/python/formatter/PyWhiteSpaceFormattingStrategy.java b/python/src/com/jetbrains/python/formatter/PyWhiteSpaceFormattingStrategy.java index 183c4e9dbc5d..6247cd1b48c0 100644 --- a/python/src/com/jetbrains/python/formatter/PyWhiteSpaceFormattingStrategy.java +++ b/python/src/com/jetbrains/python/formatter/PyWhiteSpaceFormattingStrategy.java @@ -1,7 +1,10 @@ package com.jetbrains.python.formatter; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.psi.PsiElement; import com.intellij.psi.codeStyle.CodeStyleSettings; import com.intellij.psi.formatter.StaticSymbolWhiteSpaceDefinitionStrategy; +import com.jetbrains.python.editor.PythonEnterHandler; import gnu.trove.TIntIntHashMap; import gnu.trove.TIntProcedure; import org.jetbrains.annotations.NotNull; @@ -17,6 +20,21 @@ public class PyWhiteSpaceFormattingStrategy extends StaticSymbolWhiteSpaceDefini super('\\'); } + @Override + public CharSequence adjustWhiteSpaceIfNecessary(@NotNull CharSequence whiteSpaceText, + @NotNull PsiElement startElement, + int startOffset, + int endOffset, + CodeStyleSettings codeStyleSettings) { + CharSequence whiteSpace = super.adjustWhiteSpaceIfNecessary(whiteSpaceText, startElement, startOffset, endOffset, codeStyleSettings); + if (whiteSpace.length() > 0 && whiteSpace.charAt(0) == '\n' && !StringUtil.contains(whiteSpace, 0, whiteSpace.length(), '\\') && + PythonEnterHandler.needInsertBackslash(startElement.getContainingFile(), startOffset, false)) { + PyCodeStyleSettings settings = codeStyleSettings.getCustomSettings(PyCodeStyleSettings.class); + return (settings.SPACE_BEFORE_BACKSLASH ? " \\" : "\\") + whiteSpace.toString(); + } + return whiteSpace; + } + /** * Python uses backslashes at the end of the line as indication that next line is an extension of the current one. *

diff --git a/python/testData/formatter/wrapInBinaryExpression.py b/python/testData/formatter/wrapInBinaryExpression.py new file mode 100644 index 000000000000..dfb075f72c18 --- /dev/null +++ b/python/testData/formatter/wrapInBinaryExpression.py @@ -0,0 +1 @@ +num_synapses = nest.GetDefaults("excitatory")["num_connections"] + nest.GetDefaults("inhibitory")["num_connections"] diff --git a/python/testData/formatter/wrapInBinaryExpression_after.py b/python/testData/formatter/wrapInBinaryExpression_after.py new file mode 100644 index 000000000000..de36015f1551 --- /dev/null +++ b/python/testData/formatter/wrapInBinaryExpression_after.py @@ -0,0 +1,2 @@ +num_synapses = nest.GetDefaults("excitatory")["num_connections"] + \ + nest.GetDefaults("inhibitory")["num_connections"] diff --git a/python/testSrc/com/jetbrains/python/PyFormatterTest.java b/python/testSrc/com/jetbrains/python/PyFormatterTest.java index 6eb7165ef742..a63c5b269cdb 100644 --- a/python/testSrc/com/jetbrains/python/PyFormatterTest.java +++ b/python/testSrc/com/jetbrains/python/PyFormatterTest.java @@ -312,6 +312,11 @@ public class PyFormatterTest extends PyTestCase { doTestPy3(); } + public void testWrapInBinaryExpression() { // PY-9032 + settings().RIGHT_MARGIN = 80; + doTest(); + } + private void doTest() { myFixture.configureByFile("formatter/" + getTestName(true) + ".py"); ApplicationManager.getApplication().runWriteAction(new Runnable() { From fc8a8bdedbb77d4101f78a78791440b51c185fe3 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 11:43:56 +0100 Subject: [PATCH 10/14] pull in latest pep8.py from github --- python/helpers/pep8.py | 686 ++++++++++++++++++----------------------- 1 file changed, 307 insertions(+), 379 deletions(-) diff --git a/python/helpers/pep8.py b/python/helpers/pep8.py index 3539725a3c34..60e85f373b0d 100644 --- a/python/helpers/pep8.py +++ b/python/helpers/pep8.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # pep8.py - Check Python source code formatting, according to PEP 8 -# Copyright (C) 2006 Johann C. Rocholl +# Copyright (C) 2006-2009 Johann C. Rocholl +# Copyright (C) 2009-2013 Florent Xicluna # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation files @@ -43,57 +44,8 @@ W warnings 600 deprecation 700 statements 900 syntax error - -You can add checks to this program by writing plugins. Each plugin is -a simple function that is called for each line of source code, either -physical or logical. - -Physical line: -- Raw line of text from the input file. - -Logical line: -- Multi-line statements converted to a single line. -- Stripped left and right. -- Contents of strings replaced with 'xxx' of same length. -- Comments removed. - -The check function requests physical or logical lines by the name of -the first argument: - -def maximum_line_length(physical_line) -def extraneous_whitespace(logical_line) -def blank_lines(logical_line, blank_lines, indent_level, line_number) - -The last example above demonstrates how check plugins can request -additional information with extra arguments. All attributes of the -Checker object are available. Some examples: - -lines: a list of the raw lines from the input file -tokens: the tokens that contribute to this logical line -line_number: line number in the input file -blank_lines: blank lines before this one -indent_char: first indentation character in this file (' ' or '\t') -indent_level: indentation (with tabs expanded to multiples of 8) -previous_indent_level: indentation on previous line -previous_logical: previous logical line - -The docstring of each check function shall be the relevant part of -text from PEP 8. It is printed if the user enables --show-pep8. -Several docstrings contain examples directly from the PEP 8 document. - -Okay: spam(ham[1], {eggs: 2}) -E201: spam( ham[1], {eggs: 2}) - -These examples are verified automatically when pep8.py is run with the ---doctest option. You can add examples for your own check functions. -The format is simple: "Okay" or error/warning code followed by colon -and space, the rest of the line is example source code. If you put 'r' -before the docstring, you can use \n for newline, \t for tab and \s -for space. - """ - -__version__ = '1.3.4a0' +__version__ = '1.4.5a0' import os import sys @@ -110,28 +62,30 @@ try: except ImportError: from ConfigParser import RawConfigParser -DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git' -DEFAULT_IGNORE = 'E24' +DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__' +DEFAULT_IGNORE = 'E226,E24' if sys.platform == 'win32': DEFAULT_CONFIG = os.path.expanduser(r'~\.pep8') else: DEFAULT_CONFIG = os.path.join(os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), 'pep8') +PROJECT_CONFIG = ('.pep8', 'tox.ini', 'setup.cfg') +TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') MAX_LINE_LENGTH = 79 REPORT_FORMAT = { 'default': '%(path)s:%(row)d:%(col)d: %(code)s %(text)s', 'pylint': '%(path)s:%(row)d: [%(code)s] %(text)s', } - +PyCF_ONLY_AST = 1024 SINGLETONS = frozenset(['False', 'None', 'True']) KEYWORDS = frozenset(keyword.kwlist + ['print']) - SINGLETONS -BINARY_OPERATORS = frozenset([ - '**=', '*=', '+=', '-=', '!=', '<>', - '%=', '^=', '&=', '|=', '==', '/=', '//=', '<=', '>=', '<<=', '>>=', - '%', '^', '&', '|', '=', '/', '//', '<', '>', '<<']) UNARY_OPERATORS = frozenset(['>>', '**', '*', '+', '-']) -OPERATORS = BINARY_OPERATORS | UNARY_OPERATORS +ARITHMETIC_OP = frozenset(['**', '*', '/', '//', '+', '-']) +WS_OPTIONAL_OPERATORS = ARITHMETIC_OP.union(['^', '&', '|', '<<', '>>', '%']) +WS_NEEDED_OPERATORS = frozenset([ + '**=', '*=', '/=', '//=', '+=', '-=', '!=', '<>', '<', '>', + '%=', '^=', '&=', '|=', '==', '<=', '>=', '<<=', '>>=', '=']) WHITESPACE = frozenset(' \t') SKIP_TOKENS = frozenset([tokenize.COMMENT, tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT]) @@ -140,19 +94,17 @@ BENCHMARK_KEYS = ['directories', 'files', 'logical lines', 'physical lines'] INDENT_REGEX = re.compile(r'([ \t]*)') RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*(,)') RERAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,\s*\w+\s*,\s*\w+') -SELFTEST_REGEX = re.compile(r'(Okay|[EW]\d{3}):\s(.*)') -ERRORCODE_REGEX = re.compile(r'[EW]\d{3}') +ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') DOCSTRING_REGEX = re.compile(r'u?r?["\']') EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') COMPARE_SINGLETON_REGEX = re.compile(r'([=!]=)\s*(None|False|True)') -COMPARE_TYPE_REGEX = re.compile(r'([=!]=|is|is\s+not)\s*type(?:s\.(\w+)Type' - r'|\(\s*(\(\s*\)|[^)]*[^ )])\s*\))') -KEYWORD_REGEX = re.compile(r'(?:[^\s])(\s*)\b(?:%s)\b(\s*)' % - r'|'.join(KEYWORDS)) -OPERATOR_REGEX = re.compile(r'(?:[^\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') +COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' + r'|\s*\(\s*([^)]*[^ )])\s*\))') +KEYWORD_REGEX = re.compile(r'(\s*)\b(?:%s)\b(\s*)' % r'|'.join(KEYWORDS)) +OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') LAMBDA_REGEX = re.compile(r'\blambda\b') -HUNK_REGEX = re.compile(r'^@@ -\d+,\d+ \+(\d+),(\d+) @@.*$') +HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') # Work around Python < 2.6 behaviour, which does not generate NL after # a comment which is on a line by itself. @@ -212,8 +164,8 @@ def trailing_whitespace(physical_line): The warning returned varies on whether the line itself is blank, for easier filtering for those who want to indent their blank lines. - Okay: spam(1) - W291: spam(1)\s + Okay: spam(1)\n# + W291: spam(1) \n# W293: class Foo(object):\n \n bang = 12 """ physical_line = physical_line.rstrip('\n') # chr(10), newline @@ -264,6 +216,8 @@ def maximum_line_length(physical_line, max_line_length): line = physical_line.rstrip() length = len(line) if length > max_line_length: + if noqa(line): + return if hasattr(line, 'decode'): # Python 2 # The line could contain multi-byte characters try: @@ -387,13 +341,15 @@ def missing_whitespace(logical_line): Okay: a[1:4:2] E231: ['a','b'] E231: foo(bar,baz) + E231: [{'a':'b'}] """ line = logical_line for index in range(len(line) - 1): char = line[index] if char in ',;:' and line[index + 1] not in WHITESPACE: before = line[:index] - if char == ':' and before.count('[') > before.count(']'): + if char == ':' and before.count('[') > before.count(']') and \ + before.rfind('{') < before.rfind('['): continue # Slice syntax, no space required if char == ',' and line[index + 1] == ')': continue # Allow tuple with only one element: (3,) @@ -455,7 +411,7 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): """ first_row = tokens[0][2][0] nrows = 1 + tokens[-1][2][0] - first_row - if nrows == 1: + if nrows == 1 or noqa(tokens[0][4]): return # indent_next tells us whether the next block is indented; assuming @@ -470,13 +426,14 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): # relative indents of physical lines rel_indent = [0] * nrows # visual indents - indent = [indent_level] indent_chances = {} last_indent = tokens[0][2] + indent = [last_indent[1]] if verbose >= 3: print(">>> " + tokens[0][4].rstrip()) for token_type, text, start, end, line in tokens: + newline = row < start[0] - first_row if newline: row = start[0] - first_row @@ -490,7 +447,7 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): print("... " + line.rstrip()) # record the initial indent. - rel_indent[row] = start[1] - indent_level + rel_indent[row] = expand_indent(line) - indent_level if depth: # a bracket expression in a continuation line. @@ -508,11 +465,11 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): # this line starts with a closing bracket if indent[depth]: if start[1] != indent[depth]: - yield (start, 'E124 closing bracket does not match ' - 'visual indentation') + yield (start, "E124 closing bracket does not match " + "visual indentation") elif hang: - yield (start, 'E123 closing bracket does not match ' - 'indentation of opening bracket\'s line') + yield (start, "E123 closing bracket does not match " + "indentation of opening bracket's line") elif visual_indent is True: # visual indent is verified if not indent[depth]: @@ -522,21 +479,21 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): pass elif indent[depth] and start[1] < indent[depth]: # visual indent is broken - yield (start, 'E128 continuation line ' - 'under-indented for visual indent') + yield (start, "E128 continuation line " + "under-indented for visual indent") elif hang == 4 or (indent_next and rel_indent[row] == 8): # hanging indent is verified pass else: # indent is broken if hang <= 0: - error = 'E122', 'missing indentation or outdented' + error = "E122", "missing indentation or outdented" elif indent[depth]: - error = 'E127', 'over-indented for visual indent' + error = "E127", "over-indented for visual indent" elif hang % 4: - error = 'E121', 'indentation is not a multiple of four' + error = "E121", "indentation is not a multiple of four" else: - error = 'E126', 'over-indented for hanging indent' + error = "E126", "over-indented for hanging indent" yield start, "%s continuation line %s" % error # look for visual indenting @@ -547,8 +504,12 @@ def continuation_line_indentation(logical_line, tokens, indent_level, verbose): if verbose >= 4: print("bracket depth %s indent to %s" % (depth, start[1])) # deal with implicit string concatenation - elif token_type == tokenize.STRING or text in ('u', 'ur', 'b', 'br'): + elif (token_type in (tokenize.STRING, tokenize.COMMENT) or + text in ('u', 'ur', 'b', 'br')): indent_chances[start[1]] = str + # special case for the "if" statement because len("if (") == 4 + elif not indent_chances and not row and not depth and text == 'if': + indent_chances[end[1] + 1] = True # keep track of bracket depth if token_type == tokenize.OP: @@ -665,20 +626,16 @@ def missing_whitespace_around_operator(logical_line, tokens): Okay: hypot2 = x * x + y * y Okay: c = (a + b) * (a - b) Okay: foo(bar, key='word', *args, **kwargs) - Okay: baz(**kwargs) - Okay: negative = -1 - Okay: spam(-1) Okay: alpha[:-i] - Okay: if not -5 < x < +5:\n pass - Okay: lambda *args, **kw: (args, kw) E225: i=i+1 E225: submitted +=1 - E225: x = x*2 - 1 - E225: hypot2 = x*x + y*y - E225: c = (a+b) * (a-b) - E225: c = alpha -4 + E225: x = x /2 - 1 E225: z = x **y + E226: c = (a+b) * (a-b) + E226: hypot2 = x*x + y*y + E227: c = a|b + E228: msg = fmt%(errno, errmsg) """ parens = 0 need_space = False @@ -686,7 +643,7 @@ def missing_whitespace_around_operator(logical_line, tokens): prev_text = prev_end = None for token_type, text, start, end, line in tokens: if token_type in (tokenize.NL, tokenize.NEWLINE, tokenize.ERRORTOKEN): - # ERRORTOKEN is triggered by backticks in Python 3000 + # ERRORTOKEN is triggered by backticks in Python 3 continue if text in ('(', 'lambda'): parens += 1 @@ -694,32 +651,59 @@ def missing_whitespace_around_operator(logical_line, tokens): parens -= 1 if need_space: if start != prev_end: + # Found a (probably) needed space + if need_space is not True and not need_space[1]: + yield (need_space[0], + "E225 missing whitespace around operator") need_space = False elif text == '>' and prev_text in ('<', '-'): # Tolerate the "<>" operator, even if running Python 3 # Deal with Python 3's annotated return value "->" pass else: - yield prev_end, "E225 missing whitespace around operator" + if need_space is True or need_space[1]: + # A needed trailing space was not found + yield prev_end, "E225 missing whitespace around operator" + else: + code, optype = 'E226', 'arithmetic' + if prev_text == '%': + code, optype = 'E228', 'modulo' + elif prev_text not in ARITHMETIC_OP: + code, optype = 'E227', 'bitwise or shift' + yield (need_space[0], "%s missing whitespace " + "around %s operator" % (code, optype)) need_space = False elif token_type == tokenize.OP and prev_end is not None: if text == '=' and parens: # Allow keyword args or defaults: foo(bar=None). pass - elif text in BINARY_OPERATORS: + elif text in WS_NEEDED_OPERATORS: need_space = True elif text in UNARY_OPERATORS: + # Check if the operator is being used as a binary operator # Allow unary operators: -123, -x, +1. # Allow argument unpacking: foo(*args, **kwargs). if prev_type == tokenize.OP: - if prev_text in '}])': - need_space = True + binary_usage = (prev_text in '}])') elif prev_type == tokenize.NAME: - if prev_text not in KEYWORDS: + binary_usage = (prev_text not in KEYWORDS) + else: + binary_usage = (prev_type not in SKIP_TOKENS) + + if binary_usage: + if text in WS_OPTIONAL_OPERATORS: + need_space = None + else: need_space = True - elif prev_type not in SKIP_TOKENS: - need_space = True - if need_space and start == prev_end: + elif text in WS_OPTIONAL_OPERATORS: + need_space = None + + if need_space is None: + # Surrounding space is optional, but ensure that + # trailing space matches opening space + need_space = (prev_end, start != prev_end) + elif need_space and start == prev_end: + # A needed opening space was not found yield prev_end, "E225 missing whitespace around operator" need_space = False prev_type = token_type @@ -767,12 +751,12 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): parens = 0 no_space = False prev_end = None + message = "E251 unexpected spaces around keyword / parameter equals" for token_type, text, start, end, line in tokens: if no_space: no_space = False if start != prev_end: - yield (prev_end, - "E251 no spaces around keyword / parameter equals") + yield (prev_end, message) elif token_type == tokenize.OP: if text == '(': parens += 1 @@ -781,8 +765,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): elif parens and text == '=': no_space = True if start != prev_end: - yield (prev_end, - "E251 no spaces around keyword / parameter equals") + yield (prev_end, message) prev_end = end @@ -808,7 +791,8 @@ def whitespace_before_inline_comment(logical_line, tokens): if prev_end[0] == start[0] and start[1] < prev_end[1] + 2: yield (prev_end, "E261 at least two spaces before inline comment") - if text.startswith('# ') or not text.startswith('# '): + symbol, sp, comment = text.partition(' ') + if symbol not in ('#', '#:') or comment[:1].isspace(): yield start, "E262 inline comment should start with '# '" elif token_type != tokenize.NL: prev_end = end @@ -858,10 +842,12 @@ def compound_statements(logical_line): E701: if foo == 'blah': one(); two(); three() E702: do_one(); do_two(); do_three() + E703: do_four(); # useless semicolon """ line = logical_line + last_char = len(line) - 1 found = line.find(':') - if -1 < found < len(line) - 1: + if -1 < found < last_char: before = line[:found] if (before.count('{') <= before.count('}') and # {'a': 1} (dict) before.count('[') <= before.count(']') and # [1:2] (slice) @@ -870,7 +856,10 @@ def compound_statements(logical_line): yield found, "E701 multiple statements on one line (colon)" found = line.find(';') if -1 < found: - yield found, "E702 multiple statements on one line (semicolon)" + if found < last_char: + yield found, "E702 multiple statements on one line (semicolon)" + else: + yield found, "E703 statement ends with a semicolon" def explicit_line_join(logical_line, tokens): @@ -955,16 +944,16 @@ def comparison_type(logical_line): """ match = COMPARE_TYPE_REGEX.search(logical_line) if match: - inst = match.group(3) + inst = match.group(1) if inst and isidentifier(inst) and inst not in SINGLETONS: return # Allow comparison for types which are not obvious - yield match.start(1), "E721 do not compare types, use 'isinstance()'" + yield match.start(0), "E721 do not compare types, use 'isinstance()'" def python_3000_has_key(logical_line): r""" - The {}.has_key() method will be removed in the future version of - Python. Use the 'in' operation instead. + The {}.has_key() method is removed in the Python 3. + Use the 'in' operation instead. Okay: if "alph" in d:\n print d["alph"] W601: assert d.has_key('alph') @@ -982,7 +971,7 @@ def python_3000_raise_comma(logical_line): The paren-using form is preferred because when the exception arguments are long or include string formatting, you don't need to use line continuation characters thanks to the containing parentheses. The older - form will be removed in Python 3000. + form is removed in Python 3. Okay: raise DummyError("Message") W602: raise DummyError, "Message" @@ -996,7 +985,7 @@ def python_3000_not_equal(logical_line): """ != can also be written <>, but this is an obsolete usage kept for backwards compatibility only. New code should always use !=. - The older syntax is removed in Python 3000. + The older syntax is removed in Python 3. Okay: if a != 'no': W603: if a <> 'no': @@ -1008,7 +997,7 @@ def python_3000_not_equal(logical_line): def python_3000_backticks(logical_line): """ - Backticks are removed in Python 3000. + Backticks are removed in Python 3. Use repr() instead. Okay: val = repr(1 + 2) @@ -1056,6 +1045,7 @@ else: def stdin_get_value(): return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() readlines.__doc__ = " Read the source code." +noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search def expand_indent(line): @@ -1109,6 +1099,9 @@ def mute_string(text): def parse_udiff(diff, patterns=None, parent='.'): + """Return a dictionary of matching lines.""" + # For each file of the diff, the entry key is the filename, + # and the value is a set of row numbers to consider. rv = {} path = nrows = None for line in diff.splitlines(): @@ -1117,7 +1110,8 @@ def parse_udiff(diff, patterns=None, parent='.'): nrows -= 1 continue if line[:3] == '@@ ': - row, nrows = [int(g) for g in HUNK_REGEX.match(line).groups()] + hunk_match = HUNK_REGEX.match(line) + row, nrows = [int(g or '1') for g in hunk_match.groups()] rv[path].update(range(row, row + nrows)) elif line[:3] == '+++': path = line[4:].split('\t', 1)[0] @@ -1144,18 +1138,38 @@ def filename_match(filename, patterns, default=True): ############################################################################## -def find_checks(argument_name): +_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} + + +def register_check(check, codes=None): """ - Find all globally visible functions where the first argument name - starts with argument_name. + Register a new check object. """ - for name, function in globals().items(): - if not inspect.isfunction(function): - continue - args = inspect.getargspec(function)[0] - if args and args[0].startswith(argument_name): - codes = ERRORCODE_REGEX.findall(function.__doc__ or '') - yield name, codes, function, args + def _add_check(check, kind, codes, args): + if check in _checks[kind]: + _checks[kind][check][0].extend(codes or []) + else: + _checks[kind][check] = (codes or [''], args) + if inspect.isfunction(check): + args = inspect.getargspec(check)[0] + if args and args[0] in ('physical_line', 'logical_line'): + if codes is None: + codes = ERRORCODE_REGEX.findall(check.__doc__ or '') + _add_check(check, args[0], codes, args) + elif inspect.isclass(check): + if inspect.getargspec(check.__init__)[0][:2] == ['self', 'tree']: + _add_check(check, 'tree', codes, None) + + +def init_checks_registry(): + """ + Register all globally visible functions where the first argument name + is 'physical_line' or 'logical_line'. + """ + mod = inspect.getmodule(register_check) + for (name, function) in inspect.getmembers(mod, inspect.isfunction): + register_check(function) +init_checks_registry() class Checker(object): @@ -1163,7 +1177,7 @@ class Checker(object): Load a Python source file, tokenize it, check coding style. """ - def __init__(self, filename, lines=None, + def __init__(self, filename=None, lines=None, options=None, report=None, **kwargs): if options is None: options = StyleGuide(kwargs).options @@ -1172,6 +1186,7 @@ class Checker(object): self._io_error = None self._physical_checks = options.physical_checks self._logical_checks = options.logical_checks + self._ast_checks = options.ast_checks self.max_line_length = options.max_line_length self.verbose = options.verbose self.filename = filename @@ -1193,6 +1208,16 @@ class Checker(object): self.report = report or options.report self.report_error = self.report.error + def report_invalid_syntax(self): + exc_type, exc = sys.exc_info()[:2] + offset = exc.args[1] + if len(offset) > 2: + offset = offset[1:3] + self.report_error(offset[0], offset[1] or 0, + 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), + self.report_invalid_syntax) + report_invalid_syntax.__doc__ = " Check if the syntax is valid." + def readline(self): """ Get the next line from the input buffer. @@ -1266,7 +1291,8 @@ class Checker(object): length += len(text) previous = token self.logical_line = ''.join(logical) - assert self.logical_line.strip() == self.logical_line + # With Python 2, if the line ends with '\r\r\n' the assertion fails + # assert self.logical_line.strip() == self.logical_line def check_logical(self): """ @@ -1295,6 +1321,17 @@ class Checker(object): self.report_error(orig_number, orig_offset, text, check) self.previous_logical = self.logical_line + def check_ast(self): + try: + tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + except SyntaxError: + return self.report_invalid_syntax() + for name, cls, _ in self._ast_checks: + checker = cls(tree, self.filename) + for lineno, offset, text, check in checker.run(): + if not noqa(self.lines[lineno - 1]): + self.report_error(lineno, offset, text, check) + def generate_tokens(self): if self._io_error: self.report_error(1, 0, 'E902 %s' % self._io_error, readlines) @@ -1303,20 +1340,15 @@ class Checker(object): for token in tokengen: yield token except (SyntaxError, tokenize.TokenError): - exc_type, exc = sys.exc_info()[:2] - offset = exc.args[1] - if len(offset) > 2: - offset = offset[1:3] - self.report_error(offset[0], offset[1], - 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), - self.generate_tokens) - generate_tokens.__doc__ = " Check if the syntax is valid." + self.report_invalid_syntax() def check_all(self, expected=None, line_offset=0): """ Run all checks on the input file. """ self.report.init_file(self.filename, self.lines, expected, line_offset) + if self._ast_checks: + self.check_ast() self.line_number = 0 self.indent_char = None self.indent_level = 0 @@ -1452,6 +1484,7 @@ class BaseReport(object): class FileReport(BaseReport): + """Collect the results of the checks and print only the filenames.""" print_filename = True @@ -1466,17 +1499,29 @@ class StandardReport(BaseReport): self._show_source = options.show_source self._show_pep8 = options.show_pep8 + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self._deferred_print = [] + return super(StandardReport, self).init_file( + filename, lines, expected, line_offset) + def error(self, line_number, offset, text, check): - """ - Report an error, according to options. - """ + """Report an error, according to options.""" code = super(StandardReport, self).error(line_number, offset, text, check) if code and (self.counters[code] == 1 or self._repeat): + self._deferred_print.append( + (line_number, offset, code, text[5:], check.__doc__)) + return code + + def get_file_results(self): + """Print the result and return the overall count for this file.""" + self._deferred_print.sort() + for line_number, offset, code, text, doc in self._deferred_print: print(self._fmt % { 'path': self.filename, 'row': self.line_offset + line_number, 'col': offset + 1, - 'code': code, 'text': text[5:], + 'code': code, 'text': text, }) if self._show_source: if line_number > len(self.lines): @@ -1485,9 +1530,9 @@ class StandardReport(BaseReport): line = self.lines[line_number - 1] print(line.rstrip()) print(' ' * offset + '^') - if self._show_pep8: - print(check.__doc__.lstrip('\n').rstrip()) - return code + if self._show_pep8 and doc: + print(doc.lstrip('\n').rstrip()) + return self.file_errors class DiffReport(StandardReport): @@ -1503,54 +1548,17 @@ class DiffReport(StandardReport): return super(DiffReport, self).error(line_number, offset, text, check) -class TestReport(StandardReport): - """Collect the results for the tests.""" - - def __init__(self, options): - options.benchmark_keys += ['test cases', 'failed tests'] - super(TestReport, self).__init__(options) - self._verbose = options.verbose - - def get_file_results(self): - # Check if the expected errors were found - label = '%s:%s:1' % (self.filename, self.line_offset) - codes = sorted(self.expected) - for code in codes: - if not self.counters.get(code): - self.file_errors += 1 - self.total_errors += 1 - print('%s: error %s not found' % (label, code)) - if self._verbose and not self.file_errors: - print('%s: passed (%s)' % - (label, ' '.join(codes) or 'Okay')) - self.counters['test cases'] += 1 - if self.file_errors: - self.counters['failed tests'] += 1 - # Reset counters - for key in set(self.counters) - set(self._benchmark_keys): - del self.counters[key] - self.messages = {} - return self.file_errors - - def print_results(self): - results = ("%(physical lines)d lines tested: %(files)d files, " - "%(test cases)d test cases%%s." % self.counters) - if self.total_errors: - print(results % ", %s failures" % self.total_errors) - else: - print(results % "") - print("Test failed." if self.total_errors else "Test passed.") - - class StyleGuide(object): """Initialize a PEP-8 instance with few options.""" def __init__(self, *args, **kwargs): # build options from the command line + self.checker_class = kwargs.pop('checker_class', Checker) parse_argv = kwargs.pop('parse_argv', False) config_file = kwargs.pop('config_file', None) - options, self.paths = process_options(parse_argv=parse_argv, - config_file=config_file) + parser = kwargs.pop('parser', None) + options, self.paths = process_options( + parse_argv=parse_argv, config_file=config_file, parser=parser) if args or kwargs: # build options from dict options_dict = dict(*args, **kwargs) @@ -1573,6 +1581,7 @@ class StyleGuide(object): options.ignore_code = self.ignore_code options.physical_checks = self.get_checks('physical_line') options.logical_checks = self.get_checks('logical_line') + options.ast_checks = self.get_checks('tree') self.init_report() def init_report(self, reporter=None): @@ -1587,11 +1596,14 @@ class StyleGuide(object): report = self.options.report runner = self.runner report.start() - for path in paths: - if os.path.isdir(path): - self.input_dir(path) - elif not self.excluded(path): - runner(path) + try: + for path in paths: + if os.path.isdir(path): + self.input_dir(path) + elif not self.excluded(path): + runner(path) + except KeyboardInterrupt: + print('... stopped') report.stop() return report @@ -1599,7 +1611,8 @@ class StyleGuide(object): """Run all checks on a Python source file.""" if self.options.verbose: print('checking %s' % filename) - fchecker = Checker(filename, lines=lines, options=self.options) + fchecker = self.checker_class( + filename, lines=lines, options=self.options) return fchecker.check_all(expected=expected, line_offset=line_offset) def input_dir(self, dirname): @@ -1616,7 +1629,7 @@ class StyleGuide(object): print('directory ' + root) counters['directories'] += 1 for subdir in sorted(dirs): - if self.excluded(subdir): + if self.excluded(os.path.join(root, subdir)): dirs.remove(subdir) for filename in sorted(files): # contain a pattern that matches? @@ -1629,7 +1642,10 @@ class StyleGuide(object): Check if options.exclude contains a pattern that matches filename. """ basename = os.path.basename(filename) - return filename_match(basename, self.options.exclude, default=False) + return any((filename_match(filename, self.options.exclude, + default=False), + filename_match(basename, self.options.exclude, + default=False))) def ignore_code(self, code): """ @@ -1648,168 +1664,15 @@ class StyleGuide(object): starts with argument_name and which contain selected tests. """ checks = [] - for name, codes, function, args in find_checks(argument_name): + for check, attrs in _checks[argument_name].items(): + (codes, args) = attrs if any(not (code and self.ignore_code(code)) for code in codes): - checks.append((name, function, args)) + checks.append((check.__name__, check, args)) return sorted(checks) -def init_tests(pep8style): - """ - Initialize testing framework. - - A test file can provide many tests. Each test starts with a - declaration. This declaration is a single line starting with '#:'. - It declares codes of expected failures, separated by spaces or 'Okay' - if no failure is expected. - If the file does not contain such declaration, it should pass all - tests. If the declaration is empty, following lines are not checked, - until next declaration. - - Examples: - - * Only E224 and W701 are expected: #: E224 W701 - * Following example is conform: #: Okay - * Don't check these lines: #: - """ - report = pep8style.init_report(TestReport) - runner = pep8style.input_file - - def run_tests(filename): - """Run all the tests from a file.""" - lines = readlines(filename) + ['#:\n'] - line_offset = 0 - codes = ['Okay'] - testcase = [] - count_files = report.counters['files'] - for index, line in enumerate(lines): - if not line.startswith('#:'): - if codes: - # Collect the lines of the test case - testcase.append(line) - continue - if codes and index: - codes = [c for c in codes if c != 'Okay'] - # Run the checker - runner(filename, testcase, expected=codes, - line_offset=line_offset) - # output the real line numbers - line_offset = index + 1 - # configure the expected errors - codes = line.split()[1:] - # empty the test case buffer - del testcase[:] - report.counters['files'] = count_files + 1 - return report.counters['failed tests'] - - pep8style.runner = run_tests - - -def selftest(options): - """ - Test all check functions with test cases in docstrings. - """ - count_failed = count_all = 0 - report = BaseReport(options) - counters = report.counters - checks = options.physical_checks + options.logical_checks - for name, check, argument_names in checks: - for line in check.__doc__.splitlines(): - line = line.lstrip() - match = SELFTEST_REGEX.match(line) - if match is None: - continue - code, source = match.groups() - checker = Checker(None, options=options, report=report) - for part in source.split(r'\n'): - part = part.replace(r'\t', '\t') - part = part.replace(r'\s', ' ') - checker.lines.append(part + '\n') - checker.check_all() - error = None - if code == 'Okay': - if len(counters) > len(options.benchmark_keys): - codes = [key for key in counters - if key not in options.benchmark_keys] - error = "incorrectly found %s" % ', '.join(codes) - elif not counters.get(code): - error = "failed to find %s" % code - # Keep showing errors for multiple tests - for key in set(counters) - set(options.benchmark_keys): - del counters[key] - report.messages = {} - count_all += 1 - if not error: - if options.verbose: - print("%s: %s" % (code, source)) - else: - count_failed += 1 - print("%s: %s:" % (__file__, error)) - for line in checker.lines: - print(line.rstrip()) - return count_failed, count_all - - -def read_config(options, args, arglist, parser): - """Read both user configuration and local configuration.""" - config = RawConfigParser() - - user_conf = options.config - if user_conf and os.path.isfile(user_conf): - if options.verbose: - print('user configuration: %s' % user_conf) - config.read(user_conf) - - parent = tail = args and os.path.abspath(os.path.commonprefix(args)) - while tail: - local_conf = os.path.join(parent, '.pep8') - if os.path.isfile(local_conf): - if options.verbose: - print('local configuration: %s' % local_conf) - config.read(local_conf) - break - parent, tail = os.path.split(parent) - - if config.has_section('pep8'): - option_list = dict([(o.dest, o.type or o.action) - for o in parser.option_list]) - - # First, read the default values - new_options, _ = parser.parse_args([]) - - # Second, parse the configuration - for opt in config.options('pep8'): - if options.verbose > 1: - print(' %s = %s' % (opt, config.get('pep8', opt))) - if opt.replace('_', '-') not in parser.config_options: - print('Unknown option: \'%s\'\n not in [%s]' % - (opt, ' '.join(parser.config_options))) - sys.exit(1) - normalized_opt = opt.replace('-', '_') - opt_type = option_list[normalized_opt] - if opt_type in ('int', 'count'): - value = config.getint('pep8', opt) - elif opt_type == 'string': - value = config.get('pep8', opt) - else: - assert opt_type in ('store_true', 'store_false') - value = config.getboolean('pep8', opt) - setattr(new_options, normalized_opt, value) - - # Third, overwrite with the command-line options - options, _ = parser.parse_args(arglist, values=new_options) - - return options - - -def process_options(arglist=None, parse_argv=False, config_file=None): - """Process options passed either via arglist or via command line args.""" - if not arglist and not parse_argv: - # Don't read the command line if the module is used as a library. - arglist = [] - if config_file is True: - config_file = DEFAULT_CONFIG - parser = OptionParser(version=__version__, +def get_parser(prog='pep8', version=__version__): + parser = OptionParser(prog=prog, version=version, usage="%prog [options] input ...") parser.config_options = [ 'exclude', 'filename', 'select', 'ignore', 'max-line-length', 'count', @@ -1854,27 +1717,100 @@ def process_options(arglist=None, parse_argv=False, config_file=None): help="report only lines changed according to the " "unified diff received on STDIN") group = parser.add_option_group("Testing Options") - group.add_option('--testsuite', metavar='dir', - help="run regression tests from dir") - group.add_option('--doctest', action='store_true', - help="run doctest on myself") + if os.path.exists(TESTSUITE_PATH): + group.add_option('--testsuite', metavar='dir', + help="run regression tests from dir") + group.add_option('--doctest', action='store_true', + help="run doctest on myself") group.add_option('--benchmark', action='store_true', help="measure processing speed") - group = parser.add_option_group("Configuration", description=( - "The project options are read from the [pep8] section of the .pep8 " - "file located in any parent folder of the path(s) being processed. " - "Allowed options are: %s." % ', '.join(parser.config_options))) - group.add_option('--config', metavar='path', default=config_file, - help="config file location (default: %default)") + return parser + +def read_config(options, args, arglist, parser): + """Read both user configuration and local configuration.""" + config = RawConfigParser() + + user_conf = options.config + if user_conf and os.path.isfile(user_conf): + if options.verbose: + print('user configuration: %s' % user_conf) + config.read(user_conf) + + parent = tail = args and os.path.abspath(os.path.commonprefix(args)) + while tail: + for name in PROJECT_CONFIG: + local_conf = os.path.join(parent, name) + if os.path.isfile(local_conf): + break + else: + parent, tail = os.path.split(parent) + continue + if options.verbose: + print('local configuration: %s' % local_conf) + config.read(local_conf) + break + + pep8_section = parser.prog + if config.has_section(pep8_section): + option_list = dict([(o.dest, o.type or o.action) + for o in parser.option_list]) + + # First, read the default values + new_options, _ = parser.parse_args([]) + + # Second, parse the configuration + for opt in config.options(pep8_section): + if options.verbose > 1: + print(" %s = %s" % (opt, config.get(pep8_section, opt))) + if opt.replace('_', '-') not in parser.config_options: + print("Unknown option: '%s'\n not in [%s]" % + (opt, ' '.join(parser.config_options))) + sys.exit(1) + normalized_opt = opt.replace('-', '_') + opt_type = option_list[normalized_opt] + if opt_type in ('int', 'count'): + value = config.getint(pep8_section, opt) + elif opt_type == 'string': + value = config.get(pep8_section, opt) + else: + assert opt_type in ('store_true', 'store_false') + value = config.getboolean(pep8_section, opt) + setattr(new_options, normalized_opt, value) + + # Third, overwrite with the command-line options + options, _ = parser.parse_args(arglist, values=new_options) + options.doctest = options.testsuite = False + return options + + +def process_options(arglist=None, parse_argv=False, config_file=None, + parser=None): + """Process options passed either via arglist or via command line args.""" + if not arglist and not parse_argv: + # Don't read the command line if the module is used as a library. + arglist = [] + if not parser: + parser = get_parser() + if not parser.has_option('--config'): + if config_file is True: + config_file = DEFAULT_CONFIG + group = parser.add_option_group("Configuration", description=( + "The project options are read from the [%s] section of the " + "tox.ini file or the setup.cfg file located in any parent folder " + "of the path(s) being processed. Allowed options are: %s." % + (parser.prog, ', '.join(parser.config_options)))) + group.add_option('--config', metavar='path', default=config_file, + help="user config file location (default: %default)") options, args = parser.parse_args(arglist) options.reporter = None - if options.testsuite: + if options.ensure_value('testsuite', False): args.append(options.testsuite) - elif not options.doctest: + elif not options.ensure_value('doctest', False): if parse_argv and not args: - if os.path.exists('.pep8') or options.diff: + if options.diff or any(os.path.exists(name) + for name in PROJECT_CONFIG): args = ['.'] else: parser.error('input not specified') @@ -1907,20 +1843,13 @@ def _main(): """Parse options and run checks on Python source.""" pep8style = StyleGuide(parse_argv=True, config_file=True) options = pep8style.options - if options.doctest: - import doctest - fail_d, done_d = doctest.testmod(report=False, verbose=options.verbose) - fail_s, done_s = selftest(options) - count_failed = fail_s + fail_d - if not options.quiet: - count_passed = done_d + done_s - count_failed - print("%d passed and %d failed." % (count_passed, count_failed)) - print("Test failed." if count_failed else "Test passed.") - if count_failed: - sys.exit(1) - if options.testsuite: - init_tests(pep8style) - report = pep8style.check_files() + if options.doctest or options.testsuite: + sys.path[:0] = [TESTSUITE_PATH] + from test_pep8 import run_tests + del sys.path[0] + report = run_tests(pep8style, options.doctest, options.testsuite) + else: + report = pep8style.check_files() if options.statistics: report.print_statistics() if options.benchmark: @@ -1932,6 +1861,5 @@ def _main(): sys.stderr.write(str(report.total_errors) + '\n') sys.exit(1) - if __name__ == '__main__': _main() From 30110c23719ba929a9fe25aa2f817754bc77d541 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 12:00:56 +0100 Subject: [PATCH 11/14] "space within method declaration parentheses" code style option (PY-8818) --- .../formatter/PyLanguageCodeStyleSettingsProvider.java | 1 + .../python/formatter/PythonFormattingModelBuilder.java | 1 + .../testData/formatter/spaceWithinDeclarationParentheses.py | 2 ++ .../formatter/spaceWithinDeclarationParentheses_after.py | 2 ++ python/testSrc/com/jetbrains/python/PyFormatterTest.java | 5 +++++ 5 files changed, 11 insertions(+) create mode 100644 python/testData/formatter/spaceWithinDeclarationParentheses.py create mode 100644 python/testData/formatter/spaceWithinDeclarationParentheses_after.py diff --git a/python/src/com/jetbrains/python/formatter/PyLanguageCodeStyleSettingsProvider.java b/python/src/com/jetbrains/python/formatter/PyLanguageCodeStyleSettingsProvider.java index c7ae487a20eb..63a8683d9134 100644 --- a/python/src/com/jetbrains/python/formatter/PyLanguageCodeStyleSettingsProvider.java +++ b/python/src/com/jetbrains/python/formatter/PyLanguageCodeStyleSettingsProvider.java @@ -46,6 +46,7 @@ public class PyLanguageCodeStyleSettingsProvider extends LanguageCodeStyleSettin "SPACE_AROUND_MULTIPLICATIVE_OPERATORS", "SPACE_AROUND_SHIFT_OPERATORS", "SPACE_WITHIN_METHOD_CALL_PARENTHESES", + "SPACE_WITHIN_METHOD_PARENTHESES", "SPACE_WITHIN_BRACKETS", "SPACE_AFTER_COMMA", "SPACE_BEFORE_COMMA", diff --git a/python/src/com/jetbrains/python/formatter/PythonFormattingModelBuilder.java b/python/src/com/jetbrains/python/formatter/PythonFormattingModelBuilder.java index 63f8003108a3..59994eece62f 100644 --- a/python/src/com/jetbrains/python/formatter/PythonFormattingModelBuilder.java +++ b/python/src/com/jetbrains/python/formatter/PythonFormattingModelBuilder.java @@ -96,6 +96,7 @@ public class PythonFormattingModelBuilder implements FormattingModelBuilderEx, C .around(DOT).spaces(0) .before(SEMICOLON).spaceIf(commonSettings.SPACE_BEFORE_SEMICOLON) .withinPairInside(LPAR, RPAR, ARGUMENT_LIST).spaceIf(commonSettings.SPACE_WITHIN_METHOD_CALL_PARENTHESES) + .withinPairInside(LPAR, RPAR, PARAMETER_LIST).spaceIf(commonSettings.SPACE_WITHIN_METHOD_PARENTHESES) .before(LBRACKET).spaceIf(pySettings.SPACE_BEFORE_LBRACKET) .withinPair(LBRACE, RBRACE).spaceIf(pySettings.SPACE_WITHIN_BRACES) diff --git a/python/testData/formatter/spaceWithinDeclarationParentheses.py b/python/testData/formatter/spaceWithinDeclarationParentheses.py new file mode 100644 index 000000000000..0f74b8576b43 --- /dev/null +++ b/python/testData/formatter/spaceWithinDeclarationParentheses.py @@ -0,0 +1,2 @@ +def foo(x, y): + pass diff --git a/python/testData/formatter/spaceWithinDeclarationParentheses_after.py b/python/testData/formatter/spaceWithinDeclarationParentheses_after.py new file mode 100644 index 000000000000..f4d1a4b80d26 --- /dev/null +++ b/python/testData/formatter/spaceWithinDeclarationParentheses_after.py @@ -0,0 +1,2 @@ +def foo( x, y ): + pass diff --git a/python/testSrc/com/jetbrains/python/PyFormatterTest.java b/python/testSrc/com/jetbrains/python/PyFormatterTest.java index a63c5b269cdb..960118f7ad4c 100644 --- a/python/testSrc/com/jetbrains/python/PyFormatterTest.java +++ b/python/testSrc/com/jetbrains/python/PyFormatterTest.java @@ -317,6 +317,11 @@ public class PyFormatterTest extends PyTestCase { doTest(); } + public void testSpaceWithinDeclarationParentheses() { // PY-8818 + settings().SPACE_WITHIN_METHOD_PARENTHESES = true; + doTest(); + } + private void doTest() { myFixture.configureByFile("formatter/" + getTestName(true) + ".py"); ApplicationManager.getApplication().runWriteAction(new Runnable() { From 1f28f8345337e5ed9c4187b9e4f300bc07dc135d Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 15:02:24 +0100 Subject: [PATCH 12/14] update pep8.py again --- python/helpers/pep8.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/helpers/pep8.py b/python/helpers/pep8.py index 60e85f373b0d..2ce7554840cc 100644 --- a/python/helpers/pep8.py +++ b/python/helpers/pep8.py @@ -256,7 +256,7 @@ def blank_lines(logical_line, blank_lines, indent_level, line_number, E303: def a():\n\n\n\n pass E304: @decorator\n\ndef a():\n pass """ - if line_number == 1: + if line_number < 3 and not previous_logical: return # Don't expect blank lines before the first line if previous_logical.startswith('@'): if blank_lines: From 63081aeecbedd93c6526ff267091eceb2322a5b8 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 15:51:02 +0100 Subject: [PATCH 13/14] move Python to new highlighter API, extract Python and ReST colors out of platform color scheme (PY-8841) --- python/rest/resources/META-INF/rest.xml | 2 + .../resources/colorSchemes/RestDarcula.xml | 27 +++++++ .../resources/colorSchemes/RestDefault.xml | 27 +++++++ python/src/META-INF/python-plugin-common.xml | 3 + .../config/BuildoutCfgSyntaxHighlighter.java | 15 ++-- .../python/highlighting/PyHighlighter.java | 75 +++++++------------ 6 files changed, 92 insertions(+), 57 deletions(-) create mode 100644 python/rest/resources/colorSchemes/RestDarcula.xml create mode 100644 python/rest/resources/colorSchemes/RestDefault.xml diff --git a/python/rest/resources/META-INF/rest.xml b/python/rest/resources/META-INF/rest.xml index 100387e906d9..46738ef87e5f 100644 --- a/python/rest/resources/META-INF/rest.xml +++ b/python/rest/resources/META-INF/rest.xml @@ -17,6 +17,8 @@ + + diff --git a/python/rest/resources/colorSchemes/RestDarcula.xml b/python/rest/resources/colorSchemes/RestDarcula.xml new file mode 100644 index 000000000000..84944f7ebe23 --- /dev/null +++ b/python/rest/resources/colorSchemes/RestDarcula.xml @@ -0,0 +1,27 @@ + + + + + + + \ No newline at end of file diff --git a/python/rest/resources/colorSchemes/RestDefault.xml b/python/rest/resources/colorSchemes/RestDefault.xml new file mode 100644 index 000000000000..7af0abfb513a --- /dev/null +++ b/python/rest/resources/colorSchemes/RestDefault.xml @@ -0,0 +1,27 @@ + + + + + + + \ No newline at end of file diff --git a/python/src/META-INF/python-plugin-common.xml b/python/src/META-INF/python-plugin-common.xml index 079051743919..e38d2676a05c 100644 --- a/python/src/META-INF/python-plugin-common.xml +++ b/python/src/META-INF/python-plugin-common.xml @@ -462,6 +462,9 @@ + + + diff --git a/python/src/com/jetbrains/python/buildout/config/BuildoutCfgSyntaxHighlighter.java b/python/src/com/jetbrains/python/buildout/config/BuildoutCfgSyntaxHighlighter.java index d83b981a189a..34c0567b0193 100644 --- a/python/src/com/jetbrains/python/buildout/config/BuildoutCfgSyntaxHighlighter.java +++ b/python/src/com/jetbrains/python/buildout/config/BuildoutCfgSyntaxHighlighter.java @@ -18,6 +18,7 @@ package com.jetbrains.python.buildout.config; import com.google.common.collect.Maps; import com.intellij.lexer.Lexer; +import com.intellij.openapi.editor.DefaultLanguageHighlighterColors; import com.intellij.openapi.editor.SyntaxHighlighterColors; import com.intellij.openapi.editor.colors.TextAttributesKey; import com.intellij.openapi.fileTypes.SyntaxHighlighterBase; @@ -41,34 +42,34 @@ public class BuildoutCfgSyntaxHighlighter extends SyntaxHighlighterBase implemen public static final TextAttributesKey BUILDOUT_SECTION_NAME = - TextAttributesKey.createTextAttributesKey("BUILDOUT.SECTION_NAME", SyntaxHighlighterColors.NUMBER.getDefaultAttributes().clone()); + TextAttributesKey.createTextAttributesKey("BUILDOUT.SECTION_NAME", SyntaxHighlighterColors.NUMBER); public static final TextAttributesKey BUILDOUT_KEY = TextAttributesKey.createTextAttributesKey( "BUILDOUT.KEY", - SyntaxHighlighterColors.KEYWORD.getDefaultAttributes() + DefaultLanguageHighlighterColors.KEYWORD ); public static final TextAttributesKey BUILDOUT_VALUE = TextAttributesKey.createTextAttributesKey( "BUILDOUT.VALUE", - SyntaxHighlighterColors.STRING.getDefaultAttributes() + DefaultLanguageHighlighterColors.STRING ); public static final TextAttributesKey BUILDOUT_COMMENT = TextAttributesKey.createTextAttributesKey( "BUILDOUT.LINE_COMMENT", - SyntaxHighlighterColors.LINE_COMMENT.getDefaultAttributes() + DefaultLanguageHighlighterColors.LINE_COMMENT ); public static final TextAttributesKey BUILDOUT_KEY_VALUE_SEPARATOR = TextAttributesKey.createTextAttributesKey( "BUILDOUT.KEY_VALUE_SEPARATOR", - SyntaxHighlighterColors.OPERATION_SIGN.getDefaultAttributes() + DefaultLanguageHighlighterColors.OPERATION_SIGN ); public static final TextAttributesKey BUILDOUT_VALID_STRING_ESCAPE = TextAttributesKey.createTextAttributesKey( "BUILDOUT.VALID_STRING_ESCAPE", - SyntaxHighlighterColors.VALID_STRING_ESCAPE.getDefaultAttributes() + DefaultLanguageHighlighterColors.VALID_STRING_ESCAPE ); public static final TextAttributesKey BUILDOUT_INVALID_STRING_ESCAPE = TextAttributesKey.createTextAttributesKey( "BUILDOUT.INVALID_STRING_ESCAPE", - SyntaxHighlighterColors.INVALID_STRING_ESCAPE.getDefaultAttributes() + DefaultLanguageHighlighterColors.INVALID_STRING_ESCAPE ); diff --git a/python/src/com/jetbrains/python/highlighting/PyHighlighter.java b/python/src/com/jetbrains/python/highlighting/PyHighlighter.java index b32606a196e3..3e59975f1003 100644 --- a/python/src/com/jetbrains/python/highlighting/PyHighlighter.java +++ b/python/src/com/jetbrains/python/highlighting/PyHighlighter.java @@ -3,9 +3,7 @@ package com.jetbrains.python.highlighting; import com.intellij.lexer.LayeredLexer; import com.intellij.lexer.Lexer; import com.intellij.openapi.editor.HighlighterColors; -import com.intellij.openapi.editor.colors.CodeInsightColors; import com.intellij.openapi.editor.colors.TextAttributesKey; -import com.intellij.openapi.editor.markup.TextAttributes; import com.intellij.openapi.fileTypes.SyntaxHighlighterBase; import com.intellij.psi.StringEscapesTokenTypes; import com.intellij.psi.tree.IElementType; @@ -16,11 +14,10 @@ import com.jetbrains.python.lexer.PythonHighlightingLexer; import com.jetbrains.python.psi.LanguageLevel; import org.jetbrains.annotations.NotNull; -import java.awt.*; import java.util.HashMap; import java.util.Map; -import static com.intellij.openapi.editor.SyntaxHighlighterColors.*; +import static com.intellij.openapi.editor.DefaultLanguageHighlighterColors.*; /** * Colors and lexer(s) needed for highlighting. @@ -56,72 +53,50 @@ public class PyHighlighter extends SyntaxHighlighterBase { return new PythonHighlightingLexer(myLanguageLevel); } - private static TextAttributesKey _copy(String name, TextAttributesKey src) { - return TextAttributesKey.createTextAttributesKey(name, src.getDefaultAttributes().clone()); - } + public static final TextAttributesKey PY_KEYWORD = TextAttributesKey.createTextAttributesKey("PY.KEYWORD", KEYWORD); - public static final TextAttributesKey PY_KEYWORD = _copy("PY.KEYWORD", KEYWORD); + public static final TextAttributesKey PY_BYTE_STRING = TextAttributesKey.createTextAttributesKey("PY.STRING.B", STRING); + public static final TextAttributesKey PY_UNICODE_STRING = TextAttributesKey.createTextAttributesKey("PY.STRING.U", STRING); + public static final TextAttributesKey PY_NUMBER = TextAttributesKey.createTextAttributesKey("PY.NUMBER", NUMBER); - public static final TextAttributesKey PY_BYTE_STRING = _copy("PY.STRING.B", STRING); - public static final TextAttributesKey PY_UNICODE_STRING = TextAttributesKey.createTextAttributesKey( - "PY.STRING.U", - new TextAttributes(new Color(0, 128, 128), null, null, null, STRING.getDefaultAttributes().getFontType()) - ); - public static final TextAttributesKey PY_NUMBER = _copy("PY.NUMBER", NUMBER); + static final TextAttributesKey PY_LINE_COMMENT = TextAttributesKey.createTextAttributesKey("PY.LINE_COMMENT", LINE_COMMENT); - static final TextAttributesKey PY_LINE_COMMENT = _copy("PY.LINE_COMMENT", LINE_COMMENT); + static final TextAttributesKey PY_OPERATION_SIGN = TextAttributesKey.createTextAttributesKey("PY.OPERATION_SIGN", OPERATION_SIGN); - static final TextAttributesKey PY_OPERATION_SIGN = _copy("PY.OPERATION_SIGN", OPERATION_SIGN); + static final TextAttributesKey PY_PARENTHS = TextAttributesKey.createTextAttributesKey("PY.PARENTHS", PARENTHESES); - static final TextAttributesKey PY_PARENTHS = _copy("PY.PARENTHS", PARENTHS); + static final TextAttributesKey PY_BRACKETS = TextAttributesKey.createTextAttributesKey("PY.BRACKETS", BRACKETS); - static final TextAttributesKey PY_BRACKETS = _copy("PY.BRACKETS", BRACKETS); + static final TextAttributesKey PY_BRACES = TextAttributesKey.createTextAttributesKey("PY.BRACES", BRACES); - static final TextAttributesKey PY_BRACES = _copy("PY.BRACES", BRACES); + static final TextAttributesKey PY_COMMA = TextAttributesKey.createTextAttributesKey("PY.COMMA", COMMA); - static final TextAttributesKey PY_COMMA = _copy("PY.COMMA", COMMA); + static final TextAttributesKey PY_DOT = TextAttributesKey.createTextAttributesKey("PY.DOT", DOT); - static final TextAttributesKey PY_DOT = _copy("PY.DOT", DOT); + public static final TextAttributesKey PY_DOC_COMMENT = TextAttributesKey.createTextAttributesKey("PY.DOC_COMMENT", DOC_COMMENT); - public static final TextAttributesKey PY_DOC_COMMENT = _copy("PY.DOC_COMMENT", DOC_COMMENT); + public static final TextAttributesKey PY_DOC_COMMENT_TAG = TextAttributesKey.createTextAttributesKey("PY.DOC_COMMENT_TAG", DOC_COMMENT_TAG); - public static final TextAttributesKey PY_DOC_COMMENT_TAG = _copy("PY.DOC_COMMENT_TAG", DOC_COMMENT_TAG); + public static final TextAttributesKey PY_DECORATOR = TextAttributesKey.createTextAttributesKey("PY.DECORATOR", IDENTIFIER); - public static final TextAttributesKey PY_DECORATOR = TextAttributesKey.createTextAttributesKey( - "PY.DECORATOR", new TextAttributes(Color.blue.darker(), null, null, null, Font.PLAIN) - ); + public static final TextAttributesKey PY_CLASS_DEFINITION = TextAttributesKey.createTextAttributesKey("PY.CLASS_DEFINITION", CLASS_NAME); - public static final TextAttributesKey PY_CLASS_DEFINITION = TextAttributesKey.createTextAttributesKey( - "PY.CLASS_DEFINITION", new TextAttributes(Color.black, null, null, null, Font.BOLD) - ); + public static final TextAttributesKey PY_FUNC_DEFINITION = TextAttributesKey.createTextAttributesKey("PY.FUNC_DEFINITION", FUNCTION_DECLARATION); - public static final TextAttributesKey PY_FUNC_DEFINITION = TextAttributesKey.createTextAttributesKey( - "PY.FUNC_DEFINITION", new TextAttributes(Color.black, null, null, null, Font.BOLD) - ); + public static final TextAttributesKey PY_PREDEFINED_DEFINITION = TextAttributesKey.createTextAttributesKey("PY.PREDEFINED_DEFINITION", PREDEFINED_SYMBOL); - public static final TextAttributesKey PY_PREDEFINED_DEFINITION = TextAttributesKey.createTextAttributesKey( - "PY.PREDEFINED_DEFINITION", new TextAttributes(Color.magenta.darker(), null, null, null, Font.BOLD) - ); + public static final TextAttributesKey PY_PREDEFINED_USAGE = TextAttributesKey.createTextAttributesKey("PY.PREDEFINED_USAGE", PREDEFINED_SYMBOL); - public static final TextAttributesKey PY_PREDEFINED_USAGE = TextAttributesKey.createTextAttributesKey( - "PY.PREDEFINED_USAGE", new TextAttributes(Color.magenta.darker(), null, null, null, Font.PLAIN) - ); + public static final TextAttributesKey PY_BUILTIN_NAME = TextAttributesKey.createTextAttributesKey("PY.BUILTIN_NAME", PREDEFINED_SYMBOL); - public static final TextAttributesKey PY_BUILTIN_NAME = TextAttributesKey.createTextAttributesKey( - "PY.BUILTIN_NAME", - new TextAttributes(KEYWORD.getDefaultAttributes().getForegroundColor(), null, null, null, Font.PLAIN) - ); + public static final TextAttributesKey PY_PARAMETER = TextAttributesKey.createTextAttributesKey("PY.PARAMETER", PARAMETER); + public static final TextAttributesKey PY_SELF_PARAMETER = TextAttributesKey.createTextAttributesKey( "PY.SELF_PARAMETER", PARAMETER); - public static final TextAttributesKey PY_PARAMETER =_copy("PY.PARAMETER", CodeInsightColors.PARAMETER_ATTRIBUTES); - public static final TextAttributesKey PY_SELF_PARAMETER = TextAttributesKey.createTextAttributesKey( - "PY.SELF_PARAMETER", new TextAttributes(new Color(148, 85, 141), null, null, null, Font.PLAIN)); + public static final TextAttributesKey PY_KEYWORD_ARGUMENT = TextAttributesKey.createTextAttributesKey("PY.KEYWORD_ARGUMENT", PARAMETER); - public static final TextAttributesKey PY_KEYWORD_ARGUMENT = TextAttributesKey.createTextAttributesKey( - "PY.KEYWORD_ARGUMENT", new TextAttributes(new Color(102, 0, 153), null, null, null, Font.PLAIN)); + public static final TextAttributesKey PY_VALID_STRING_ESCAPE = TextAttributesKey.createTextAttributesKey("PY.VALID_STRING_ESCAPE", VALID_STRING_ESCAPE); - public static final TextAttributesKey PY_VALID_STRING_ESCAPE = _copy("PY.VALID_STRING_ESCAPE", VALID_STRING_ESCAPE); - - public static final TextAttributesKey PY_INVALID_STRING_ESCAPE = _copy("PY.INVALID_STRING_ESCAPE", INVALID_STRING_ESCAPE); + public static final TextAttributesKey PY_INVALID_STRING_ESCAPE = TextAttributesKey.createTextAttributesKey("PY.INVALID_STRING_ESCAPE", INVALID_STRING_ESCAPE); /** * The 'heavy' constructor that initializes everything. PySyntaxHighlighterFactory caches such instances per level. From 5433c5949a08de6e6e2757b8a8701467c2cf30e7 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 1 Mar 2013 15:57:15 +0100 Subject: [PATCH 14/14] restore my version of the colors --- .../com/jetbrains/rest/RestSyntaxHighlighter.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java b/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java index 5d84b70b17f1..456011f3afa5 100644 --- a/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java +++ b/python/rest/src/com/jetbrains/rest/RestSyntaxHighlighter.java @@ -19,15 +19,12 @@ package com.jetbrains.rest; import com.google.common.collect.Maps; import com.intellij.lexer.Lexer; import com.intellij.openapi.editor.DefaultLanguageHighlighterColors; -import com.intellij.openapi.editor.SyntaxHighlighterColors; import com.intellij.openapi.editor.colors.TextAttributesKey; -import com.intellij.openapi.editor.markup.TextAttributes; import com.intellij.openapi.fileTypes.SyntaxHighlighterBase; import com.intellij.psi.tree.IElementType; import com.jetbrains.rest.lexer.RestFlexLexer; import org.jetbrains.annotations.NotNull; -import java.awt.*; import java.util.Map; /** @@ -46,20 +43,20 @@ public class RestSyntaxHighlighter extends SyntaxHighlighterBase implements Rest public static final TextAttributesKey REST_BOLD = TextAttributesKey.createTextAttributesKey( "REST.BOLD", - new TextAttributes(Color.black, null, null, null, Font.BOLD) + DefaultLanguageHighlighterColors.IDENTIFIER ); public static final TextAttributesKey REST_ITALIC = TextAttributesKey.createTextAttributesKey( "REST.ITALIC", - new TextAttributes(Color.black, null, null, null, Font.ITALIC) + DefaultLanguageHighlighterColors.IDENTIFIER ); public static final TextAttributesKey REST_FIXED = TextAttributesKey.createTextAttributesKey( "REST.FIXED", - new TextAttributes(Color.black, new Color(217, 217, 240), null, null, Font.PLAIN) + DefaultLanguageHighlighterColors.IDENTIFIER ); public static final TextAttributesKey REST_INTERPRETED = TextAttributesKey.createTextAttributesKey( "REST.INTERPRETED", - DefaultLanguageHighlighterColors.TEMPLATE_LANGUAGE_COLOR + DefaultLanguageHighlighterColors.IDENTIFIER ); public static final TextAttributesKey REST_REF_NAME = TextAttributesKey.createTextAttributesKey( @@ -73,12 +70,12 @@ public class RestSyntaxHighlighter extends SyntaxHighlighterBase implements Rest ); public static final TextAttributesKey REST_FIELD = TextAttributesKey.createTextAttributesKey( "REST.FIELD", - DefaultLanguageHighlighterColors.INSTANCE_FIELD + DefaultLanguageHighlighterColors.KEYWORD ); public static final TextAttributesKey REST_INLINE = TextAttributesKey.createTextAttributesKey( "REST.INLINE", - new TextAttributes(null, new Color(237, 252, 237), null, null, Font.PLAIN) + DefaultLanguageHighlighterColors.IDENTIFIER ); private static final Map ATTRIBUTES = Maps.newHashMap();