From 6a4acd9794b85c4f375bc225d83ab3f8e009fa8f Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Mon, 1 Nov 2010 19:38:41 +0300 Subject: [PATCH] attach trailing comments to statement list (PY-2137) --- .../parsing/FollowingCommentBinder.java | 32 +++++++++++++++++++ .../python/parsing/StatementParsing.java | 1 + python/testData/psi/CommentAtEndOfMethod.py | 3 ++ python/testData/psi/CommentAtEndOfMethod.txt | 21 ++++++++++++ python/testData/psi/CommentBeforeMethod.txt | 4 +-- .../com/jetbrains/python/PyIndentTest.java | 11 ++++++- .../com/jetbrains/python/PythonLexerTest.java | 9 ++++++ .../jetbrains/python/PythonParsingTest.java | 4 +++ 8 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 python/src/com/jetbrains/python/parsing/FollowingCommentBinder.java create mode 100644 python/testData/psi/CommentAtEndOfMethod.py create mode 100644 python/testData/psi/CommentAtEndOfMethod.txt diff --git a/python/src/com/jetbrains/python/parsing/FollowingCommentBinder.java b/python/src/com/jetbrains/python/parsing/FollowingCommentBinder.java new file mode 100644 index 000000000000..64ce872a60ba --- /dev/null +++ b/python/src/com/jetbrains/python/parsing/FollowingCommentBinder.java @@ -0,0 +1,32 @@ +package com.jetbrains.python.parsing; + +import com.intellij.lang.WhitespacesAndCommentsBinder; +import com.intellij.psi.tree.IElementType; +import com.jetbrains.python.PyTokenTypes; + +import java.util.List; + +/** +* @author yole +*/ +class FollowingCommentBinder implements WhitespacesAndCommentsBinder { + static final FollowingCommentBinder INSTANCE = new FollowingCommentBinder(); + + @Override + public int getEdgePosition(List tokens, boolean atStreamEdge, TokenTextGetter getter) { + int pos = 0; + // TODO[yole] handle more cases? + while (pos < tokens.size() && tokens.get(pos) == PyTokenTypes.LINE_BREAK) { + final CharSequence charSequence = getter.get(pos); + if (charSequence.length() == 0 || charSequence.charAt(charSequence.length()-1) != ' ') { + break; + } + pos++; + if (pos == tokens.size() || tokens.get(pos) != PyTokenTypes.END_OF_LINE_COMMENT) { + break; + } + pos++; + } + return pos; + } +} diff --git a/python/src/com/jetbrains/python/parsing/StatementParsing.java b/python/src/com/jetbrains/python/parsing/StatementParsing.java index 0e1a6fb77a0e..8d49292437eb 100644 --- a/python/src/com/jetbrains/python/parsing/StatementParsing.java +++ b/python/src/com/jetbrains/python/parsing/StatementParsing.java @@ -720,6 +720,7 @@ public class StatementParsing extends Parsing implements ITokenTypeRemapper { } marker.done(PyElementTypes.STATEMENT_LIST); + marker.setCustomEdgeTokenBinders(null, FollowingCommentBinder.INSTANCE); if (endMarker != null) { endMarker.done(elType); } diff --git a/python/testData/psi/CommentAtEndOfMethod.py b/python/testData/psi/CommentAtEndOfMethod.py new file mode 100644 index 000000000000..13c2bfa03bfb --- /dev/null +++ b/python/testData/psi/CommentAtEndOfMethod.py @@ -0,0 +1,3 @@ +def foo(): + a = 1 + #comment diff --git a/python/testData/psi/CommentAtEndOfMethod.txt b/python/testData/psi/CommentAtEndOfMethod.txt new file mode 100644 index 000000000000..235f5392c74d --- /dev/null +++ b/python/testData/psi/CommentAtEndOfMethod.txt @@ -0,0 +1,21 @@ +PyFile:CommentAtEndOfMethod.py + PyFunction('foo') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyAssignmentStatement + PyTargetExpression: a + PsiElement(Py:IDENTIFIER)('a') + PsiWhiteSpace(' ') + PsiElement(Py:EQ)('=') + PsiWhiteSpace(' ') + PyNumericLiteralExpression + PsiElement(Py:INTEGER_LITERAL)('1') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('#comment') \ No newline at end of file diff --git a/python/testData/psi/CommentBeforeMethod.txt b/python/testData/psi/CommentBeforeMethod.txt index 193078803424..81fd046f3ecd 100644 --- a/python/testData/psi/CommentBeforeMethod.txt +++ b/python/testData/psi/CommentBeforeMethod.txt @@ -10,8 +10,8 @@ PyFile:CommentBeforeMethod.py PyStatementList PyPassStatement PsiElement(Py:PASS_KEYWORD)('pass') - PsiWhiteSpace('\n ') - PsiComment(Py:END_OF_LINE_COMMENT)('# trailing comment\n\n') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# trailing comment\n\n') PsiWhiteSpace('\n\n') PsiComment(Py:END_OF_LINE_COMMENT)('#leading comment') PsiWhiteSpace('\n') diff --git a/python/testSrc/com/jetbrains/python/PyIndentTest.java b/python/testSrc/com/jetbrains/python/PyIndentTest.java index aae064bc44c0..31a195891490 100644 --- a/python/testSrc/com/jetbrains/python/PyIndentTest.java +++ b/python/testSrc/com/jetbrains/python/PyIndentTest.java @@ -1,7 +1,6 @@ package com.jetbrains.python; import com.intellij.openapi.actionSystem.IdeActions; -import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.command.CommandProcessor; import com.jetbrains.python.fixtures.PyLightFixtureTestCase; @@ -177,6 +176,16 @@ public class PyIndentTest extends PyLightFixtureTestCase { ); } + public void testIndentAfterTrailingComment() { // PY-2137 + doTest("def foo()\n" + + " a = 1\n" + + " #comment\n", + "def foo()\n" + + " a = 1\n" + + " #comment\n" + + " \n"); + } + /* TODO: formatter core problem? public void testAlignListBeforeEquals() throws Exception { diff --git a/python/testSrc/com/jetbrains/python/PythonLexerTest.java b/python/testSrc/com/jetbrains/python/PythonLexerTest.java index 5d68dbad1489..ef4fbe879251 100644 --- a/python/testSrc/com/jetbrains/python/PythonLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonLexerTest.java @@ -172,6 +172,15 @@ public class PythonLexerTest extends PyLexerTestCase { "Py:INDENT", "Py:PASS_KEYWORD"); } + public void testDedentAfterComment() { // PY-2137 + doTest("def foo():\n" + + " pass\n" + + " #comment\n", + "Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:RPAR", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", + "Py:INDENT", "Py:PASS_KEYWORD", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", + "Py:END_OF_LINE_COMMENT", "Py:DEDENT", "Py:LINE_BREAK"); + } + private static void doTest(String text, String... expectedTokens) { doLexerTest(text, new PythonIndentingLexer(), expectedTokens); } diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index 11af64942d5d..f0c2692c98c9 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -227,6 +227,10 @@ public class PythonParsingTest extends ParsingTestCase { doTest(); } + public void testCommentAtEndOfMethod() { // PY-2137 + doTest(); + } + public void doTest() { doTest(LanguageLevel.PYTHON25); }