From 98cc3d928c786ba988c8267b5f433cee5069ec5b Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Wed, 20 Sep 2017 15:18:31 +0300 Subject: [PATCH] PY-17017 Properly add DEDENT tokens after a series of trailing comments --- .../lexer/PythonIndentingProcessor.java | 93 ++++++++++++++----- python/testData/psi/CommentAtTheEndOfBlock.py | 4 + .../testData/psi/CommentAtTheEndOfBlock.txt | 25 +++++ python/testData/psi/CommentBetweenClasses.txt | 2 +- python/testData/psi/RaggedTrailingComments.py | 7 ++ .../testData/psi/RaggedTrailingComments.txt | 39 ++++++++ ...edTrailingCommentsWithTrailingStatement.py | 8 ++ ...dTrailingCommentsWithTrailingStatement.txt | 42 +++++++++ .../jetbrains/python/PythonParsingTest.java | 13 +++ 9 files changed, 208 insertions(+), 25 deletions(-) create mode 100644 python/testData/psi/CommentAtTheEndOfBlock.py create mode 100644 python/testData/psi/CommentAtTheEndOfBlock.txt create mode 100644 python/testData/psi/RaggedTrailingComments.py create mode 100644 python/testData/psi/RaggedTrailingComments.txt create mode 100644 python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.py create mode 100644 python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.txt diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java index 61ac5dc8984f..2255370f794c 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java @@ -35,6 +35,8 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { protected boolean myLineHasSignificantTokens; protected int myLastNewLineIndent = -1; private int myCurrentNewLineIndent = 0; + protected List myTokenQueue = new ArrayList<>(); + protected boolean myProcessSpecialTokensPending = false; private static final boolean DUMP_TOKENS = false; private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens(); @@ -89,10 +91,6 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { } } - protected List myTokenQueue = new ArrayList<>(); - - protected boolean myProcessSpecialTokensPending = false; - @Nullable protected IElementType getBaseTokenType() { return super.getTokenType(); @@ -251,12 +249,19 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { processLineBreak(tokenStart); while (isBaseAt(getCommentTokenType())) { // comment at start of line; maybe we need to generate dedent before the comments - myTokenQueue.add(new PendingCommentToken(getBaseTokenType(), getBaseTokenStart(), getBaseTokenEnd(), myLastNewLineIndent)); + final int commentEnd = getBaseTokenEnd(); + myTokenQueue.add(new PendingCommentToken(getBaseTokenType(), getBaseTokenStart(), commentEnd, myLastNewLineIndent)); advanceBase(); - if (!isBaseAt(PyTokenTypes.LINE_BREAK)) { + if (isBaseAt(PyTokenTypes.LINE_BREAK)) { + processLineBreak(getBaseTokenStart()); + } + // Treat EOF as an indent of size 0 + else if (getBaseTokenType() == null) { + closeDanglingSuitesWithComments(0, commentEnd); + } + else { break; } - processLineBreak(getBaseTokenStart()); } } else if (isBaseAt(PyTokenTypes.BACKSLASH)) { @@ -346,23 +351,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { myTokenQueue.add(insertIndex, new PendingToken(PyTokenTypes.INDENT, indentOffset, indentOffset)); } else if (indent < lastIndent) { - while (indent < lastIndent) { - myIndentStack.pop(); - lastIndent = myIndentStack.peek(); - int insertIndex = myTokenQueue.size(); - int dedentOffset = whiteSpaceStart; - if (indent > lastIndent) { - myTokenQueue.add(new PendingToken(PyTokenTypes.INCONSISTENT_DEDENT, whiteSpaceStart, whiteSpaceStart)); - insertIndex++; - } - else { - insertIndex = skipPrecedingCommentsWithIndent(indent, insertIndex); - } - if (insertIndex != myTokenQueue.size()) { - dedentOffset = myTokenQueue.get(insertIndex).getStart(); - } - myTokenQueue.add(insertIndex, new PendingToken(PyTokenTypes.DEDENT, dedentOffset, dedentOffset)); - } + closeDanglingSuitesWithComments(indent, whiteSpaceStart); myTokenQueue.add(new PendingToken(whitespaceTokenType, whiteSpaceStart, whiteSpaceEnd)); } else { @@ -370,6 +359,47 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { } } + private void closeDanglingSuitesWithComments(int indent, int whiteSpaceStart) { + int lastIndent = myIndentStack.peek(); + + int firstCommentAnchor = myTokenQueue.size(); + boolean foundComment = false; + for (int i = myTokenQueue.size() - 1; i >= 0; i--) { + final PendingToken token = myTokenQueue.get(i); + if (token.getType() == PyTokenTypes.LINE_BREAK) { + if (foundComment) { + firstCommentAnchor = i; + } + } + else if (token instanceof PendingCommentToken) { + foundComment = true; + firstCommentAnchor = i; + } + else { + break; + } + } + int insertIndex = firstCommentAnchor; + int lastSuiteIndent; + while (indent < lastIndent) { + lastSuiteIndent = myIndentStack.pop(); + lastIndent = myIndentStack.peek(); + int dedentOffset = whiteSpaceStart; + if (indent > lastIndent) { + myTokenQueue.add(new PendingToken(PyTokenTypes.INCONSISTENT_DEDENT, whiteSpaceStart, whiteSpaceStart)); + insertIndex = myTokenQueue.size(); + } + else { + insertIndex = skipPrecedingCommentsWithSameIndentOnSuiteClose(lastSuiteIndent, insertIndex); + } + if (insertIndex != myTokenQueue.size()) { + dedentOffset = myTokenQueue.get(insertIndex).getStart(); + } + myTokenQueue.add(insertIndex, new PendingToken(PyTokenTypes.DEDENT, dedentOffset, dedentOffset)); + insertIndex++; + } + } + protected int skipPrecedingCommentsWithIndent(int indent, int index) { // insert the DEDENT before previous comments that have the same indent as the current token indent boolean foundComment = false; @@ -389,6 +419,21 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { return foundComment ? index : myTokenQueue.size(); } + protected int skipPrecedingCommentsWithSameIndentOnSuiteClose(int indent, int anchorIndex) { + // insert the DEDENT before previous comments that have the same indent as the current token indent + int result = anchorIndex; + for (int i = anchorIndex; i < myTokenQueue.size(); i++) { + final PendingToken token = myTokenQueue.get(i); + if (token instanceof PendingCommentToken) { + if (((PendingCommentToken)token).getIndent() != indent) { + break; + } + result = i + 1; + } + } + return result; + } + protected int getNextLineIndent() { int indent = 0; while (getBaseTokenType() != null && PyTokenTypes.WHITESPACE_OR_LINEBREAK.contains(getBaseTokenType())) { diff --git a/python/testData/psi/CommentAtTheEndOfBlock.py b/python/testData/psi/CommentAtTheEndOfBlock.py new file mode 100644 index 000000000000..46d57aca7f2a --- /dev/null +++ b/python/testData/psi/CommentAtTheEndOfBlock.py @@ -0,0 +1,4 @@ +def f(): + def g(): + pass + # comment diff --git a/python/testData/psi/CommentAtTheEndOfBlock.txt b/python/testData/psi/CommentAtTheEndOfBlock.txt new file mode 100644 index 000000000000..f3e2c191ed63 --- /dev/null +++ b/python/testData/psi/CommentAtTheEndOfBlock.txt @@ -0,0 +1,25 @@ +PyFile:CommentAtTheEndOfBlock.py + PyFunction('f') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('f') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyFunction('g') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('g') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# comment') \ No newline at end of file diff --git a/python/testData/psi/CommentBetweenClasses.txt b/python/testData/psi/CommentBetweenClasses.txt index 6fe6e4944490..f1c606e6e29a 100644 --- a/python/testData/psi/CommentBetweenClasses.txt +++ b/python/testData/psi/CommentBetweenClasses.txt @@ -25,7 +25,7 @@ PyFile:CommentBetweenClasses.py PyStatementList PyPassStatement PsiElement(Py:PASS_KEYWORD)('pass') - PsiWhiteSpace('\n\n') + PsiWhiteSpace('\n\n') PsiComment(Py:END_OF_LINE_COMMENT)('# comment about T2') PsiWhiteSpace('\n\n') PyClass: T2 diff --git a/python/testData/psi/RaggedTrailingComments.py b/python/testData/psi/RaggedTrailingComments.py new file mode 100644 index 000000000000..95811cdd3c9a --- /dev/null +++ b/python/testData/psi/RaggedTrailingComments.py @@ -0,0 +1,7 @@ +def foo(): + def bar(): + def baz(): + pass + # baz + # bar + # foo diff --git a/python/testData/psi/RaggedTrailingComments.txt b/python/testData/psi/RaggedTrailingComments.txt new file mode 100644 index 000000000000..defef7b03789 --- /dev/null +++ b/python/testData/psi/RaggedTrailingComments.txt @@ -0,0 +1,39 @@ +PyFile:RaggedTrailingComments.py + PyFunction('foo') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyFunction('bar') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('bar') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyFunction('baz') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('baz') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# baz') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# bar') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# foo') \ No newline at end of file diff --git a/python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.py b/python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.py new file mode 100644 index 000000000000..019e39d66398 --- /dev/null +++ b/python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.py @@ -0,0 +1,8 @@ +def foo(): + def bar(): + def baz(): + pass + # baz + # bar + # foo +pass \ No newline at end of file diff --git a/python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.txt b/python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.txt new file mode 100644 index 000000000000..9e10290ff0de --- /dev/null +++ b/python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.txt @@ -0,0 +1,42 @@ +PyFile:RaggedTrailingCommentsWithTrailingStatement.py + PyFunction('foo') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('foo') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyFunction('bar') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('bar') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyFunction('baz') + PsiElement(Py:DEF_KEYWORD)('def') + PsiWhiteSpace(' ') + PsiElement(Py:IDENTIFIER)('baz') + PyParameterList + PsiElement(Py:LPAR)('(') + PsiElement(Py:RPAR)(')') + PsiElement(Py:COLON)(':') + PsiWhiteSpace('\n ') + PyStatementList + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# baz') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# bar') + PsiWhiteSpace('\n ') + PsiComment(Py:END_OF_LINE_COMMENT)('# foo') + PsiWhiteSpace('\n') + PyPassStatement + PsiElement(Py:PASS_KEYWORD)('pass') \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonParsingTest.java b/python/testSrc/com/jetbrains/python/PythonParsingTest.java index d66dcb77dbac..a056afcfa89c 100644 --- a/python/testSrc/com/jetbrains/python/PythonParsingTest.java +++ b/python/testSrc/com/jetbrains/python/PythonParsingTest.java @@ -545,6 +545,19 @@ public class PythonParsingTest extends ParsingTestCase { doTest(LanguageLevel.PYTHON35); } + // PY-17017 + public void testCommentAtTheEndOfBlock() { + doTest(); + } + + public void testRaggedTrailingComments() { + doTest(); + } + + public void testRaggedTrailingCommentsWithTrailingStatement() { + doTest(); + } + public void doTest(LanguageLevel languageLevel) { LanguageLevel prev = myLanguageLevel; myLanguageLevel = languageLevel;