MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-17017 Properly add DEDENT tokens after a series of trailing comments

Browse Source
This commit is contained in:
Mikhail Golubev
2017-10-02 15:20:24 +03:00
parent 34ed303b18
commit 98cc3d928c
9 changed files with 208 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java
+69 -24
View File
@@ -35,6 +35,8 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
protected boolean myLineHasSignificantTokens;
protected int myLastNewLineIndent = -1;
private int myCurrentNewLineIndent = 0;
protected List<PendingToken> myTokenQueue = new ArrayList<>();
protected boolean myProcessSpecialTokensPending = false;
private static final boolean DUMP_TOKENS = false;
private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens();
@@ -89,10 +91,6 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
}
}
protected List<PendingToken> myTokenQueue = new ArrayList<>();
protected boolean myProcessSpecialTokensPending = false;
@Nullable
protected IElementType getBaseTokenType() {
return super.getTokenType();
@@ -251,12 +249,19 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
processLineBreak(tokenStart);
while (isBaseAt(getCommentTokenType())) {
// comment at start of line; maybe we need to generate dedent before the comments
myTokenQueue.add(new PendingCommentToken(getBaseTokenType(), getBaseTokenStart(), getBaseTokenEnd(), myLastNewLineIndent));
final int commentEnd = getBaseTokenEnd();
myTokenQueue.add(new PendingCommentToken(getBaseTokenType(), getBaseTokenStart(), commentEnd, myLastNewLineIndent));
advanceBase();
if (!isBaseAt(PyTokenTypes.LINE_BREAK)) {
if (isBaseAt(PyTokenTypes.LINE_BREAK)) {
processLineBreak(getBaseTokenStart());
}
// Treat EOF as an indent of size 0
else if (getBaseTokenType() == null) {
closeDanglingSuitesWithComments(0, commentEnd);
}
else {
break;
}
processLineBreak(getBaseTokenStart());
}
}
else if (isBaseAt(PyTokenTypes.BACKSLASH)) {
@@ -346,23 +351,7 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
myTokenQueue.add(insertIndex, new PendingToken(PyTokenTypes.INDENT, indentOffset, indentOffset));
}
else if (indent < lastIndent) {
while (indent < lastIndent) {
myIndentStack.pop();
lastIndent = myIndentStack.peek();
int insertIndex = myTokenQueue.size();
int dedentOffset = whiteSpaceStart;
if (indent > lastIndent) {
myTokenQueue.add(new PendingToken(PyTokenTypes.INCONSISTENT_DEDENT, whiteSpaceStart, whiteSpaceStart));
insertIndex++;
}
else {
insertIndex = skipPrecedingCommentsWithIndent(indent, insertIndex);
}
if (insertIndex != myTokenQueue.size()) {
dedentOffset = myTokenQueue.get(insertIndex).getStart();
}
myTokenQueue.add(insertIndex, new PendingToken(PyTokenTypes.DEDENT, dedentOffset, dedentOffset));
}
closeDanglingSuitesWithComments(indent, whiteSpaceStart);
myTokenQueue.add(new PendingToken(whitespaceTokenType, whiteSpaceStart, whiteSpaceEnd));
}
else {
@@ -370,6 +359,47 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
}
}
private void closeDanglingSuitesWithComments(int indent, int whiteSpaceStart) {
int lastIndent = myIndentStack.peek();
int firstCommentAnchor = myTokenQueue.size();
boolean foundComment = false;
for (int i = myTokenQueue.size() - 1; i >= 0; i--) {
final PendingToken token = myTokenQueue.get(i);
if (token.getType() == PyTokenTypes.LINE_BREAK) {
if (foundComment) {
firstCommentAnchor = i;
}
}
else if (token instanceof PendingCommentToken) {
foundComment = true;
firstCommentAnchor = i;
}
else {
break;
}
}
int insertIndex = firstCommentAnchor;
int lastSuiteIndent;
while (indent < lastIndent) {
lastSuiteIndent = myIndentStack.pop();
lastIndent = myIndentStack.peek();
int dedentOffset = whiteSpaceStart;
if (indent > lastIndent) {
myTokenQueue.add(new PendingToken(PyTokenTypes.INCONSISTENT_DEDENT, whiteSpaceStart, whiteSpaceStart));
insertIndex = myTokenQueue.size();
}
else {
insertIndex = skipPrecedingCommentsWithSameIndentOnSuiteClose(lastSuiteIndent, insertIndex);
}
if (insertIndex != myTokenQueue.size()) {
dedentOffset = myTokenQueue.get(insertIndex).getStart();
}
myTokenQueue.add(insertIndex, new PendingToken(PyTokenTypes.DEDENT, dedentOffset, dedentOffset));
insertIndex++;
}
}
protected int skipPrecedingCommentsWithIndent(int indent, int index) {
// insert the DEDENT before previous comments that have the same indent as the current token indent
boolean foundComment = false;
@@ -389,6 +419,21 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
return foundComment ? index : myTokenQueue.size();
}
protected int skipPrecedingCommentsWithSameIndentOnSuiteClose(int indent, int anchorIndex) {
// insert the DEDENT before previous comments that have the same indent as the current token indent
int result = anchorIndex;
for (int i = anchorIndex; i < myTokenQueue.size(); i++) {
final PendingToken token = myTokenQueue.get(i);
if (token instanceof PendingCommentToken) {
if (((PendingCommentToken)token).getIndent() != indent) {
break;
}
result = i + 1;
}
}
return result;
}
protected int getNextLineIndent() {
int indent = 0;
while (getBaseTokenType() != null && PyTokenTypes.WHITESPACE_OR_LINEBREAK.contains(getBaseTokenType())) {
python/testData/psi/CommentAtTheEndOfBlock.py
+4
View File
@@ -0,0 +1,4 @@
def f():
def g():
pass
# comment
python/testData/psi/CommentAtTheEndOfBlock.txt
+25
View File
@@ -0,0 +1,25 @@
PyFile:CommentAtTheEndOfBlock.py
PyFunction('f')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('f')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyFunction('g')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('g')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyPassStatement
PsiElement(Py:PASS_KEYWORD)('pass')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# comment')
python/testData/psi/CommentBetweenClasses.txt
+1 -1
View File
@@ -25,7 +25,7 @@ PyFile:CommentBetweenClasses.py
PyStatementList
PyPassStatement
PsiElement(Py:PASS_KEYWORD)('pass')
PsiWhiteSpace('\n\n')
PsiWhiteSpace('\n\n')
PsiComment(Py:END_OF_LINE_COMMENT)('# comment about T2')
PsiWhiteSpace('\n\n')
PyClass: T2
python/testData/psi/RaggedTrailingComments.py
+7
View File
@@ -0,0 +1,7 @@
def foo():
def bar():
def baz():
pass
# baz
# bar
# foo
python/testData/psi/RaggedTrailingComments.txt
+39
View File
@@ -0,0 +1,39 @@
PyFile:RaggedTrailingComments.py
PyFunction('foo')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('foo')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyFunction('bar')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('bar')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyFunction('baz')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('baz')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyPassStatement
PsiElement(Py:PASS_KEYWORD)('pass')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# baz')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# bar')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# foo')
python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.py
+8
View File
@@ -0,0 +1,8 @@
def foo():
def bar():
def baz():
pass
# baz
# bar
# foo
pass
python/testData/psi/RaggedTrailingCommentsWithTrailingStatement.txt
+42
View File
@@ -0,0 +1,42 @@
PyFile:RaggedTrailingCommentsWithTrailingStatement.py
PyFunction('foo')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('foo')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyFunction('bar')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('bar')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyFunction('baz')
PsiElement(Py:DEF_KEYWORD)('def')
PsiWhiteSpace(' ')
PsiElement(Py:IDENTIFIER)('baz')
PyParameterList
PsiElement(Py:LPAR)('(')
PsiElement(Py:RPAR)(')')
PsiElement(Py:COLON)(':')
PsiWhiteSpace('\n ')
PyStatementList
PyPassStatement
PsiElement(Py:PASS_KEYWORD)('pass')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# baz')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# bar')
PsiWhiteSpace('\n ')
PsiComment(Py:END_OF_LINE_COMMENT)('# foo')
PsiWhiteSpace('\n')
PyPassStatement
PsiElement(Py:PASS_KEYWORD)('pass')
python/testSrc/com/jetbrains/python/PythonParsingTest.java
+13
View File
@@ -545,6 +545,19 @@ public class PythonParsingTest extends ParsingTestCase {
doTest(LanguageLevel.PYTHON35);
}
// PY-17017
public void testCommentAtTheEndOfBlock() {
doTest();
}
public void testRaggedTrailingComments() {
doTest();
}
public void testRaggedTrailingCommentsWithTrailingStatement() {
doTest();
}
public void doTest(LanguageLevel languageLevel) {
LanguageLevel prev = myLanguageLevel;
myLanguageLevel = languageLevel;