diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java index c31f4ee5289a..4e459043a1c4 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java @@ -1,6 +1,7 @@ package com.jetbrains.python.lexer; import com.intellij.psi.tree.TokenSet; +import com.jetbrains.python.PyTokenTypes; import java.io.Reader; @@ -11,4 +12,14 @@ public class PythonIndentingLexer extends PythonIndentingProcessor { public PythonIndentingLexer() { super(new _PythonLexer((Reader)null), TokenSet.EMPTY); } + + boolean addFinalBreak = true; + protected void processSpecialTokens() { + super.processSpecialTokens(); + int tokenStart = getBaseTokenStart(); + if (getBaseTokenType() == null && addFinalBreak) { + pushToken(PyTokenTypes.STATEMENT_BREAK, tokenStart, tokenStart); + addFinalBreak = false; + } + } } diff --git a/python/testData/inspections/AugmentAssignment1.py b/python/testData/inspections/AugmentAssignment1.py index 66c09439a93e..88e406213e5c 100644 --- a/python/testData/inspections/AugmentAssignment1.py +++ b/python/testData/inspections/AugmentAssignment1.py @@ -1 +1 @@ -value = value / 10**exp #comment \ No newline at end of file +value = value / 10**exp #comment \ No newline at end of file diff --git a/python/testData/inspections/AugmentAssignment1_after.py b/python/testData/inspections/AugmentAssignment1_after.py index 5646e7f040ec..3104d20d1c46 100644 --- a/python/testData/inspections/AugmentAssignment1_after.py +++ b/python/testData/inspections/AugmentAssignment1_after.py @@ -1 +1 @@ -value /= 10 ** exp #comment \ No newline at end of file +value /= 10 ** exp#comment \ No newline at end of file diff --git a/python/testData/psi/ColonBeforeEof.txt b/python/testData/psi/ColonBeforeEof.txt index 246f155b21dd..eefdea5c8a19 100644 --- a/python/testData/psi/ColonBeforeEof.txt +++ b/python/testData/psi/ColonBeforeEof.txt @@ -12,5 +12,5 @@ PyFile:ColonBeforeEof.py PsiElement(Py:IDENTIFIER)('b') PsiElement(Py:COLON)(':') PyStatementList - PsiErrorElement:statement expected + PsiErrorElement:indent expected \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/PythonLexerTest.java b/python/testSrc/com/jetbrains/python/PythonLexerTest.java index 98424cdc3a53..80113d62bb5a 100644 --- a/python/testSrc/com/jetbrains/python/PythonLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonLexerTest.java @@ -8,135 +8,135 @@ import com.jetbrains.python.lexer.PythonIndentingLexer; */ public class PythonLexerTest extends PyLexerTestCase { public void testSimpleExpression() { - doTest("a=1", "Py:IDENTIFIER", "Py:EQ", "Py:INTEGER_LITERAL"); + doTest("a=1", "Py:IDENTIFIER", "Py:EQ", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK"); } public void testMergeSpaces() { - doTest("a = 1", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL"); + doTest("a = 1", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK"); } public void testLineBreakInBraces() { - doTest("[a,\n b]", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET"); + doTest("[a,\n b]", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK"); } public void testLineBreakInBraces2() { doTest("x=[a,\n b]", "Py:IDENTIFIER", "Py:EQ", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", - "Py:IDENTIFIER", "Py:RBRACKET"); + "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK"); } public void testLineBreakInBracesAfterComment() { doTest("x=[a, #c\n b]", "Py:IDENTIFIER", "Py:EQ", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:SPACE", "Py:END_OF_LINE_COMMENT", - "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET"); + "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK"); } public void testBraceAfterIndent() { doTest("x=\n [a,\n b]", "Py:IDENTIFIER", "Py:EQ", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", - "Py:INDENT", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET"); + "Py:INDENT", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK"); } public void testBackslash() { doTest("a=\\\nb", - "Py:IDENTIFIER", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER"); + "Py:IDENTIFIER", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testBackslashAfterSpace() { doTest("a = \\\n b", - "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER"); + "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testMultipleBackslashes() { doTest("[\\\n\\\n]", - "Py:LBRACKET", "Py:SPACE", "Py:LINE_BREAK", "Py:SPACE", "Py:LINE_BREAK", "Py:RBRACKET"); + "Py:LBRACKET", "Py:SPACE", "Py:LINE_BREAK", "Py:SPACE", "Py:LINE_BREAK", "Py:RBRACKET", "Py:STATEMENT_BREAK"); } public void testIndent() { doTest("if a:\n b", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", - "Py:INDENT", "Py:IDENTIFIER"); + "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testMultiLineIndent() { doTest("if a:\n b\n c", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", - "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER"); + "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testDedent() { doTest("if a:\n b\nc", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:DEDENT", "Py:LINE_BREAK", - "Py:IDENTIFIER"); + "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testMultiDedent() { doTest("if a:\n b\n c\nd", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:DEDENT", - "Py:DEDENT", "Py:LINE_BREAK", "Py:IDENTIFIER"); + "Py:DEDENT", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testEmptyLine() { doTest("if a:\n b\n \n c", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", - "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER"); + "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testEndOfLineSpace() { doTest("if a:\n b\n c \n \n d", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", - "Py:LINE_BREAK", "Py:IDENTIFIER"); + "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testComment() { doTest("if a:\n b\n #comment\nc", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:DEDENT", "Py:LINE_BREAK", - "Py:IDENTIFIER"); + "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testIndentedComment() { doTest("#a\n #b\n#c", - "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT"); + "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:STATEMENT_BREAK"); } public void testIndentedCommentAndCode() { doTest("if a:\n #b\n c", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", - "Py:INDENT", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:IDENTIFIER"); + "Py:INDENT", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testWithNotKeyword() { - doTest("with=as", "Py:IDENTIFIER", "Py:EQ", "Py:IDENTIFIER"); + doTest("with=as", "Py:IDENTIFIER", "Py:EQ", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testBytesLiteral() { - doTest("a=b'ABC'", "Py:IDENTIFIER", "Py:EQ", "Py:SINGLE_QUOTED_STRING"); + doTest("a=b'ABC'", "Py:IDENTIFIER", "Py:EQ", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK"); } public void testOctalLiteral() { - doTest("0o123", "Py:INTEGER_LITERAL"); + doTest("0o123", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK"); } public void testBinaryLiteral() { - doTest("0b0101", "Py:INTEGER_LITERAL"); + doTest("0b0101", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK"); } private static final String THREE_QUOTES = "\"\"\""; public void testLongString() { doTest(THREE_QUOTES + THREE_QUOTES + "\nb=" + THREE_QUOTES + THREE_QUOTES, - "Py:DOCSTRING", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING"); + "Py:DOCSTRING", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK"); } public void testLongStringEscaped() { - doTest("''' \\'''foo\\''' ''';", "Py:DOCSTRING", "Py:SEMICOLON"); + doTest("''' \\'''foo\\''' ''';", "Py:DOCSTRING", "Py:SEMICOLON", "Py:STATEMENT_BREAK"); } public void testLongStringWithLineBreak() { - doTest(THREE_QUOTES + "\\\na\n\n" + THREE_QUOTES + ";", "Py:DOCSTRING", "Py:SEMICOLON"); + doTest(THREE_QUOTES + "\\\na\n\n" + THREE_QUOTES + ";", "Py:DOCSTRING", "Py:SEMICOLON", "Py:STATEMENT_BREAK"); } @@ -151,27 +151,27 @@ public class PythonLexerTest extends PyLexerTestCase { public void testBackslashBeforeEmptyLine() { doTest("x=\"a\" + \\\n \"b\" \\\n\nprint x", "Py:IDENTIFIER", "Py:EQ", "Py:SINGLE_QUOTED_STRING", "Py:SPACE", "Py:PLUS", "Py:SPACE", "Py:LINE_BREAK", "Py:SINGLE_QUOTED_STRING", "Py:SPACE", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", - "Py:IDENTIFIER" , "Py:SPACE", "Py:IDENTIFIER"); + "Py:IDENTIFIER" , "Py:SPACE", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } public void testIncompleteTripleQuotedString() { // PY-1768 - doTest("tmp = '''abc\nd", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING"); + doTest("tmp = '''abc\nd", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK"); } public void testEscapedClosingTripleApos() { // PY-1777 - doTest("a=''' foo '\\''' bar '''", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING"); + doTest("a=''' foo '\\''' bar '''", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK"); } public void testEscapedClosingTripleQuote() { // PY-1777 - doTest("a="+THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING"); + doTest("a="+THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK"); } public void testEOFDocstring() { // PY-4169 - doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES,"Py:DOCSTRING"); + doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES,"Py:DOCSTRING", "Py:STATEMENT_BREAK"); } public void testOddNumberOfQuotes() { // PY-2802 - doTest("'''foo''''", "Py:TRIPLE_QUOTED_STRING", "Py:SINGLE_QUOTED_STRING"); + doTest("'''foo''''", "Py:TRIPLE_QUOTED_STRING", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK"); } public void testDedentBeforeComment() { // PY-2209 & friends @@ -185,7 +185,7 @@ public class PythonLexerTest extends PyLexerTestCase { "Py:INDENT", "Py:PASS_KEYWORD", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:DEDENT", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", - "Py:INDENT", "Py:PASS_KEYWORD"); + "Py:INDENT", "Py:PASS_KEYWORD", "Py:STATEMENT_BREAK"); } public void testDedentAfterComment() { // PY-2137 @@ -194,11 +194,11 @@ public class PythonLexerTest extends PyLexerTestCase { " #comment\n", "Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:RPAR", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:PASS_KEYWORD", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", - "Py:END_OF_LINE_COMMENT", "Py:DEDENT", "Py:LINE_BREAK"); + "Py:END_OF_LINE_COMMENT", "Py:DEDENT", "Py:LINE_BREAK", "Py:STATEMENT_BREAK"); } public void testIndentAtStartOfFile() { // PY-4941 - doTest(" a", "Py:SPACE", "Py:INDENT", "Py:IDENTIFIER"); + doTest(" a", "Py:SPACE", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK"); } private static void doTest(String text, String... expectedTokens) {