diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java
index c31f4ee5289a..4e459043a1c4 100644
--- a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java
+++ b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java
@@ -1,6 +1,7 @@
package com.jetbrains.python.lexer;
import com.intellij.psi.tree.TokenSet;
+import com.jetbrains.python.PyTokenTypes;
import java.io.Reader;
@@ -11,4 +12,14 @@ public class PythonIndentingLexer extends PythonIndentingProcessor {
public PythonIndentingLexer() {
super(new _PythonLexer((Reader)null), TokenSet.EMPTY);
}
+
+ boolean addFinalBreak = true;
+ protected void processSpecialTokens() {
+ super.processSpecialTokens();
+ int tokenStart = getBaseTokenStart();
+ if (getBaseTokenType() == null && addFinalBreak) {
+ pushToken(PyTokenTypes.STATEMENT_BREAK, tokenStart, tokenStart);
+ addFinalBreak = false;
+ }
+ }
}
diff --git a/python/testData/inspections/AugmentAssignment1.py b/python/testData/inspections/AugmentAssignment1.py
index 66c09439a93e..88e406213e5c 100644
--- a/python/testData/inspections/AugmentAssignment1.py
+++ b/python/testData/inspections/AugmentAssignment1.py
@@ -1 +1 @@
-value = value / 10**exp #comment
\ No newline at end of file
+value = value / 10**exp #comment
\ No newline at end of file
diff --git a/python/testData/inspections/AugmentAssignment1_after.py b/python/testData/inspections/AugmentAssignment1_after.py
index 5646e7f040ec..3104d20d1c46 100644
--- a/python/testData/inspections/AugmentAssignment1_after.py
+++ b/python/testData/inspections/AugmentAssignment1_after.py
@@ -1 +1 @@
-value /= 10 ** exp #comment
\ No newline at end of file
+value /= 10 ** exp#comment
\ No newline at end of file
diff --git a/python/testData/psi/ColonBeforeEof.txt b/python/testData/psi/ColonBeforeEof.txt
index 246f155b21dd..eefdea5c8a19 100644
--- a/python/testData/psi/ColonBeforeEof.txt
+++ b/python/testData/psi/ColonBeforeEof.txt
@@ -12,5 +12,5 @@ PyFile:ColonBeforeEof.py
PsiElement(Py:IDENTIFIER)('b')
PsiElement(Py:COLON)(':')
PyStatementList
- PsiErrorElement:statement expected
+ PsiErrorElement:indent expected
\ No newline at end of file
diff --git a/python/testSrc/com/jetbrains/python/PythonLexerTest.java b/python/testSrc/com/jetbrains/python/PythonLexerTest.java
index 98424cdc3a53..80113d62bb5a 100644
--- a/python/testSrc/com/jetbrains/python/PythonLexerTest.java
+++ b/python/testSrc/com/jetbrains/python/PythonLexerTest.java
@@ -8,135 +8,135 @@ import com.jetbrains.python.lexer.PythonIndentingLexer;
*/
public class PythonLexerTest extends PyLexerTestCase {
public void testSimpleExpression() {
- doTest("a=1", "Py:IDENTIFIER", "Py:EQ", "Py:INTEGER_LITERAL");
+ doTest("a=1", "Py:IDENTIFIER", "Py:EQ", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK");
}
public void testMergeSpaces() {
- doTest("a = 1", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL");
+ doTest("a = 1", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK");
}
public void testLineBreakInBraces() {
- doTest("[a,\n b]", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET");
+ doTest("[a,\n b]", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK");
}
public void testLineBreakInBraces2() {
doTest("x=[a,\n b]", "Py:IDENTIFIER", "Py:EQ", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK",
- "Py:IDENTIFIER", "Py:RBRACKET");
+ "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK");
}
public void testLineBreakInBracesAfterComment() {
doTest("x=[a, #c\n b]",
"Py:IDENTIFIER", "Py:EQ", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:SPACE", "Py:END_OF_LINE_COMMENT",
- "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET");
+ "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK");
}
public void testBraceAfterIndent() {
doTest("x=\n [a,\n b]",
"Py:IDENTIFIER", "Py:EQ", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
- "Py:INDENT", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET");
+ "Py:INDENT", "Py:LBRACKET", "Py:IDENTIFIER", "Py:COMMA", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:RBRACKET", "Py:STATEMENT_BREAK");
}
public void testBackslash() {
doTest("a=\\\nb",
- "Py:IDENTIFIER", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER");
+ "Py:IDENTIFIER", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testBackslashAfterSpace() {
doTest("a = \\\n b",
- "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER");
+ "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testMultipleBackslashes() {
doTest("[\\\n\\\n]",
- "Py:LBRACKET", "Py:SPACE", "Py:LINE_BREAK", "Py:SPACE", "Py:LINE_BREAK", "Py:RBRACKET");
+ "Py:LBRACKET", "Py:SPACE", "Py:LINE_BREAK", "Py:SPACE", "Py:LINE_BREAK", "Py:RBRACKET", "Py:STATEMENT_BREAK");
}
public void testIndent() {
doTest("if a:\n b",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
- "Py:INDENT", "Py:IDENTIFIER");
+ "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testMultiLineIndent() {
doTest("if a:\n b\n c",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
- "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER");
+ "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testDedent() {
doTest("if a:\n b\nc",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
"Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:DEDENT", "Py:LINE_BREAK",
- "Py:IDENTIFIER");
+ "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testMultiDedent() {
doTest("if a:\n b\n c\nd",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT",
"Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:DEDENT",
- "Py:DEDENT", "Py:LINE_BREAK", "Py:IDENTIFIER");
+ "Py:DEDENT", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testEmptyLine() {
doTest("if a:\n b\n \n c",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT",
- "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER");
+ "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testEndOfLineSpace() {
doTest("if a:\n b\n c \n \n d",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
"Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK",
- "Py:LINE_BREAK", "Py:IDENTIFIER");
+ "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testComment() {
doTest("if a:\n b\n #comment\nc",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
"Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:DEDENT", "Py:LINE_BREAK",
- "Py:IDENTIFIER");
+ "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testIndentedComment() {
doTest("#a\n #b\n#c",
- "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT");
+ "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:END_OF_LINE_COMMENT", "Py:STATEMENT_BREAK");
}
public void testIndentedCommentAndCode() {
doTest("if a:\n #b\n c",
"Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
- "Py:INDENT", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:IDENTIFIER");
+ "Py:INDENT", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testWithNotKeyword() {
- doTest("with=as", "Py:IDENTIFIER", "Py:EQ", "Py:IDENTIFIER");
+ doTest("with=as", "Py:IDENTIFIER", "Py:EQ", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testBytesLiteral() {
- doTest("a=b'ABC'", "Py:IDENTIFIER", "Py:EQ", "Py:SINGLE_QUOTED_STRING");
+ doTest("a=b'ABC'", "Py:IDENTIFIER", "Py:EQ", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testOctalLiteral() {
- doTest("0o123", "Py:INTEGER_LITERAL");
+ doTest("0o123", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK");
}
public void testBinaryLiteral() {
- doTest("0b0101", "Py:INTEGER_LITERAL");
+ doTest("0b0101", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK");
}
private static final String THREE_QUOTES = "\"\"\"";
public void testLongString() {
doTest(THREE_QUOTES + THREE_QUOTES + "\nb=" + THREE_QUOTES + THREE_QUOTES,
- "Py:DOCSTRING", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING");
+ "Py:DOCSTRING", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testLongStringEscaped() {
- doTest("''' \\'''foo\\''' ''';", "Py:DOCSTRING", "Py:SEMICOLON");
+ doTest("''' \\'''foo\\''' ''';", "Py:DOCSTRING", "Py:SEMICOLON", "Py:STATEMENT_BREAK");
}
public void testLongStringWithLineBreak() {
- doTest(THREE_QUOTES + "\\\na\n\n" + THREE_QUOTES + ";", "Py:DOCSTRING", "Py:SEMICOLON");
+ doTest(THREE_QUOTES + "\\\na\n\n" + THREE_QUOTES + ";", "Py:DOCSTRING", "Py:SEMICOLON", "Py:STATEMENT_BREAK");
}
@@ -151,27 +151,27 @@ public class PythonLexerTest extends PyLexerTestCase {
public void testBackslashBeforeEmptyLine() {
doTest("x=\"a\" + \\\n \"b\" \\\n\nprint x", "Py:IDENTIFIER", "Py:EQ", "Py:SINGLE_QUOTED_STRING", "Py:SPACE", "Py:PLUS", "Py:SPACE", "Py:LINE_BREAK",
"Py:SINGLE_QUOTED_STRING", "Py:SPACE", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
- "Py:IDENTIFIER" , "Py:SPACE", "Py:IDENTIFIER");
+ "Py:IDENTIFIER" , "Py:SPACE", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
public void testIncompleteTripleQuotedString() { // PY-1768
- doTest("tmp = '''abc\nd", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING");
+ doTest("tmp = '''abc\nd", "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testEscapedClosingTripleApos() { // PY-1777
- doTest("a=''' foo '\\''' bar '''", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING");
+ doTest("a=''' foo '\\''' bar '''", "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testEscapedClosingTripleQuote() { // PY-1777
- doTest("a="+THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING");
+ doTest("a="+THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES, "Py:IDENTIFIER", "Py:EQ", "Py:TRIPLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testEOFDocstring() { // PY-4169
- doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES,"Py:DOCSTRING");
+ doTest(THREE_QUOTES + " foo \"\\" + THREE_QUOTES + " bar " + THREE_QUOTES,"Py:DOCSTRING", "Py:STATEMENT_BREAK");
}
public void testOddNumberOfQuotes() { // PY-2802
- doTest("'''foo''''", "Py:TRIPLE_QUOTED_STRING", "Py:SINGLE_QUOTED_STRING");
+ doTest("'''foo''''", "Py:TRIPLE_QUOTED_STRING", "Py:SINGLE_QUOTED_STRING", "Py:STATEMENT_BREAK");
}
public void testDedentBeforeComment() { // PY-2209 & friends
@@ -185,7 +185,7 @@ public class PythonLexerTest extends PyLexerTestCase {
"Py:INDENT", "Py:PASS_KEYWORD", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
"Py:DEDENT", "Py:END_OF_LINE_COMMENT", "Py:LINE_BREAK",
"Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:IDENTIFIER", "Py:RPAR", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
- "Py:INDENT", "Py:PASS_KEYWORD");
+ "Py:INDENT", "Py:PASS_KEYWORD", "Py:STATEMENT_BREAK");
}
public void testDedentAfterComment() { // PY-2137
@@ -194,11 +194,11 @@ public class PythonLexerTest extends PyLexerTestCase {
" #comment\n",
"Py:DEF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:LPAR", "Py:RPAR", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
"Py:INDENT", "Py:PASS_KEYWORD", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",
- "Py:END_OF_LINE_COMMENT", "Py:DEDENT", "Py:LINE_BREAK");
+ "Py:END_OF_LINE_COMMENT", "Py:DEDENT", "Py:LINE_BREAK", "Py:STATEMENT_BREAK");
}
public void testIndentAtStartOfFile() { // PY-4941
- doTest(" a", "Py:SPACE", "Py:INDENT", "Py:IDENTIFIER");
+ doTest(" a", "Py:SPACE", "Py:INDENT", "Py:IDENTIFIER", "Py:STATEMENT_BREAK");
}
private static void doTest(String text, String... expectedTokens) {