diff --git a/python/src/com/jetbrains/python/lexer/PythonFutureAwareLexer.java b/python/src/com/jetbrains/python/lexer/PythonFutureAwareLexer.java index 8a2dca55d265..6cca1e41e99d 100644 --- a/python/src/com/jetbrains/python/lexer/PythonFutureAwareLexer.java +++ b/python/src/com/jetbrains/python/lexer/PythonFutureAwareLexer.java @@ -43,6 +43,8 @@ public class PythonFutureAwareLexer extends PythonLexer { // Limited size list of previously processed tokens used for 'from __future__ import ...' processing protected List myTokenHistory = new ArrayList(); + protected boolean myProcessSpecialTokensPending = false; + protected IElementType getBaseTokenType() { return super.getTokenType(); } @@ -90,6 +92,10 @@ public class PythonFutureAwareLexer extends PythonLexer { } if (myTokenQueue.size() > 0) { myTokenQueue.remove(0); + if (myProcessSpecialTokensPending) { + myProcessSpecialTokensPending = false; + processSpecialTokens(); + } } else { advanceBase(); diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java index b58fc2bfa883..b2bfad28d9c3 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java @@ -1,19 +1,3 @@ -/* - * Copyright 2005 Pythonid Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS"; BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.jetbrains.python.lexer; import com.intellij.psi.tree.IElementType; @@ -139,6 +123,7 @@ public class PythonIndentingLexer extends PythonFutureAwareLexer { backslashToken.setType(PyTokenTypes.SPACE); processInsignificantLineBreak(getBaseTokenStart()); } + myProcessSpecialTokensPending = true; } private void processLineBreak(int startPos) { diff --git a/python/testSrc/com/jetbrains/python/PythonLexerTest.java b/python/testSrc/com/jetbrains/python/PythonLexerTest.java index 588714457070..73a67c6b29f1 100644 --- a/python/testSrc/com/jetbrains/python/PythonLexerTest.java +++ b/python/testSrc/com/jetbrains/python/PythonLexerTest.java @@ -47,6 +47,11 @@ public class PythonLexerTest extends TestCase { "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER"); } + public void testMultipleBackslashes() throws Exception { + doTest("[\\\n\\\n]", + "Py:LBRACKET", "Py:SPACE", "Py:LINE_BREAK", "Py:SPACE", "Py:LINE_BREAK", "Py:RBRACKET"); + } + public void testIndent() throws Exception { doTest("if a:\n b", "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",