lexer correctly handles sequences of multiple backslashes (PY-287)

python/src/com/jetbrains/python/lexer/PythonFutureAwareLexer.java

@@ -43,6 +43,8 @@ public class PythonFutureAwareLexer extends PythonLexer {
   // Limited size list of previously processed tokens used for 'from __future__ import ...' processing
   protected List<PendingToken> myTokenHistory = new ArrayList<PendingToken>();
 
+  protected boolean myProcessSpecialTokensPending = false;
+
   protected IElementType getBaseTokenType() {
     return super.getTokenType();
   }
@@ -90,6 +92,10 @@ public class PythonFutureAwareLexer extends PythonLexer {
     }
     if (myTokenQueue.size() > 0) {
       myTokenQueue.remove(0);
+      if (myProcessSpecialTokensPending) {
+        myProcessSpecialTokensPending = false;
+        processSpecialTokens();
+      }
     }
     else {
       advanceBase();

python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java

@@ -1,19 +1,3 @@
-/*
- *  Copyright 2005 Pythonid Project
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS"; BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
 package com.jetbrains.python.lexer;
 
 import com.intellij.psi.tree.IElementType;
@@ -139,6 +123,7 @@ public class PythonIndentingLexer extends PythonFutureAwareLexer {
       backslashToken.setType(PyTokenTypes.SPACE);
       processInsignificantLineBreak(getBaseTokenStart());
     }
+    myProcessSpecialTokensPending = true;
   }
 
   private void processLineBreak(int startPos) {

python/testSrc/com/jetbrains/python/PythonLexerTest.java

@@ -47,6 +47,11 @@ public class PythonLexerTest extends TestCase {
            "Py:IDENTIFIER", "Py:SPACE", "Py:EQ", "Py:SPACE", "Py:LINE_BREAK", "Py:IDENTIFIER");
   }
 
+  public void testMultipleBackslashes() throws Exception {
+    doTest("[\\\n\\\n]",
+           "Py:LBRACKET", "Py:SPACE", "Py:LINE_BREAK", "Py:SPACE", "Py:LINE_BREAK", "Py:RBRACKET");
+  }
+
   public void testIndent() throws Exception {
     doTest("if a:\n b",
            "Py:IF_KEYWORD", "Py:SPACE", "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK",