PY-31442 As in lexers of other languages, reset the state in the skeleton

This way it's more self-contained, and we no longer require to use PythonLexer instead
everywhere. Thus I reverted previously made changes.

python/gen/com/jetbrains/python/lexer/_PythonLexer.java

@@ -870,6 +870,7 @@ return yylength()-s.length();
     zzAtEOF  = false;
     zzAtBOL = true;
     zzEndRead = end;
+    fStringHelper.reset();
     yybegin(initialState);
   }
 

python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt

@@ -153,20 +153,10 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) {
     }
   }
 
-  fun reset(offset: Int) {
-    if (offset == 0) {
-      myFStringStates.clear()
-      return
-    }
-    while (!myFStringStates.isEmpty() && offset < myFStringStates.peek().offset) {
-      myFStringStates.pop()
-    }
-    if (!myFStringStates.isEmpty()) {
-      val fragmentStates = myFStringStates.peek().fragmentStates
-      while (!fragmentStates.isEmpty() && offset < fragmentStates.peek().offset) {
-        fragmentStates.pop()
-      }
-    }
+  fun reset() {
+    // There is no need to be smarter about it, since LexerEditorHighlighter always resets 
+    // the lexer state to YYINITIAL where there can't be any f-strings.
+    myFStringStates.clear()
   }
 
   private data class FStringState(val oldState: Int, val offset: Int, val openingQuotes: String) {

python/src/com/jetbrains/python/lexer/Python.flex

@@ -86,7 +86,7 @@ FSTRING_FRAGMENT_TYPE_CONVERSION = "!" [^=:'\"} \t\r\n]*
 %state FSTRING_FRAGMENT
 %xstate FSTRING_FRAGMENT_FORMAT
 %{
-final PyLexerFStringHelper fStringHelper = new PyLexerFStringHelper(this);
+private final PyLexerFStringHelper fStringHelper = new PyLexerFStringHelper(this);
 
 private int getSpaceLength(CharSequence string) {
 String string1 = string.toString();

python/src/com/jetbrains/python/lexer/Python.skeleton

@@ -0,0 +1,249 @@
+
+  /** This character denotes the end of file */
+  public static final int YYEOF = -1;
+
+  /** initial size of the lookahead buffer */
+--- private static final int ZZ_BUFFERSIZE = ...;
+
+  /** lexical states */
+---  lexical states, charmap
+
+  /* error codes */
+  private static final int ZZ_UNKNOWN_ERROR = 0;
+  private static final int ZZ_NO_MATCH = 1;
+  private static final int ZZ_PUSHBACK_2BIG = 2;
+
+  /* error messages for the codes above */
+  private static final String[] ZZ_ERROR_MSG = {
+    "Unknown internal scanner error",
+    "Error: could not match input",
+    "Error: pushback value was too large"
+  };
+
+--- isFinal list
+  /** the input device */
+  private java.io.Reader zzReader;
+
+  /** the current state of the DFA */
+  private int zzState;
+
+  /** the current lexical state */
+  private int zzLexicalState = YYINITIAL;
+
+  /** this buffer contains the current text to be matched and is
+      the source of the yytext() string */
+  private CharSequence zzBuffer = "";
+
+  /** the textposition at the last accepting state */
+  private int zzMarkedPos;
+
+  /** the current text position in the buffer */
+  private int zzCurrentPos;
+
+  /** startRead marks the beginning of the yytext() string in the buffer */
+  private int zzStartRead;
+
+  /** endRead marks the last character in the buffer, that has been read
+      from input */
+  private int zzEndRead;
+
+  /**
+   * zzAtBOL == true <=> the scanner is currently at the beginning of a line
+   */
+  private boolean zzAtBOL = true;
+
+  /** zzAtEOF == true <=> the scanner is at the EOF */
+  private boolean zzAtEOF;
+
+  /** denotes if the user-EOF-code has already been executed */
+  private boolean zzEOFDone;
+
+--- user class code
+
+--- constructor declaration
+
+  public final int getTokenStart() {
+    return zzStartRead;
+  }
+
+  public final int getTokenEnd() {
+    return getTokenStart() + yylength();
+  }
+
+  public void reset(CharSequence buffer, int start, int end, int initialState) {
+    zzBuffer = buffer;
+    zzCurrentPos = zzMarkedPos = zzStartRead = start;
+    zzAtEOF  = false;
+    zzAtBOL = true;
+    zzEndRead = end;
+    fStringHelper.reset();
+    yybegin(initialState);
+  }
+
+  /**
+   * Refills the input buffer.
+   *
+   * @return      <code>false</code>, iff there was new input.
+   *
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+  private boolean zzRefill() throws java.io.IOException {
+    return true;
+  }
+
+
+  /**
+   * Returns the current lexical state.
+   */
+  public final int yystate() {
+    return zzLexicalState;
+  }
+
+
+  /**
+   * Enters a new lexical state
+   *
+   * @param newState the new lexical state
+   */
+  public final void yybegin(int newState) {
+    zzLexicalState = newState;
+  }
+
+
+  /**
+   * Returns the text matched by the current regular expression.
+   */
+  public final CharSequence yytext() {
+    return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
+  }
+
+
+  /**
+   * Returns the character at position <tt>pos</tt> from the
+   * matched text.
+   *
+   * It is equivalent to yytext().charAt(pos), but faster
+   *
+   * @param pos the position of the character to fetch.
+   *            A value from 0 to yylength()-1.
+   *
+   * @return the character at position pos
+   */
+  public final char yycharat(int pos) {
+    return zzBuffer.charAt(zzStartRead+pos);
+  }
+
+
+  /**
+   * Returns the length of the matched text region.
+   */
+  public final int yylength() {
+    return zzMarkedPos-zzStartRead;
+  }
+
+
+  /**
+   * Reports an error that occured while scanning.
+   *
+   * In a wellformed scanner (no or only correct usage of
+   * yypushback(int) and a match-all fallback rule) this method
+   * will only be called with things that "Can't Possibly Happen".
+   * If this method is called, something is seriously wrong
+   * (e.g. a JFlex bug producing a faulty scanner etc.).
+   *
+   * Usual syntax/scanner level error handling should be done
+   * in error fallback rules.
+   *
+   * @param   errorCode  the code of the errormessage to display
+   */
+--- zzScanError declaration
+    String message;
+    try {
+      message = ZZ_ERROR_MSG[errorCode];
+    }
+    catch (ArrayIndexOutOfBoundsException e) {
+      message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
+    }
+
+--- throws clause
+  }
+
+
+  /**
+   * Pushes the specified amount of characters back into the input stream.
+   *
+   * They will be read again by then next call of the scanning method
+   *
+   * @param number  the number of characters to be read again.
+   *                This number must not be greater than yylength()!
+   */
+--- yypushback decl (contains zzScanError exception)
+    if ( number > yylength() )
+      zzScanError(ZZ_PUSHBACK_2BIG);
+
+    zzMarkedPos -= number;
+  }
+
+
+--- zzDoEOF
+  /**
+   * Resumes scanning until the next regular expression is matched,
+   * the end of input is encountered or an I/O-Error occurs.
+   *
+   * @return      the next token
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+--- yylex declaration
+    int zzInput;
+    int zzAction;
+
+    // cached fields:
+    int zzCurrentPosL;
+    int zzMarkedPosL;
+    int zzEndReadL = zzEndRead;
+    CharSequence zzBufferL = zzBuffer;
+
+--- local declarations
+
+    while (true) {
+      zzMarkedPosL = zzMarkedPos;
+
+--- start admin (line, char, col count)
+      zzAction = -1;
+
+      zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+
+--- start admin (lexstate etc)
+
+      zzForAction: {
+        while (true) {
+
+--- next input, line, col, char count, next transition, isFinal action
+            zzAction = zzState;
+            zzMarkedPosL = zzCurrentPosL;
+--- line count update
+          }
+
+        }
+      }
+
+      // store back cached position
+      zzMarkedPos = zzMarkedPosL;
+--- char count update
+
+      if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+        zzAtEOF = true;
+--- eofvalue
+      }
+      else {
+--- actions
+          default:
+--- no match
+          }
+      }
+    }
+  }
+
+--- main
+
+}

python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java

@@ -4,12 +4,14 @@ package com.jetbrains.python.lexer;
 import com.intellij.psi.tree.TokenSet;
 import com.jetbrains.python.PyTokenTypes;
 
+import java.io.Reader;
+
 /**
  * @author yole
  */
 public class PythonIndentingLexer extends PythonIndentingProcessor {
   public PythonIndentingLexer() {
-    super(new PythonLexer(), TokenSet.EMPTY);
+    super(new _PythonLexer((Reader)null), TokenSet.EMPTY);
   }
 
   boolean addFinalBreak = true;

python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java

@@ -15,7 +15,8 @@
  */
 package com.jetbrains.python.lexer;
 
-import com.intellij.lexer.LexerBase;
+import com.intellij.lexer.FlexAdapter;
+import com.intellij.lexer.FlexLexer;
 import com.intellij.lexer.MergingLexerAdapter;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.TokenSet;
@@ -47,8 +48,8 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
   private static final boolean DUMP_TOKENS = false;
   private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens();
 
-  public PythonIndentingProcessor(LexerBase lexer, TokenSet tokens) {
-    super(lexer, tokens);
+  public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) {
+    super(new FlexAdapter(lexer), tokens);
   }
 
   protected static class PendingToken {

python/src/com/jetbrains/python/lexer/PythonLexer.java

@@ -2,24 +2,19 @@
 package com.jetbrains.python.lexer;
 
 import com.intellij.lexer.FlexAdapter;
-import org.jetbrains.annotations.NotNull;
+
+import java.io.Reader;
 
 /**
  * @author yole
  */
 public class PythonLexer extends FlexAdapter {
   public PythonLexer() {
-    super(new _PythonLexer(null));
+    super(new _PythonLexer((Reader)null));
   }
 
   @Override
   public _PythonLexer getFlex() {
     return (_PythonLexer)super.getFlex();
   }
-
-  @Override
-  public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
-    super.start(buffer, startOffset, endOffset, initialState);
-    getFlex().fStringHelper.reset(startOffset);
-  }
 }