diff --git a/python/gen/com/jetbrains/python/lexer/_PythonLexer.java b/python/gen/com/jetbrains/python/lexer/_PythonLexer.java
index 1becb6921660..8ca1155f85d8 100644
--- a/python/gen/com/jetbrains/python/lexer/_PythonLexer.java
+++ b/python/gen/com/jetbrains/python/lexer/_PythonLexer.java
@@ -870,6 +870,7 @@ return yylength()-s.length();
zzAtEOF = false;
zzAtBOL = true;
zzEndRead = end;
+ fStringHelper.reset();
yybegin(initialState);
}
diff --git a/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt b/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt
index 89d67b3534cc..103c1ce140f5 100644
--- a/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt
+++ b/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt
@@ -153,20 +153,10 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) {
}
}
- fun reset(offset: Int) {
- if (offset == 0) {
- myFStringStates.clear()
- return
- }
- while (!myFStringStates.isEmpty() && offset < myFStringStates.peek().offset) {
- myFStringStates.pop()
- }
- if (!myFStringStates.isEmpty()) {
- val fragmentStates = myFStringStates.peek().fragmentStates
- while (!fragmentStates.isEmpty() && offset < fragmentStates.peek().offset) {
- fragmentStates.pop()
- }
- }
+ fun reset() {
+ // There is no need to be smarter about it, since LexerEditorHighlighter always resets
+ // the lexer state to YYINITIAL where there can't be any f-strings.
+ myFStringStates.clear()
}
private data class FStringState(val oldState: Int, val offset: Int, val openingQuotes: String) {
diff --git a/python/src/com/jetbrains/python/lexer/Python.flex b/python/src/com/jetbrains/python/lexer/Python.flex
index 4319483b1315..ded1f5223068 100644
--- a/python/src/com/jetbrains/python/lexer/Python.flex
+++ b/python/src/com/jetbrains/python/lexer/Python.flex
@@ -86,7 +86,7 @@ FSTRING_FRAGMENT_TYPE_CONVERSION = "!" [^=:'\"} \t\r\n]*
%state FSTRING_FRAGMENT
%xstate FSTRING_FRAGMENT_FORMAT
%{
-final PyLexerFStringHelper fStringHelper = new PyLexerFStringHelper(this);
+private final PyLexerFStringHelper fStringHelper = new PyLexerFStringHelper(this);
private int getSpaceLength(CharSequence string) {
String string1 = string.toString();
diff --git a/python/src/com/jetbrains/python/lexer/Python.skeleton b/python/src/com/jetbrains/python/lexer/Python.skeleton
new file mode 100644
index 000000000000..946f183858c4
--- /dev/null
+++ b/python/src/com/jetbrains/python/lexer/Python.skeleton
@@ -0,0 +1,249 @@
+
+ /** This character denotes the end of file */
+ public static final int YYEOF = -1;
+
+ /** initial size of the lookahead buffer */
+--- private static final int ZZ_BUFFERSIZE = ...;
+
+ /** lexical states */
+--- lexical states, charmap
+
+ /* error codes */
+ private static final int ZZ_UNKNOWN_ERROR = 0;
+ private static final int ZZ_NO_MATCH = 1;
+ private static final int ZZ_PUSHBACK_2BIG = 2;
+
+ /* error messages for the codes above */
+ private static final String[] ZZ_ERROR_MSG = {
+ "Unknown internal scanner error",
+ "Error: could not match input",
+ "Error: pushback value was too large"
+ };
+
+--- isFinal list
+ /** the input device */
+ private java.io.Reader zzReader;
+
+ /** the current state of the DFA */
+ private int zzState;
+
+ /** the current lexical state */
+ private int zzLexicalState = YYINITIAL;
+
+ /** this buffer contains the current text to be matched and is
+ the source of the yytext() string */
+ private CharSequence zzBuffer = "";
+
+ /** the textposition at the last accepting state */
+ private int zzMarkedPos;
+
+ /** the current text position in the buffer */
+ private int zzCurrentPos;
+
+ /** startRead marks the beginning of the yytext() string in the buffer */
+ private int zzStartRead;
+
+ /** endRead marks the last character in the buffer, that has been read
+ from input */
+ private int zzEndRead;
+
+ /**
+ * zzAtBOL == true <=> the scanner is currently at the beginning of a line
+ */
+ private boolean zzAtBOL = true;
+
+ /** zzAtEOF == true <=> the scanner is at the EOF */
+ private boolean zzAtEOF;
+
+ /** denotes if the user-EOF-code has already been executed */
+ private boolean zzEOFDone;
+
+--- user class code
+
+--- constructor declaration
+
+ public final int getTokenStart() {
+ return zzStartRead;
+ }
+
+ public final int getTokenEnd() {
+ return getTokenStart() + yylength();
+ }
+
+ public void reset(CharSequence buffer, int start, int end, int initialState) {
+ zzBuffer = buffer;
+ zzCurrentPos = zzMarkedPos = zzStartRead = start;
+ zzAtEOF = false;
+ zzAtBOL = true;
+ zzEndRead = end;
+ fStringHelper.reset();
+ yybegin(initialState);
+ }
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return false, iff there was new input.
+ *
+ * @exception java.io.IOException if any I/O-Error occurs
+ */
+ private boolean zzRefill() throws java.io.IOException {
+ return true;
+ }
+
+
+ /**
+ * Returns the current lexical state.
+ */
+ public final int yystate() {
+ return zzLexicalState;
+ }
+
+
+ /**
+ * Enters a new lexical state
+ *
+ * @param newState the new lexical state
+ */
+ public final void yybegin(int newState) {
+ zzLexicalState = newState;
+ }
+
+
+ /**
+ * Returns the text matched by the current regular expression.
+ */
+ public final CharSequence yytext() {
+ return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
+ }
+
+
+ /**
+ * Returns the character at position pos from the
+ * matched text.
+ *
+ * It is equivalent to yytext().charAt(pos), but faster
+ *
+ * @param pos the position of the character to fetch.
+ * A value from 0 to yylength()-1.
+ *
+ * @return the character at position pos
+ */
+ public final char yycharat(int pos) {
+ return zzBuffer.charAt(zzStartRead+pos);
+ }
+
+
+ /**
+ * Returns the length of the matched text region.
+ */
+ public final int yylength() {
+ return zzMarkedPos-zzStartRead;
+ }
+
+
+ /**
+ * Reports an error that occured while scanning.
+ *
+ * In a wellformed scanner (no or only correct usage of
+ * yypushback(int) and a match-all fallback rule) this method
+ * will only be called with things that "Can't Possibly Happen".
+ * If this method is called, something is seriously wrong
+ * (e.g. a JFlex bug producing a faulty scanner etc.).
+ *
+ * Usual syntax/scanner level error handling should be done
+ * in error fallback rules.
+ *
+ * @param errorCode the code of the errormessage to display
+ */
+--- zzScanError declaration
+ String message;
+ try {
+ message = ZZ_ERROR_MSG[errorCode];
+ }
+ catch (ArrayIndexOutOfBoundsException e) {
+ message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
+ }
+
+--- throws clause
+ }
+
+
+ /**
+ * Pushes the specified amount of characters back into the input stream.
+ *
+ * They will be read again by then next call of the scanning method
+ *
+ * @param number the number of characters to be read again.
+ * This number must not be greater than yylength()!
+ */
+--- yypushback decl (contains zzScanError exception)
+ if ( number > yylength() )
+ zzScanError(ZZ_PUSHBACK_2BIG);
+
+ zzMarkedPos -= number;
+ }
+
+
+--- zzDoEOF
+ /**
+ * Resumes scanning until the next regular expression is matched,
+ * the end of input is encountered or an I/O-Error occurs.
+ *
+ * @return the next token
+ * @exception java.io.IOException if any I/O-Error occurs
+ */
+--- yylex declaration
+ int zzInput;
+ int zzAction;
+
+ // cached fields:
+ int zzCurrentPosL;
+ int zzMarkedPosL;
+ int zzEndReadL = zzEndRead;
+ CharSequence zzBufferL = zzBuffer;
+
+--- local declarations
+
+ while (true) {
+ zzMarkedPosL = zzMarkedPos;
+
+--- start admin (line, char, col count)
+ zzAction = -1;
+
+ zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+
+--- start admin (lexstate etc)
+
+ zzForAction: {
+ while (true) {
+
+--- next input, line, col, char count, next transition, isFinal action
+ zzAction = zzState;
+ zzMarkedPosL = zzCurrentPosL;
+--- line count update
+ }
+
+ }
+ }
+
+ // store back cached position
+ zzMarkedPos = zzMarkedPosL;
+--- char count update
+
+ if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+ zzAtEOF = true;
+--- eofvalue
+ }
+ else {
+--- actions
+ default:
+--- no match
+ }
+ }
+ }
+ }
+
+--- main
+
+}
diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java
index fe0c3374d1e9..ab246ae75f16 100644
--- a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java
+++ b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java
@@ -4,12 +4,14 @@ package com.jetbrains.python.lexer;
import com.intellij.psi.tree.TokenSet;
import com.jetbrains.python.PyTokenTypes;
+import java.io.Reader;
+
/**
* @author yole
*/
public class PythonIndentingLexer extends PythonIndentingProcessor {
public PythonIndentingLexer() {
- super(new PythonLexer(), TokenSet.EMPTY);
+ super(new _PythonLexer((Reader)null), TokenSet.EMPTY);
}
boolean addFinalBreak = true;
diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java
index c458aaa379b8..64ff8bd561b4 100644
--- a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java
+++ b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java
@@ -15,7 +15,8 @@
*/
package com.jetbrains.python.lexer;
-import com.intellij.lexer.LexerBase;
+import com.intellij.lexer.FlexAdapter;
+import com.intellij.lexer.FlexLexer;
import com.intellij.lexer.MergingLexerAdapter;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
@@ -47,8 +48,8 @@ public class PythonIndentingProcessor extends MergingLexerAdapter {
private static final boolean DUMP_TOKENS = false;
private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens();
- public PythonIndentingProcessor(LexerBase lexer, TokenSet tokens) {
- super(lexer, tokens);
+ public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) {
+ super(new FlexAdapter(lexer), tokens);
}
protected static class PendingToken {
diff --git a/python/src/com/jetbrains/python/lexer/PythonLexer.java b/python/src/com/jetbrains/python/lexer/PythonLexer.java
index 9ed0c8801211..4b2ea5b10cec 100644
--- a/python/src/com/jetbrains/python/lexer/PythonLexer.java
+++ b/python/src/com/jetbrains/python/lexer/PythonLexer.java
@@ -2,24 +2,19 @@
package com.jetbrains.python.lexer;
import com.intellij.lexer.FlexAdapter;
-import org.jetbrains.annotations.NotNull;
+
+import java.io.Reader;
/**
* @author yole
*/
public class PythonLexer extends FlexAdapter {
public PythonLexer() {
- super(new _PythonLexer(null));
+ super(new _PythonLexer((Reader)null));
}
@Override
public _PythonLexer getFlex() {
return (_PythonLexer)super.getFlex();
}
-
- @Override
- public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
- super.start(buffer, startOffset, endOffset, initialState);
- getFlex().fStringHelper.reset(startOffset);
- }
}