From 274ca1e19a53771dd95afbaa36f4c677b7f16586 Mon Sep 17 00:00:00 2001 From: Mikhail Golubev Date: Fri, 7 Sep 2018 19:53:28 +0300 Subject: [PATCH] PY-31442 As in lexers of other languages, reset the state in the skeleton This way it's more self-contained, and we no longer require to use PythonLexer instead everywhere. Thus I reverted previously made changes. --- .../jetbrains/python/lexer/_PythonLexer.java | 1 + .../python/lexer/PyLexerFStringHelper.kt | 18 +- .../com/jetbrains/python/lexer/Python.flex | 2 +- .../jetbrains/python/lexer/Python.skeleton | 249 ++++++++++++++++++ .../python/lexer/PythonIndentingLexer.java | 4 +- .../lexer/PythonIndentingProcessor.java | 7 +- .../jetbrains/python/lexer/PythonLexer.java | 11 +- 7 files changed, 265 insertions(+), 27 deletions(-) create mode 100644 python/src/com/jetbrains/python/lexer/Python.skeleton diff --git a/python/gen/com/jetbrains/python/lexer/_PythonLexer.java b/python/gen/com/jetbrains/python/lexer/_PythonLexer.java index 1becb6921660..8ca1155f85d8 100644 --- a/python/gen/com/jetbrains/python/lexer/_PythonLexer.java +++ b/python/gen/com/jetbrains/python/lexer/_PythonLexer.java @@ -870,6 +870,7 @@ return yylength()-s.length(); zzAtEOF = false; zzAtBOL = true; zzEndRead = end; + fStringHelper.reset(); yybegin(initialState); } diff --git a/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt b/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt index 89d67b3534cc..103c1ce140f5 100644 --- a/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt +++ b/python/src/com/jetbrains/python/lexer/PyLexerFStringHelper.kt @@ -153,20 +153,10 @@ class PyLexerFStringHelper(private val myLexer: FlexLexerEx) { } } - fun reset(offset: Int) { - if (offset == 0) { - myFStringStates.clear() - return - } - while (!myFStringStates.isEmpty() && offset < myFStringStates.peek().offset) { - myFStringStates.pop() - } - if (!myFStringStates.isEmpty()) { - val fragmentStates = myFStringStates.peek().fragmentStates - while (!fragmentStates.isEmpty() && offset < fragmentStates.peek().offset) { - fragmentStates.pop() - } - } + fun reset() { + // There is no need to be smarter about it, since LexerEditorHighlighter always resets + // the lexer state to YYINITIAL where there can't be any f-strings. + myFStringStates.clear() } private data class FStringState(val oldState: Int, val offset: Int, val openingQuotes: String) { diff --git a/python/src/com/jetbrains/python/lexer/Python.flex b/python/src/com/jetbrains/python/lexer/Python.flex index 4319483b1315..ded1f5223068 100644 --- a/python/src/com/jetbrains/python/lexer/Python.flex +++ b/python/src/com/jetbrains/python/lexer/Python.flex @@ -86,7 +86,7 @@ FSTRING_FRAGMENT_TYPE_CONVERSION = "!" [^=:'\"} \t\r\n]* %state FSTRING_FRAGMENT %xstate FSTRING_FRAGMENT_FORMAT %{ -final PyLexerFStringHelper fStringHelper = new PyLexerFStringHelper(this); +private final PyLexerFStringHelper fStringHelper = new PyLexerFStringHelper(this); private int getSpaceLength(CharSequence string) { String string1 = string.toString(); diff --git a/python/src/com/jetbrains/python/lexer/Python.skeleton b/python/src/com/jetbrains/python/lexer/Python.skeleton new file mode 100644 index 000000000000..946f183858c4 --- /dev/null +++ b/python/src/com/jetbrains/python/lexer/Python.skeleton @@ -0,0 +1,249 @@ + + /** This character denotes the end of file */ + public static final int YYEOF = -1; + + /** initial size of the lookahead buffer */ +--- private static final int ZZ_BUFFERSIZE = ...; + + /** lexical states */ +--- lexical states, charmap + + /* error codes */ + private static final int ZZ_UNKNOWN_ERROR = 0; + private static final int ZZ_NO_MATCH = 1; + private static final int ZZ_PUSHBACK_2BIG = 2; + + /* error messages for the codes above */ + private static final String[] ZZ_ERROR_MSG = { + "Unknown internal scanner error", + "Error: could not match input", + "Error: pushback value was too large" + }; + +--- isFinal list + /** the input device */ + private java.io.Reader zzReader; + + /** the current state of the DFA */ + private int zzState; + + /** the current lexical state */ + private int zzLexicalState = YYINITIAL; + + /** this buffer contains the current text to be matched and is + the source of the yytext() string */ + private CharSequence zzBuffer = ""; + + /** the textposition at the last accepting state */ + private int zzMarkedPos; + + /** the current text position in the buffer */ + private int zzCurrentPos; + + /** startRead marks the beginning of the yytext() string in the buffer */ + private int zzStartRead; + + /** endRead marks the last character in the buffer, that has been read + from input */ + private int zzEndRead; + + /** + * zzAtBOL == true <=> the scanner is currently at the beginning of a line + */ + private boolean zzAtBOL = true; + + /** zzAtEOF == true <=> the scanner is at the EOF */ + private boolean zzAtEOF; + + /** denotes if the user-EOF-code has already been executed */ + private boolean zzEOFDone; + +--- user class code + +--- constructor declaration + + public final int getTokenStart() { + return zzStartRead; + } + + public final int getTokenEnd() { + return getTokenStart() + yylength(); + } + + public void reset(CharSequence buffer, int start, int end, int initialState) { + zzBuffer = buffer; + zzCurrentPos = zzMarkedPos = zzStartRead = start; + zzAtEOF = false; + zzAtBOL = true; + zzEndRead = end; + fStringHelper.reset(); + yybegin(initialState); + } + + /** + * Refills the input buffer. + * + * @return false, iff there was new input. + * + * @exception java.io.IOException if any I/O-Error occurs + */ + private boolean zzRefill() throws java.io.IOException { + return true; + } + + + /** + * Returns the current lexical state. + */ + public final int yystate() { + return zzLexicalState; + } + + + /** + * Enters a new lexical state + * + * @param newState the new lexical state + */ + public final void yybegin(int newState) { + zzLexicalState = newState; + } + + + /** + * Returns the text matched by the current regular expression. + */ + public final CharSequence yytext() { + return zzBuffer.subSequence(zzStartRead, zzMarkedPos); + } + + + /** + * Returns the character at position pos from the + * matched text. + * + * It is equivalent to yytext().charAt(pos), but faster + * + * @param pos the position of the character to fetch. + * A value from 0 to yylength()-1. + * + * @return the character at position pos + */ + public final char yycharat(int pos) { + return zzBuffer.charAt(zzStartRead+pos); + } + + + /** + * Returns the length of the matched text region. + */ + public final int yylength() { + return zzMarkedPos-zzStartRead; + } + + + /** + * Reports an error that occured while scanning. + * + * In a wellformed scanner (no or only correct usage of + * yypushback(int) and a match-all fallback rule) this method + * will only be called with things that "Can't Possibly Happen". + * If this method is called, something is seriously wrong + * (e.g. a JFlex bug producing a faulty scanner etc.). + * + * Usual syntax/scanner level error handling should be done + * in error fallback rules. + * + * @param errorCode the code of the errormessage to display + */ +--- zzScanError declaration + String message; + try { + message = ZZ_ERROR_MSG[errorCode]; + } + catch (ArrayIndexOutOfBoundsException e) { + message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; + } + +--- throws clause + } + + + /** + * Pushes the specified amount of characters back into the input stream. + * + * They will be read again by then next call of the scanning method + * + * @param number the number of characters to be read again. + * This number must not be greater than yylength()! + */ +--- yypushback decl (contains zzScanError exception) + if ( number > yylength() ) + zzScanError(ZZ_PUSHBACK_2BIG); + + zzMarkedPos -= number; + } + + +--- zzDoEOF + /** + * Resumes scanning until the next regular expression is matched, + * the end of input is encountered or an I/O-Error occurs. + * + * @return the next token + * @exception java.io.IOException if any I/O-Error occurs + */ +--- yylex declaration + int zzInput; + int zzAction; + + // cached fields: + int zzCurrentPosL; + int zzMarkedPosL; + int zzEndReadL = zzEndRead; + CharSequence zzBufferL = zzBuffer; + +--- local declarations + + while (true) { + zzMarkedPosL = zzMarkedPos; + +--- start admin (line, char, col count) + zzAction = -1; + + zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; + +--- start admin (lexstate etc) + + zzForAction: { + while (true) { + +--- next input, line, col, char count, next transition, isFinal action + zzAction = zzState; + zzMarkedPosL = zzCurrentPosL; +--- line count update + } + + } + } + + // store back cached position + zzMarkedPos = zzMarkedPosL; +--- char count update + + if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { + zzAtEOF = true; +--- eofvalue + } + else { +--- actions + default: +--- no match + } + } + } + } + +--- main + +} diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java index fe0c3374d1e9..ab246ae75f16 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingLexer.java @@ -4,12 +4,14 @@ package com.jetbrains.python.lexer; import com.intellij.psi.tree.TokenSet; import com.jetbrains.python.PyTokenTypes; +import java.io.Reader; + /** * @author yole */ public class PythonIndentingLexer extends PythonIndentingProcessor { public PythonIndentingLexer() { - super(new PythonLexer(), TokenSet.EMPTY); + super(new _PythonLexer((Reader)null), TokenSet.EMPTY); } boolean addFinalBreak = true; diff --git a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java index c458aaa379b8..64ff8bd561b4 100644 --- a/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java +++ b/python/src/com/jetbrains/python/lexer/PythonIndentingProcessor.java @@ -15,7 +15,8 @@ */ package com.jetbrains.python.lexer; -import com.intellij.lexer.LexerBase; +import com.intellij.lexer.FlexAdapter; +import com.intellij.lexer.FlexLexer; import com.intellij.lexer.MergingLexerAdapter; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; @@ -47,8 +48,8 @@ public class PythonIndentingProcessor extends MergingLexerAdapter { private static final boolean DUMP_TOKENS = false; private final TokenSet RECOVERY_TOKENS = PythonDialectsTokenSetProvider.INSTANCE.getUnbalancedBracesRecoveryTokens(); - public PythonIndentingProcessor(LexerBase lexer, TokenSet tokens) { - super(lexer, tokens); + public PythonIndentingProcessor(FlexLexer lexer, TokenSet tokens) { + super(new FlexAdapter(lexer), tokens); } protected static class PendingToken { diff --git a/python/src/com/jetbrains/python/lexer/PythonLexer.java b/python/src/com/jetbrains/python/lexer/PythonLexer.java index 9ed0c8801211..4b2ea5b10cec 100644 --- a/python/src/com/jetbrains/python/lexer/PythonLexer.java +++ b/python/src/com/jetbrains/python/lexer/PythonLexer.java @@ -2,24 +2,19 @@ package com.jetbrains.python.lexer; import com.intellij.lexer.FlexAdapter; -import org.jetbrains.annotations.NotNull; + +import java.io.Reader; /** * @author yole */ public class PythonLexer extends FlexAdapter { public PythonLexer() { - super(new _PythonLexer(null)); + super(new _PythonLexer((Reader)null)); } @Override public _PythonLexer getFlex() { return (_PythonLexer)super.getFlex(); } - - @Override - public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) { - super.start(buffer, startOffset, endOffset, initialState); - getFlex().fStringHelper.reset(startOffset); - } }