MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[groovy] lexer: encode braces stack with lexical states

Browse Source 
Before the change YYINITIAL state was returned from lexer even with non-empty braces stack, which was leading to
different token types after reparse. After the change YYINITIAL is returned only outside of parentheses/brackets.
Also added assertions for ensuring that initial state is never pushed on/popped from states stack.
This commit is contained in:
Daniil Ovchinnikov
2018-07-06 14:29:59 +03:00
parent a843f5cd22
commit 22b1027d1e
3 changed files with 1199 additions and 1145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/groovy/groovy-psi/gen/org/jetbrains/plugins/groovy/lang/lexer/_GroovyLexer.java
+1130 -1081
View File
File diff suppressed because it is too large Load Diff
plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/lexer/GroovyLexerBase.java
+10 -32
View File
@@ -9,7 +9,6 @@ import com.intellij.util.containers.Stack;
import static com.intellij.util.ArrayUtil.indexOf;
import static org.jetbrains.plugins.groovy.lang.lexer.GroovyTokenTypes.*;
import static org.jetbrains.plugins.groovy.lang.lexer.TokenSets.KEYWORDS;
import static org.jetbrains.plugins.groovy.lang.lexer.TokenSets.LEFT_BRACES;
public abstract class GroovyLexerBase implements FlexLexer {
@@ -24,10 +23,15 @@ public abstract class GroovyLexerBase implements FlexLexer {
));
public final Stack<Integer> stateStack = new Stack<>();
private final Stack<IElementType> bracesStack = new Stack<>();
protected void resetState() {
stateStack.clear();
}
protected void yybeginstate(int... states) {
assert states.length > 0;
for (int state : states) {
assert state != getInitialState();
stateStack.push(state);
yybegin(state);
}
@@ -35,47 +39,21 @@ public abstract class GroovyLexerBase implements FlexLexer {
protected void yyendstate(int... states) {
for (int state : states) {
int previous = stateStack.isEmpty() ? getInitialState() : stateStack.pop();
if (previous != state) {
throw new RuntimeException("States does not match: previous=" + previous + ", expected=" + state);
}
assert state != getInitialState();
assert !stateStack.isEmpty() : stateStack;
int previous = stateStack.pop();
assert previous == state : "States does not match: previous=" + previous + ", expected=" + state;
}
yybegin(stateStack.isEmpty() ? getInitialState() : stateStack.peek());
}
protected void resetState() {
stateStack.clear();
bracesStack.clear();
}
protected IElementType storeToken(IElementType tokenType) {
if (LEFT_BRACES.contains(tokenType)) {
bracesStack.push(tokenType);
}
else if (tokenType == mRCURLY) {
IElementType leftType = mLCURLY;
while (!bracesStack.isEmpty() && leftType != bracesStack.peek()) {
bracesStack.pop();
}
if (!bracesStack.isEmpty() && leftType == bracesStack.peek()) {
bracesStack.pop();
}
}
else if (tokenType == mRPAREN || tokenType == mRBRACK) {
if (!bracesStack.isEmpty() && bracesStack.peek() != mLCURLY) {
bracesStack.pop();
}
}
if (indexOf(getDivisionStates(), yystate()) != -1 && DIVISION_IS_EXPECTED_AFTER.contains(tokenType)) {
yybeginstate(getDivisionExpectedState());
}
return tokenType;
}
protected boolean isWithinBraces() {
return !bracesStack.empty() && bracesStack.peek() != mLCURLY;
}
protected abstract int getInitialState();
protected abstract int[] getDivisionStates();
plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/lexer/groovy.flex
+59 -32
View File
@@ -31,11 +31,16 @@ import static org.jetbrains.plugins.groovy.lang.psi.GroovyElementTypes.*;
@Override
protected int[] getDivisionStates() {
return new int[] {YYINITIAL, IN_INNER_BLOCK};
return new int[] {YYINITIAL, IN_INJECTION, IN_PARENS_BRACKETS, IN_BRACES};
}
%}
%state IN_INNER_BLOCK
// nls as whitespace
%state IN_PARENS_BRACKETS
// nls as nl but return to previous state instead of YYINITIAL
%state IN_BRACES
// nls as nl but return to previous IN_xxx_STRING state
%state IN_INJECTION
%xstate DIVISION_EXPECTED
@@ -135,22 +140,7 @@ mSTRING_LITERAL = {mSINGLE_QUOTED_LITERAL} | {mTRIPLE_SINGLE_QUOTED_LITERAL}
mGSTRING_LITERAL = {mDOUBLE_QUOTED_LITERAL} | {mTRIPLE_DOUBLE_QUOTED_LITERAL}
%%
<YYINITIAL> {
"}" {
yyendstate(YYINITIAL);
return storeToken(T_RBRACE);
}
}
<IN_INNER_BLOCK> {
"}" {
yyendstate(IN_INNER_BLOCK, IN_GSTRING_DOLLAR);
return storeToken(T_RBRACE);
}
}
<YYINITIAL, IN_INNER_BLOCK, IN_GSTRING_DOLLAR> {
<YYINITIAL, IN_PARENS_BRACKETS, IN_BRACES, IN_INJECTION, IN_GSTRING_DOLLAR> {
"package" { return storeToken(KW_PACKAGE); }
"strictfp" { return storeToken(KW_STRICTFP); }
"import" { return storeToken(KW_IMPORT); }
@@ -318,7 +308,7 @@ mGSTRING_LITERAL = {mDOUBLE_QUOTED_LITERAL} | {mTRIPLE_DOUBLE_QUOTED_LITERAL}
}
"{" {
yybeginstate(IN_INNER_BLOCK, NLS_AFTER_LBRACE);
yybeginstate(IN_INJECTION, NLS_AFTER_LBRACE);
return storeToken(T_LBRACE);
}
@@ -351,14 +341,59 @@ mGSTRING_LITERAL = {mDOUBLE_QUOTED_LITERAL} | {mTRIPLE_DOUBLE_QUOTED_LITERAL}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////// White spaces & NewLines //////////////////////////////////////////////////////////////////////
///////////////////////// Parentheses and braces ///////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
<YYINITIAL, IN_BRACES, IN_INJECTION> {
{mNLS} { yybeginstate(NLS_AFTER_NLS); return storeToken(NL); }
}
<IN_PARENS_BRACKETS> {
{mNLS} { yybeginstate(NLS_AFTER_NLS); return TokenType.WHITE_SPACE; }
}
<YYINITIAL, IN_PARENS_BRACKETS, IN_BRACES, IN_INJECTION> {
"(" { yybeginstate(IN_PARENS_BRACKETS); return storeToken(T_LPAREN); }
"[" { yybeginstate(IN_PARENS_BRACKETS); return storeToken(T_LBRACK); }
}
<IN_PARENS_BRACKETS, IN_BRACES, IN_INJECTION> {
"{" { yybeginstate(IN_BRACES, NLS_AFTER_LBRACE); return storeToken(T_LBRACE); }
}
<YYINITIAL> {
"{" { yybeginstate(NLS_AFTER_LBRACE); return storeToken(T_LBRACE); }
}
<YYINITIAL, IN_BRACES, IN_INJECTION> {
")" { return storeToken(T_RPAREN); }
"]" { return storeToken(T_RBRACK); }
}
<IN_PARENS_BRACKETS> {
")" { yyendstate(IN_PARENS_BRACKETS); return storeToken(T_RPAREN); }
"]" { yyendstate(IN_PARENS_BRACKETS); return storeToken(T_RBRACK); }
}
<YYINITIAL> {
"}" { return storeToken(T_RBRACE); }
}
<IN_PARENS_BRACKETS> {
"}" {
while (yystate() == IN_PARENS_BRACKETS) {
yyendstate(IN_PARENS_BRACKETS);
}
return storeToken(T_RBRACE);
}
}
<IN_BRACES> {
"}" { yyendstate(IN_BRACES); return storeToken(T_RBRACE); }
}
<IN_INJECTION> {
"}" { yyendstate(IN_INJECTION, IN_GSTRING_DOLLAR); return storeToken(T_RBRACE); }
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////// White spaces ////////// //////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{WHITE_SPACE} { return TokenType.WHITE_SPACE; }
{mNLS} {
yybeginstate(NLS_AFTER_NLS);
return isWithinBraces() ? TokenType.WHITE_SPACE : storeToken(NL);
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////Comments //////////////////////////////////////////////////////////////////////////////////////
@@ -440,15 +475,7 @@ mGSTRING_LITERAL = {mDOUBLE_QUOTED_LITERAL} | {mTRIPLE_DOUBLE_QUOTED_LITERAL}
yybeginstate(IN_DOLLAR_SLASH_STRING);
return storeToken(DOLLAR_SLASHY_BEGIN);
}
"{" {
yybeginstate(YYINITIAL, NLS_AFTER_LBRACE);
return storeToken(T_LBRACE);
}
"?" { return storeToken(T_Q); }
"(" { return storeToken(T_LPAREN); }
")" { return storeToken(T_RPAREN); }
"[" { return storeToken(T_LBRACK); }
"]" { return storeToken(T_RBRACK); }
":" { return storeToken(T_COLON); }
"," { return storeToken(T_COMMA); }
"." { return storeToken(T_DOT); }