MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-18 09:34:34 +07:00
Code Issues Packages Projects Releases Wiki Activity

[groovy] lexer: separate state for when division is expected (IDEA-170650)

Browse Source 
On document change lexer was started with initial state where '/' is
matched as a beginning of the regex literal.
This commit is contained in:
Daniil Ovchinnikov
2017-03-31 15:47:04 +03:00
parent a4e4356f7c
commit 9d74ed70a6
8 changed files with 1155 additions and 1065 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/groovy/groovy-psi/gen/org/jetbrains/plugins/groovy/lang/lexer/_GroovyLexer.java
+1068 -1043
View File
File diff suppressed because it is too large Load Diff
plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/lexer/GroovyLexerBase.java
+8 -7
View File
@@ -20,6 +20,7 @@ import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.containers.Stack;
import static com.intellij.util.ArrayUtil.indexOf;
import static org.jetbrains.plugins.groovy.lang.lexer.GroovyTokenTypes.*;
import static org.jetbrains.plugins.groovy.lang.lexer.TokenSets.KEYWORDS;
import static org.jetbrains.plugins.groovy.lang.lexer.TokenSets.LEFT_BRACES;
@@ -38,7 +39,6 @@ public abstract class GroovyLexerBase implements FlexLexer {
public final Stack<Integer> stateStack = new Stack<>();
private final Stack<IElementType> bracesStack = new Stack<>();
private IElementType lastToken = null;
protected void yybeginstate(int... states) {
for (int state : states) {
@@ -60,7 +60,6 @@ public abstract class GroovyLexerBase implements FlexLexer {
protected void resetState() {
stateStack.clear();
bracesStack.clear();
lastToken = null;
}
protected IElementType storeToken(IElementType tokenType) {
@@ -81,17 +80,19 @@ public abstract class GroovyLexerBase implements FlexLexer {
bracesStack.pop();
}
}
lastToken = tokenType;
if (indexOf(getDivisionStates(), yystate()) != -1 && DIVISION_IS_EXPECTED_AFTER.contains(tokenType)) {
yybeginstate(getDivisionExpectedState());
}
return tokenType;
}
protected boolean isRegexExpected() {
return !DIVISION_IS_EXPECTED_AFTER.contains(lastToken);
}
protected boolean isWithinBraces() {
return !bracesStack.empty() && bracesStack.peek() != mLCURLY;
}
protected abstract int getInitialState();
protected abstract int[] getDivisionStates();
protected abstract int getDivisionExpectedState();
}
plugins/groovy/groovy-psi/src/org/jetbrains/plugins/groovy/lang/lexer/groovy.flex
+36 -13
View File
@@ -34,13 +34,26 @@ import static org.jetbrains.plugins.groovy.lang.lexer.GroovyTokenTypes.*;
%type IElementType
%{
@Override
protected int getInitialState() {
return YYINITIAL;
}
@Override
protected int getDivisionExpectedState() {
return DIVISION_EXPECTED;
}
@Override
protected int[] getDivisionStates() {
return new int[] {YYINITIAL, IN_INNER_BLOCK};
}
%}
%state IN_INNER_BLOCK
%xstate DIVISION_EXPECTED
%xstate IN_SINGLE_GSTRING
%xstate IN_TRIPLE_GSTRING
%xstate IN_SLASHY_STRING
@@ -407,22 +420,32 @@ mGSTRING_LITERAL = {mDOUBLE_QUOTED_LITERAL} | {mTRIPLE_DOUBLE_QUOTED_LITERAL}
///////////////////////// Reserved shorthands //////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
<DIVISION_EXPECTED> {
{WHITE_SPACE} {
return TokenType.WHITE_SPACE;
}
"/"/[^/*=] {
yyendstate(DIVISION_EXPECTED);
return storeToken(mDIV);
}
"$/" {
yypushback(1);
yyendstate(DIVISION_EXPECTED);
return storeToken(mDOLLAR);
}
[^] {
yypushback(1);
yyendstate(DIVISION_EXPECTED);
}
}
"/" {
if (isRegexExpected()) {
yybeginstate(IN_SLASHY_STRING);
return storeToken(mREGEX_BEGIN);
} else {
return storeToken(mDIV);
}
yybeginstate(IN_SLASHY_STRING);
return storeToken(mREGEX_BEGIN);
}
"$/" {
if (isRegexExpected()) {
yybeginstate(IN_DOLLAR_SLASH_STRING);
return storeToken(mDOLLAR_SLASH_REGEX_BEGIN);
} else {
yypushback(1);
return storeToken(mDOLLAR);
}
yybeginstate(IN_DOLLAR_SLASH_STRING);
return storeToken(mDOLLAR_SLASH_REGEX_BEGIN);
}
"{" {
yybeginstate(YYINITIAL, NLS_AFTER_LBRACE);
plugins/groovy/test/org/jetbrains/plugins/groovy/lang/parser/CommentsParsingTest.groovy
+3 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2016 JetBrains s.r.o.
* Copyright 2000-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,5 +35,6 @@ class CommentsParsingTest extends GroovyParsingTestCase {
void testNls2() throws Throwable { doTest() }
void testRocher3() throws Throwable { doTest() }
void testAfterIdentifier() throws Throwable { doTest() }
}
plugins/groovy/test/org/jetbrains/plugins/groovy/lang/parser/ExpressionsParsingTest.groovy
+4
View File
@@ -50,6 +50,8 @@ class ExpressionsParsingTest extends GroovyParsingTestCase {
void testarithmetic$mul3() throws Throwable { doTest() }
void testarithmetic$mul4() throws Throwable { doTest() }
void testarithmetic$post1() throws Throwable { doTest() }
void testarithmetic$sh1() throws Throwable { doTest() }
@@ -408,6 +410,8 @@ class ExpressionsParsingTest extends GroovyParsingTestCase {
void testregex$dollarSlashyUltimate() { doTest() }
void testregex$afterNewLine() { doTest() }
void testrelational$eq1() throws Throwable { doTest() }
void testrelational$inst0() throws Throwable { doTest() }
plugins/groovy/testdata/parsing/groovy/comments/afterIdentifier.test
Vendored
+13
View File
@@ -0,0 +1,13 @@
a //
a /* */
-----
Groovy script
Reference expression
PsiElement(identifier)('a')
PsiWhiteSpace(' ')
PsiComment(line comment)('//')
PsiElement(new line)('\n')
Reference expression
PsiElement(identifier)('a')
PsiWhiteSpace(' ')
PsiComment(block comment)('/* */')
plugins/groovy/testdata/parsing/groovy/expressions/arithmetic/mul4.test
Vendored
+11
View File
@@ -0,0 +1,11 @@
a /= b
-----
Groovy script
Assignment expression
Reference expression
PsiElement(identifier)('a')
PsiWhiteSpace(' ')
PsiElement(/=)('/=')
PsiWhiteSpace(' ')
Reference expression
PsiElement(identifier)('b')
plugins/groovy/testdata/parsing/groovy/expressions/regex/afterNewLine.test
Vendored
+12
View File
@@ -0,0 +1,12 @@
a
/foo/
-----
Groovy script
Reference expression
PsiElement(identifier)('a')
PsiElement(new line)('\n')
Literal
GroovyASTPsiElementImpl(regex literal)
PsiElement(regex begin)('/')
PsiElement(regex content)('foo')
PsiElement(regex end)('/')