MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

RegExp: support MySQL syntax extension

Browse Source
This commit is contained in:
Alexander Kass
2017-10-09 15:28:25 +03:00
parent 322f0941cc
commit 2364a690b0
8 changed files with 451 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files
RegExpSupport/gen/org/intellij/lang/regexp/_RegExLexer.java
+361 -307
View File
File diff suppressed because it is too large Load Diff
RegExpSupport/src/org/intellij/lang/regexp/RegExpCapability.java
+5
View File
@@ -111,4 +111,9 @@ public enum RegExpCapability {
* Allow \x9 in addition to \x09 (ruby)
*/
ONE_HEX_CHAR_ESCAPE,
/**
* MySQL character classes [=c=] [.class.] [:<:] [:>:]
*/
MYSQL_BRACKET_EXPRESSIONS,
}
RegExpSupport/src/org/intellij/lang/regexp/RegExpElementTypes.java
+2
View File
@@ -40,6 +40,8 @@ public interface RegExpElementTypes {
IElementType NAMED_GROUP_REF = new RegExpElementType("NAMED_GROUP_REF");
IElementType PY_COND_REF = new RegExpElementType("PY_COND_REF");
IElementType POSIX_BRACKET_EXPRESSION = new RegExpElementType("POSIX_BRACKET_EXPRESSION");
IElementType MYSQL_CHAR_EXPRESSION = new RegExpElementType("MYSQL_CHAR_EXPRESSION");
IElementType MYSQL_CHAR_EQ_EXPRESSION = new RegExpElementType("MYSQL_CHAR_EQ_EXPRESSION");
IElementType NUMBER = new RegExpElementType("NUMBER");
TokenSet ATOMS = TokenSet.create(CLOSURE, BOUNDARY, SIMPLE_CLASS, CLASS, CHAR, GROUP, PROPERTY, BACKREF, NAMED_GROUP_REF,
RegExpSupport/src/org/intellij/lang/regexp/RegExpParser.java
+27
View File
@@ -256,6 +256,12 @@ public class RegExpParser implements PsiParser, LightPsiParser {
else if (token == RegExpTT.BRACKET_EXPRESSION_BEGIN) {
parseBracketExpression(builder);
}
else if (token == RegExpTT.MYSQL_CHAR_BEGIN) {
parseMysqlCharExpression(builder);
}
else if (token == RegExpTT.MYSQL_CHAR_EQ_BEGIN) {
parseMysqlCharEqExpression(builder);
}
else if (RegExpTT.CHARACTERS.contains(token) || token == RegExpTT.NAMED_CHARACTER) {
parseCharacterRange(builder);
}
@@ -285,6 +291,27 @@ public class RegExpParser implements PsiParser, LightPsiParser {
marker.done(RegExpElementTypes.POSIX_BRACKET_EXPRESSION);
}
private static void parseMysqlCharExpression(PsiBuilder builder) {
final PsiBuilder.Marker marker = builder.mark();
builder.advanceLexer();
if (builder.getTokenType() == RegExpTT.NAME) {
builder.advanceLexer();
}
else {
checkMatches(builder, RegExpTT.CHARACTER, "Character or MySQL character name expected");
}
checkMatches(builder, RegExpTT.MYSQL_CHAR_END, "Unclosed MySQL character expression");
marker.done(RegExpElementTypes.MYSQL_CHAR_EXPRESSION);
}
private static void parseMysqlCharEqExpression(PsiBuilder builder) {
final PsiBuilder.Marker marker = builder.mark();
builder.advanceLexer();
checkMatches(builder, RegExpTT.CHARACTER, "Character expected");
checkMatches(builder, RegExpTT.MYSQL_CHAR_EQ_END, "Unclosed MySQL character equivalence class");
marker.done(RegExpElementTypes.MYSQL_CHAR_EQ_EXPRESSION);
}
private void parseCharacterRange(PsiBuilder builder) {
final PsiBuilder.Marker rangeMarker = builder.mark();
parseCharacter(builder);
RegExpSupport/src/org/intellij/lang/regexp/RegExpTT.java
+8
View File
@@ -48,6 +48,14 @@ public interface RegExpTT {
IElementType BRACKET_EXPRESSION_BEGIN = new RegExpElementType("BRACKET_EXPRESSION_BEGIN");
/** ":]" */
IElementType BRACKET_EXPRESSION_END = new RegExpElementType("BRACKET_EXPRESSION_END");
/** "[." */
IElementType MYSQL_CHAR_BEGIN = new RegExpElementType("MYSQL_CHAR_BEGIN");
/** ".]" */
IElementType MYSQL_CHAR_END = new RegExpElementType("MYSQL_CHAR_END");
/** "[=" */
IElementType MYSQL_CHAR_EQ_BEGIN = new RegExpElementType("MYSQL_CHAR_EQ_BEGIN");
/** "=]" */
IElementType MYSQL_CHAR_EQ_END = new RegExpElementType("MYSQL_CHAR_EQ_END");
/** "(" */
IElementType GROUP_BEGIN = new RegExpElementType("GROUP_BEGIN");
RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex
+29 -1
View File
@@ -41,6 +41,7 @@ import static org.intellij.lang.regexp.RegExpCapability.*;
private boolean allowTransformationEscapes;
private boolean allowExtendedUnicodeCharacter;
private boolean allowOneHexCharEscape;
private boolean allowMysqlBracketExpressions;
private int maxOctal = 0777;
private int minOctalDigits = 1;
@@ -59,6 +60,7 @@ import static org.intellij.lang.regexp.RegExpCapability.*;
this.allowEmptyCharacterClass = capabilities.contains(ALLOW_EMPTY_CHARACTER_CLASS);
this.allowPosixBracketExpressions = capabilities.contains(POSIX_BRACKET_EXPRESSIONS);
this.allowTransformationEscapes = capabilities.contains(TRANSFORMATION_ESCAPES);
this.allowMysqlBracketExpressions = capabilities.contains(MYSQL_BRACKET_EXPRESSIONS);
if (capabilities.contains(MAX_OCTAL_177)) {
maxOctal = 0177;
}
@@ -113,6 +115,8 @@ import static org.intellij.lang.regexp.RegExpCapability.*;
%xstate PY_NAMED_GROUP_REF
%xstate PY_COND_REF
%xstate BRACKET_EXPRESSION
%xstate MYSQL_CHAR_EXPRESSION
%xstate MYSQL_CHAR_EQ_EXPRESSION
%xstate EMBRACED_HEX
DOT="."
@@ -126,6 +130,7 @@ RBRACKET="]"
ESCAPE="\\"
NAME=[:letter:]([:letter:]|_|-|" "|"("|")"|[:digit:])*
GROUP_NAME=[:letter:]([:letter:]|_|-|" "|[:digit:])*
MYSQL_CHAR_NAME=[:letter:](-|[:letter:])*[:digit:]?
ANY=[^]
META1 = {ESCAPE} | {LBRACKET}
@@ -322,6 +327,7 @@ HEX_CHAR=[0-9a-fA-F]
{ESCAPE} {RBRACKET} { yybegin(CLASS2); return allowEmptyCharacterClass ? RegExpTT.ESC_CHARACTER : RegExpTT.REDUNDANT_ESCAPE; }
{RBRACKET} { if (allowEmptyCharacterClass) { yypopstate(); return RegExpTT.CLASS_END; } yybegin(CLASS2); return RegExpTT.CHARACTER; }
{LBRACKET} / ":" { yybegin(CLASS2); if (allowPosixBracketExpressions) { yypushback(1); } else if (allowNestedCharacterClasses) { yypushstate(CLASS1); return RegExpTT.CLASS_BEGIN; } else { return RegExpTT.CHARACTER; } }
{LBRACKET} / [.=] { yybegin(CLASS2); if (allowMysqlBracketExpressions) { yypushback(1); } else if (allowNestedCharacterClasses) { yypushstate(CLASS1); return RegExpTT.CLASS_BEGIN; } else { return RegExpTT.CHARACTER; } }
{LBRACKET} / "^" { yybegin(CLASS2); if (allowNestedCharacterClasses) { yypushstate(NEGATED_CLASS); return RegExpTT.CLASS_BEGIN; } return RegExpTT.CHARACTER; }
{LBRACKET} { yybegin(CLASS2); if (allowNestedCharacterClasses) { yypushstate(CLASS1); return RegExpTT.CLASS_BEGIN; } return RegExpTT.CHARACTER; }
[\n\b\t\r\f ] { if (commentMode) return com.intellij.psi.TokenType.WHITE_SPACE; yypushback(1); yybegin(CLASS2); }
@@ -329,9 +335,16 @@ HEX_CHAR=[0-9a-fA-F]
}
<CLASS2> {
{LBRACKET} ":" { if (allowPosixBracketExpressions) {
{LBRACKET} [:=.] { char c = yycharat(1);
if (allowPosixBracketExpressions && c == ':') {
yybegin(BRACKET_EXPRESSION);
return RegExpTT.BRACKET_EXPRESSION_BEGIN;
} else if (allowMysqlBracketExpressions && c == '=') {
yybegin(MYSQL_CHAR_EQ_EXPRESSION);
return RegExpTT.MYSQL_CHAR_EQ_BEGIN;
} else if (allowMysqlBracketExpressions && c == '.') {
yybegin(MYSQL_CHAR_EXPRESSION);
return RegExpTT.MYSQL_CHAR_BEGIN;
} else {
yypushback(1);
return allowNestedCharacterClasses ? RegExpTT.CLASS_BEGIN : RegExpTT.CHARACTER;
@@ -351,6 +364,21 @@ HEX_CHAR=[0-9a-fA-F]
"^" { return RegExpTT.CARET; }
{NAME} { return RegExpTT.NAME; }
":" {RBRACKET} { yybegin(CLASS2); return RegExpTT.BRACKET_EXPRESSION_END; }
[<>] { return allowMysqlBracketExpressions ? RegExpTT.NAME : RegExpTT.BAD_CHARACTER; }
{ANY} { return RegExpTT.BAD_CHARACTER; }
}
<MYSQL_CHAR_EXPRESSION> {
{MYSQL_CHAR_NAME} { return RegExpTT.NAME; }
"." {RBRACKET} { yybegin(CLASS2); return RegExpTT.MYSQL_CHAR_END; }
{ANY} / "." {RBRACKET} { return RegExpTT.CHARACTER; }
{ANY} { return RegExpTT.BAD_CHARACTER; }
}
<MYSQL_CHAR_EQ_EXPRESSION> {
{NAME} { return RegExpTT.NAME; }
"=" {RBRACKET} { yybegin(CLASS2); return RegExpTT.MYSQL_CHAR_EQ_END; }
{ANY} / "=" {RBRACKET} { return RegExpTT.CHARACTER; }
{ANY} { return RegExpTT.BAD_CHARACTER; }
}
RegExpSupport/src/org/intellij/lang/regexp/validation/RegExpAnnotator.java
+1 -1
View File
@@ -418,7 +418,7 @@ public final class RegExpAnnotator extends RegExpElementVisitor implements Annot
@Override
public void visitPosixBracketExpression(RegExpPosixBracketExpression posixBracketExpression) {
final String className = posixBracketExpression.getClassName();
if (!POSIX_CHARACTER_CLASSES.contains(className)) {
if (!POSIX_CHARACTER_CLASSES.contains(className) && !"<".equals(className) && !">".equals(className)) {
final ASTNode node = posixBracketExpression.getNode().findChildByType(RegExpTT.NAME);
if (node != null) {
final Annotation annotation = myHolder.createErrorAnnotation(node, "Unknown POSIX character class");
RegExpSupport/test/org/intellij/lang/regexp/RegExpLexerTest.java
+18
View File
@@ -132,6 +132,24 @@ public class RegExpLexerTest extends LexerTestCase {
"CLASS_END (']')", lexer);
}
public void testMysqlCharExpressions() {
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(MYSQL_BRACKET_EXPRESSIONS));
doTest("[[.~.][.tilda.][.NUL.][.plus-sign.]]", "CLASS_BEGIN ('[')\n" +
"MYSQL_CHAR_BEGIN ('[.')\nCHARACTER ('~')\nMYSQL_CHAR_END ('.]')\n" +
"MYSQL_CHAR_BEGIN ('[.')\nNAME ('tilda')\nMYSQL_CHAR_END ('.]')\n" +
"MYSQL_CHAR_BEGIN ('[.')\nNAME ('NUL')\nMYSQL_CHAR_END ('.]')\n" +
"MYSQL_CHAR_BEGIN ('[.')\nNAME ('plus-sign')\nMYSQL_CHAR_END ('.]')\n" +
"CLASS_END (']')", lexer);
}
public void testMysqlCharEqExpressions() {
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(MYSQL_BRACKET_EXPRESSIONS));
doTest("[[=.=][=c=]]", "CLASS_BEGIN ('[')\n" +
"MYSQL_CHAR_EQ_BEGIN ('[=')\nCHARACTER ('.')\nMYSQL_CHAR_EQ_END ('=]')\n" +
"MYSQL_CHAR_EQ_BEGIN ('[=')\nCHARACTER ('c')\nMYSQL_CHAR_EQ_END ('=]')\n" +
"CLASS_END (']')", lexer);
}
/**
* \\177 is the maximum valid octal character under Ruby.
*/