From e586f1760849d6ed19a35fe51ac343eabb53363b Mon Sep 17 00:00:00 2001 From: Oleg Sukhodolsky Date: Tue, 2 Oct 2012 13:06:19 +0400 Subject: [PATCH] RUBY-12161: partial fix - RegExpLexer should accept \0 as valid octal char if allowOctalNoLeadingZero is set - Ruby19RegExp language needs its own highlighter --- .../RegExpSyntaxHighlighterFactory.java | 8 +- .../org/intellij/lang/regexp/_RegExLexer.java | 453 +++++++++--------- .../intellij/lang/regexp/regexp-lexer.flex | 2 +- 3 files changed, 235 insertions(+), 228 deletions(-) diff --git a/RegExpSupport/src/org/intellij/lang/regexp/RegExpSyntaxHighlighterFactory.java b/RegExpSupport/src/org/intellij/lang/regexp/RegExpSyntaxHighlighterFactory.java index 5c48f05dff63..9850a68376d4 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/RegExpSyntaxHighlighterFactory.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/RegExpSyntaxHighlighterFactory.java @@ -15,6 +15,7 @@ */ package org.intellij.lang.regexp; +import com.intellij.lang.Language; import com.intellij.lang.LanguageParserDefinitions; import com.intellij.lang.ParserDefinition; import com.intellij.openapi.fileTypes.SingleLazyInstanceSyntaxHighlighterFactory; @@ -25,9 +26,14 @@ public class RegExpSyntaxHighlighterFactory extends SingleLazyInstanceSyntaxHigh private final ParserDefinition myParserDefinition; public RegExpSyntaxHighlighterFactory() { - myParserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(RegExpLanguage.INSTANCE); + this(RegExpLanguage.INSTANCE); } + protected RegExpSyntaxHighlighterFactory(@NotNull Language language) { + myParserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language); + } + + @NotNull protected SyntaxHighlighter createHighlighter() { return new RegExpHighlighter(null, myParserDefinition); diff --git a/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java b/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java index aa038f4bca01..5402d7ee78d2 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java @@ -1,13 +1,14 @@ -/* The following code was generated by JFlex 1.4.3 on 21.06.11 21:06 */ +/* The following code was generated by JFlex 1.4.3 on 10/1/12 11:19 AM */ /* It's an automatically generated code. Do not modify it. */ package org.intellij.lang.regexp; import com.intellij.lexer.FlexLexer; -import com.intellij.psi.tree.IElementType; -import java.util.LinkedList; -import java.util.EnumSet; import com.intellij.psi.StringEscapesTokenTypes; +import com.intellij.psi.tree.IElementType; + +import java.util.EnumSet; +import java.util.LinkedList; // IDEADEV-11055 @SuppressWarnings({ "ALL", "SameParameterValue", "WeakerAccess", "SameReturnValue", "RedundantThrows", "UnusedDeclaration", "UnusedDeclaration" }) @@ -15,8 +16,8 @@ import com.intellij.psi.StringEscapesTokenTypes; /** * This class is a scanner generated by * JFlex 1.4.3 - * on 21.06.11 21:06 from the specification file - * C:/IDEA/tools/lexer/../../community/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex + * on 10/1/12 11:19 AM from the specification file + * C:/work/IDEA/tools/lexer/../../community/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex */ class _RegExLexer implements FlexLexer { /** initial size of the lookahead buffer */ @@ -680,106 +681,6 @@ class _RegExLexer implements FlexLexer { { return RegExpTT.STAR; } case 81: break; - case 58: - { yybegin(PY_COND_REF); return RegExpTT.PYTHON_COND_REF; - } - case 82: break; - case 48: - { return RegExpTT.CHAR_CLASS; - } - case 83: break; - case 35: - { handleOptions(); return RegExpTT.OPTIONS_OFF; - } - case 84: break; - case 54: - { yypopstate(); return RegExpTT.QUOTE_END; - } - case 85: break; - case 34: - { yybegin(YYINITIAL); return RegExpTT.GROUP_END; - } - case 86: break; - case 5: - { return RegExpTT.GROUP_BEGIN; - } - case 87: break; - case 36: - { yybegin(YYINITIAL); return RegExpTT.COLON; - } - case 88: break; - case 6: - { return RegExpTT.GROUP_END; - } - case 89: break; - case 25: - { assert false : yytext(); - } - case 90: break; - case 18: - { return commentMode ? com.intellij.psi.TokenType.WHITE_SPACE : RegExpTT.CHARACTER; - } - case 91: break; - case 45: - { return commentMode ? RegExpTT.CHARACTER : RegExpTT.REDUNDANT_ESCAPE; - } - case 92: break; - case 23: - { return RegExpTT.NAME; - } - case 93: break; - case 72: - { if (allowOctalNoLeadingZero) return RegExpTT.OCT_CHAR; - return yystate() != CLASS2 ? RegExpTT.BACKREF : RegExpTT.ESC_CHARACTER; - } - case 94: break; - case 29: - { yypopstate(); return RegExpTT.CLASS_END; - } - case 95: break; - case 33: - { yybegin(YYINITIAL); return RegExpTT.BAD_CHARACTER; - } - case 96: break; - case 51: - { yypushstate(QUOTED); return RegExpTT.QUOTE_BEGIN; - } - case 97: break; - case 21: - { return RegExpTT.NUMBER; - } - case 98: break; - case 4: - { return RegExpTT.DOT; - } - case 99: break; - case 70: - { return RegExpTT.NEG_LOOKBEHIND; - } - case 100: break; - case 40: - // lookahead expression with fixed base length - zzMarkedPos = zzStartRead + 1; - { yypushstate(CLASS1); - return RegExpTT.CLASS_BEGIN; - } - case 101: break; - case 61: - { return RegExpTT.NEG_LOOKAHEAD; - } - case 102: break; - case 10: - { return commentMode ? com.intellij.psi.TokenType.WHITE_SPACE : RegExpTT.CTRL_CHARACTER; - } - case 103: break; - case 59: - { return RegExpTT.NON_CAPT_GROUP; - } - case 104: break; - case 63: - { yybegin(QUOTED_NAMED_GROUP); return RegExpTT.RUBY_QUOTED_NAMED_GROUP; - } - case 105: break; case 28: { if (allowNestedCharacterClasses) { yypushstate(CLASS2); @@ -787,6 +688,105 @@ class _RegExLexer implements FlexLexer { } return RegExpTT.CHARACTER; } + case 82: break; + case 58: + { yybegin(PY_COND_REF); return RegExpTT.PYTHON_COND_REF; + } + case 83: break; + case 48: + { return RegExpTT.CHAR_CLASS; + } + case 84: break; + case 35: + { handleOptions(); return RegExpTT.OPTIONS_OFF; + } + case 85: break; + case 40: + // lookahead expression with fixed base length + zzMarkedPos = zzStartRead + 1; + { yypushstate(CLASS1); + return RegExpTT.CLASS_BEGIN; + } + case 86: break; + case 54: + { yypopstate(); return RegExpTT.QUOTE_END; + } + case 87: break; + case 34: + { yybegin(YYINITIAL); return RegExpTT.GROUP_END; + } + case 88: break; + case 5: + { return RegExpTT.GROUP_BEGIN; + } + case 89: break; + case 36: + { yybegin(YYINITIAL); return RegExpTT.COLON; + } + case 90: break; + case 6: + { return RegExpTT.GROUP_END; + } + case 91: break; + case 25: + { assert false : yytext(); + } + case 92: break; + case 18: + { return commentMode ? com.intellij.psi.TokenType.WHITE_SPACE : RegExpTT.CHARACTER; + } + case 93: break; + case 45: + { return commentMode ? RegExpTT.CHARACTER : RegExpTT.REDUNDANT_ESCAPE; + } + case 94: break; + case 23: + { return RegExpTT.NAME; + } + case 95: break; + case 29: + { yypopstate(); return RegExpTT.CLASS_END; + } + case 96: break; + case 33: + { yybegin(YYINITIAL); return RegExpTT.BAD_CHARACTER; + } + case 97: break; + case 51: + { yypushstate(QUOTED); return RegExpTT.QUOTE_BEGIN; + } + case 98: break; + case 21: + { return RegExpTT.NUMBER; + } + case 99: break; + case 4: + { return RegExpTT.DOT; + } + case 100: break; + case 70: + { return RegExpTT.NEG_LOOKBEHIND; + } + case 101: break; + case 43: + { return (allowOctalNoLeadingZero ? RegExpTT.OCT_CHAR : RegExpTT.BAD_OCT_VALUE); + } + case 102: break; + case 61: + { return RegExpTT.NEG_LOOKAHEAD; + } + case 103: break; + case 10: + { return commentMode ? com.intellij.psi.TokenType.WHITE_SPACE : RegExpTT.CTRL_CHARACTER; + } + case 104: break; + case 59: + { return RegExpTT.NON_CAPT_GROUP; + } + case 105: break; + case 63: + { yybegin(QUOTED_NAMED_GROUP); return RegExpTT.RUBY_QUOTED_NAMED_GROUP; + } case 106: break; case 19: { if (commentMode) { yypushstate(COMMENT); return RegExpTT.COMMENT; } else return RegExpTT.CHARACTER; @@ -796,75 +796,51 @@ class _RegExLexer implements FlexLexer { { yypushstate(PROP); return RegExpTT.PROPERTY; } case 108: break; - case 8: - { yypushstate(CLASS2); - return RegExpTT.CLASS_BEGIN; - } - case 109: break; case 69: { return RegExpTT.POS_LOOKBEHIND; } - case 110: break; + case 109: break; case 42: { return yystate() != CLASS2 ? RegExpTT.BACKREF : RegExpTT.ESC_CHARACTER; } - case 111: break; + case 110: break; case 71: { return RegExpTT.HEX_CHAR; } - case 112: break; + case 111: break; case 1: { handleOptions(); return RegExpTT.OPTIONS_ON; } - case 113: break; + case 112: break; case 57: { return allowNestedCharacterClasses ? RegExpTT.ANDAND : RegExpTT.CHARACTER; } - case 114: break; + case 113: break; case 2: { yypopstate(); return RegExpTT.COMMENT; } - case 115: break; + case 114: break; case 17: { return RegExpTT.MINUS; } - case 116: break; + case 115: break; case 46: { return RegExpTT.ESC_CTRL_CHARACTER; } - case 117: break; + case 116: break; case 32: { yypopstate(); yypushstate(EMBRACED); return RegExpTT.LBRACE; } + case 117: break; + case 56: + { if (!allowNestedCharacterClasses) return RegExpTT.CHARACTER; + return RegExpTT.REDUNDANT_ESCAPE; + } case 118: break; case 13: { return RegExpTT.QUEST; } case 119: break; - case 15: - { return RegExpTT.PLUS; - } - case 120: break; - case 65: - { return RegExpTT.OCT_CHAR; - } - case 121: break; - case 38: - { yybegin(YYINITIAL); return RegExpTT.QUOTE; - } - case 122: break; - case 64: - // lookahead expression with fixed base length - zzMarkedPos = zzStartRead + 1; - { if (!allowNestedCharacterClasses) { - yypushstate(CLASS1PY); - } - else { - yypushstate(CLASS2); - } - return RegExpTT.CLASS_BEGIN; - } - case 123: break; case 20: { if (allowDanglingMetacharacters) { yypopstate(); yypushback(1); @@ -872,6 +848,23 @@ class _RegExLexer implements FlexLexer { return RegExpTT.BAD_CHARACTER; } } + case 120: break; + case 15: + { return RegExpTT.PLUS; + } + case 121: break; + case 65: + { return RegExpTT.OCT_CHAR; + } + case 122: break; + case 72: + { if (allowOctalNoLeadingZero) return RegExpTT.OCT_CHAR; + return yystate() != CLASS2 ? RegExpTT.BACKREF : RegExpTT.ESC_CHARACTER; + } + case 123: break; + case 38: + { yybegin(YYINITIAL); return RegExpTT.QUOTE; + } case 124: break; case 31: { yypopstate(); yypushback(1); @@ -881,6 +874,87 @@ class _RegExLexer implements FlexLexer { { yybegin(CLASS1); return RegExpTT.CARET; } case 126: break; + case 37: + { yybegin(YYINITIAL); return RegExpTT.GT; + } + case 127: break; + case 67: + { yybegin(PY_NAMED_GROUP_REF); return RegExpTT.PYTHON_NAMED_GROUP_REF; + } + case 128: break; + case 16: + { return RegExpTT.UNION; + } + case 129: break; + case 3: + { return RegExpTT.CHARACTER; + } + case 130: break; + case 62: + { yybegin(NAMED_GROUP); return RegExpTT.RUBY_NAMED_GROUP; + } + case 131: break; + case 39: + { yybegin(OPTIONS); return RegExpTT.SET_OPTIONS; + } + case 132: break; + case 26: + { yybegin(CLASS2); return RegExpTT.CHARACTER; + } + case 133: break; + case 66: + { if (xmlSchemaMode) { yypushback(1); return RegExpTT.CHAR_CLASS; } else return RegExpTT.CTRL; + } + case 134: break; + case 30: + { return commentMode ? com.intellij.psi.TokenType.WHITE_SPACE : RegExpTT.ESC_CHARACTER; + } + case 135: break; + case 12: + { return RegExpTT.DOLLAR; + } + case 136: break; + case 49: + { if (xmlSchemaMode) return RegExpTT.CHAR_CLASS; else return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; + } + case 137: break; + case 41: + { return RegExpTT.REDUNDANT_ESCAPE; + } + case 138: break; + case 47: + { return yystate() != CLASS2 ? RegExpTT.BOUNDARY : RegExpTT.ESC_CHARACTER; + } + case 139: break; + case 8: + { yypushstate(CLASS2); + return RegExpTT.CLASS_BEGIN; + } + case 140: break; + case 22: + { yypopstate(); return RegExpTT.RBRACE; + } + case 141: break; + case 53: + { return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; + } + case 142: break; + case 52: + { return RegExpTT.BAD_HEX_VALUE; + } + case 143: break; + case 9: + { return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; + } + case 144: break; + case 11: + { return RegExpTT.CARET; + } + case 145: break; + case 44: + { return RegExpTT.ESC_CHARACTER; + } + case 146: break; case 55: // lookahead expression with fixed base length zzMarkedPos = zzStartRead + 1; @@ -890,90 +964,17 @@ class _RegExLexer implements FlexLexer { } return RegExpTT.CHARACTER; } - case 127: break; - case 56: - { if (!allowNestedCharacterClasses) return RegExpTT.CHARACTER; - return RegExpTT.REDUNDANT_ESCAPE; - } - case 128: break; - case 37: - { yybegin(YYINITIAL); return RegExpTT.GT; - } - case 129: break; - case 67: - { yybegin(PY_NAMED_GROUP_REF); return RegExpTT.PYTHON_NAMED_GROUP_REF; - } - case 130: break; - case 16: - { return RegExpTT.UNION; - } - case 131: break; - case 3: - { return RegExpTT.CHARACTER; - } - case 132: break; - case 62: - { yybegin(NAMED_GROUP); return RegExpTT.RUBY_NAMED_GROUP; - } - case 133: break; - case 39: - { yybegin(OPTIONS); return RegExpTT.SET_OPTIONS; - } - case 134: break; - case 26: - { yybegin(CLASS2); return RegExpTT.CHARACTER; - } - case 135: break; - case 66: - { if (xmlSchemaMode) { yypushback(1); return RegExpTT.CHAR_CLASS; } else return RegExpTT.CTRL; - } - case 136: break; - case 30: - { return commentMode ? com.intellij.psi.TokenType.WHITE_SPACE : RegExpTT.ESC_CHARACTER; - } - case 137: break; - case 12: - { return RegExpTT.DOLLAR; - } - case 138: break; - case 49: - { if (xmlSchemaMode) return RegExpTT.CHAR_CLASS; else return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; - } - case 139: break; - case 41: - { return RegExpTT.REDUNDANT_ESCAPE; - } - case 140: break; - case 47: - { return yystate() != CLASS2 ? RegExpTT.BOUNDARY : RegExpTT.ESC_CHARACTER; - } - case 141: break; - case 22: - { yypopstate(); return RegExpTT.RBRACE; - } - case 142: break; - case 53: - { return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN; - } - case 143: break; - case 43: - { return RegExpTT.BAD_OCT_VALUE; - } - case 144: break; - case 52: - { return RegExpTT.BAD_HEX_VALUE; - } - case 145: break; - case 9: - { return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; - } - case 146: break; - case 11: - { return RegExpTT.CARET; - } case 147: break; - case 44: - { return RegExpTT.ESC_CHARACTER; + case 64: + // lookahead expression with fixed base length + zzMarkedPos = zzStartRead + 1; + { if (!allowNestedCharacterClasses) { + yypushstate(CLASS1PY); + } + else { + yypushstate(CLASS2); + } + return RegExpTT.CLASS_BEGIN; } case 148: break; default: diff --git a/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex b/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex index 2dc4f76afa46..e3d403c369ce 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex +++ b/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex @@ -123,7 +123,7 @@ HEX_CHAR=[0-9a-fA-F] /* octal escapes */ {ESCAPE} "0" [0-7]{1,3} { return RegExpTT.OCT_CHAR; } -{ESCAPE} "0" { return RegExpTT.BAD_OCT_VALUE; } +{ESCAPE} "0" { return (allowOctalNoLeadingZero ? RegExpTT.OCT_CHAR : RegExpTT.BAD_OCT_VALUE); } /* single character after "\c" */ {ESCAPE} "c" {ANY} { if (xmlSchemaMode) { yypushback(1); return RegExpTT.CHAR_CLASS; } else return RegExpTT.CTRL; }