From 85ced7fef505894ceb492d006aee42bdf7f3564a Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Fri, 18 Dec 2009 20:40:31 +0300 Subject: [PATCH] allow dangling lbrace in Ruby 1.9 regular expressions (RUBY-5505) --- .../org/intellij/lang/regexp/RegExpFile.java | 5 +- .../org/intellij/lang/regexp/RegExpLexer.java | 6 +- .../intellij/lang/regexp/RegExpParser.java | 25 +++++-- .../lang/regexp/RegExpParserDefinition.java | 8 +- .../org/intellij/lang/regexp/_RegExLexer.java | 73 ++++++++++--------- .../intellij/lang/regexp/regexp-lexer.flex | 14 +++- 6 files changed, 78 insertions(+), 53 deletions(-) diff --git a/RegExpSupport/src/org/intellij/lang/regexp/RegExpFile.java b/RegExpSupport/src/org/intellij/lang/regexp/RegExpFile.java index 9b5afbc2877c..298e1c663a34 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/RegExpFile.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/RegExpFile.java @@ -16,14 +16,15 @@ package org.intellij.lang.regexp; import com.intellij.extapi.psi.PsiFileBase; +import com.intellij.lang.Language; import com.intellij.openapi.fileTypes.FileType; import com.intellij.psi.FileViewProvider; import org.jetbrains.annotations.NotNull; public class RegExpFile extends PsiFileBase { - public RegExpFile(FileViewProvider viewProvider) { - super(viewProvider, RegExpLanguage.INSTANCE); + public RegExpFile(FileViewProvider viewProvider, final Language language) { + super(viewProvider, language); } @NotNull diff --git a/RegExpSupport/src/org/intellij/lang/regexp/RegExpLexer.java b/RegExpSupport/src/org/intellij/lang/regexp/RegExpLexer.java index 03773f7d4f7f..82642e336963 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/RegExpLexer.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/RegExpLexer.java @@ -17,12 +17,12 @@ package org.intellij.lang.regexp; import com.intellij.lexer.FlexAdapter; -class RegExpLexer extends FlexAdapter { +public class RegExpLexer extends FlexAdapter { private static final int COMMENT_MODE = 1 << 14; - public RegExpLexer(boolean xmlSchemaMode) { - super(new _RegExLexer(xmlSchemaMode)); + public RegExpLexer(boolean xmlSchemaMode, boolean allowDanglingMetacharacters) { + super(new _RegExLexer(xmlSchemaMode, allowDanglingMetacharacters)); } public void start(CharSequence buffer, int startOffset, int endOffset, int initialState) { diff --git a/RegExpSupport/src/org/intellij/lang/regexp/RegExpParser.java b/RegExpSupport/src/org/intellij/lang/regexp/RegExpParser.java index 4785d1759251..b8497416ba6b 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/RegExpParser.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/RegExpParser.java @@ -18,13 +18,19 @@ package org.intellij.lang.regexp; import com.intellij.lang.ASTNode; import com.intellij.lang.PsiBuilder; import com.intellij.lang.PsiParser; -import com.intellij.psi.tree.IElementType; import com.intellij.psi.StringEscapesTokenTypes; +import com.intellij.psi.tree.IElementType; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @SuppressWarnings({ "RedundantIfStatement" }) -class RegExpParser implements PsiParser { +public class RegExpParser implements PsiParser { + private boolean myAllowDanglingMetacharacters; + + public void setAllowDanglingMetacharacters(boolean allowDanglingMetacharacters) { + myAllowDanglingMetacharacters = allowDanglingMetacharacters; + } + @NotNull public ASTNode parse(IElementType root, PsiBuilder builder) { // builder.setDebugMode(true); @@ -123,6 +129,10 @@ class RegExpParser implements PsiParser { if (builder.getTokenType() == RegExpTT.LBRACE) { builder.advanceLexer(); + if (builder.getTokenType() != RegExpTT.NUMBER && myAllowDanglingMetacharacters) { + marker.done(RegExpTT.CHARACTER); + return true; + } checkMatches(builder, RegExpTT.NUMBER, "Number expected"); if (builder.getTokenType() == RegExpTT.RBRACE) { builder.advanceLexer(); @@ -157,7 +167,7 @@ class RegExpParser implements PsiParser { return true; } - private void parseQuantifierType(PsiBuilder builder) { + private static void parseQuantifierType(PsiBuilder builder) { if (builder.getTokenType() == RegExpTT.PLUS) { builder.advanceLexer(); } else if (builder.getTokenType() == RegExpTT.QUEST) { @@ -344,14 +354,19 @@ class RegExpParser implements PsiParser { } else if (type == RegExpTT.CLASS_BEGIN) { marker.drop(); return parseClass(builder); - } else { + } + else if (type == RegExpTT.LBRACE && myAllowDanglingMetacharacters) { + builder.advanceLexer(); + marker.done(RegExpElementTypes.CHAR); + } + else { marker.drop(); return null; } return marker; } - private void parseProperty(PsiBuilder builder) { + private static void parseProperty(PsiBuilder builder) { checkMatches(builder, RegExpTT.PROPERTY, "'\\p' expected"); checkMatches(builder, RegExpTT.LBRACE, "Character category expected"); diff --git a/RegExpSupport/src/org/intellij/lang/regexp/RegExpParserDefinition.java b/RegExpSupport/src/org/intellij/lang/regexp/RegExpParserDefinition.java index 5ff73845191e..95f9100736f1 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/RegExpParserDefinition.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/RegExpParserDefinition.java @@ -34,9 +34,9 @@ import org.jetbrains.annotations.NotNull; public class RegExpParserDefinition implements ParserDefinition { private static final TokenSet COMMENT_TOKENS = TokenSet.create(RegExpTT.COMMENT); - @NotNull + @NotNull public Lexer createLexer(Project project) { - return new RegExpLexer(false); + return new RegExpLexer(false, false); } public PsiParser createParser(Project project) { @@ -86,7 +86,7 @@ public class RegExpParserDefinition implements ParserDefinition { return new RegExpSetOptionsImpl(node); } else if (type == RegExpElementTypes.OPTIONS) { return new RegExpOptionsImpl(node); - } else if (type == RegExpElementTypes.BACKREF) { + } else if (type == RegExpElementTypes.BACKREF) { return new RegExpBackrefImpl(node); } else if (type == RegExpElementTypes.CLOSURE) { return new RegExpClosureImpl(node); @@ -101,7 +101,7 @@ public class RegExpParserDefinition implements ParserDefinition { } public PsiFile createFile(FileViewProvider viewProvider) { - return new RegExpFile(viewProvider); + return new RegExpFile(viewProvider, RegExpLanguage.INSTANCE); } public SpaceRequirements spaceExistanceTypeBetweenTokens(ASTNode left, ASTNode right) { diff --git a/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java b/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java index 89ccdee550b8..8de1c26644c5 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java +++ b/RegExpSupport/src/org/intellij/lang/regexp/_RegExLexer.java @@ -1,26 +1,13 @@ -/* - * Copyright 2000-2009 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +/* The following code was generated by JFlex 1.4.1 on 18.12.09 20:27 */ /* It's an automatically generated code. Do not modify it. */ package org.intellij.lang.regexp; import com.intellij.lexer.FlexLexer; -import com.intellij.psi.tree.IElementType; -import java.util.LinkedList; import com.intellij.psi.StringEscapesTokenTypes; +import com.intellij.psi.tree.IElementType; + +import java.util.LinkedList; // IDEADEV-11055 @SuppressWarnings({ "ALL", "SameParameterValue", "WeakerAccess", "SameReturnValue", "RedundantThrows", "UnusedDeclaration", "UnusedDeclaration" }) @@ -28,8 +15,8 @@ import com.intellij.psi.StringEscapesTokenTypes; /** * This class is a scanner generated by * JFlex 1.4.1 - * on 04.11.09 17:54 from the specification file - * RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex + * on 18.12.09 20:27 from the specification file + * C:/JetBrains/IDEA/tools/lexer/../../community/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex */ class _RegExLexer implements FlexLexer { /** initial size of the lookahead buffer */ @@ -352,9 +339,12 @@ class _RegExLexer implements FlexLexer { // as well, but is currently unfinished as it requires to tweak more places than just the lexer. private boolean xmlSchemaMode; - _RegExLexer(boolean xmlSchemaMode) { + private boolean allowDanglingMetacharacters; + + _RegExLexer(boolean xmlSchemaMode, boolean allowDanglingMetacharacters) { this((java.io.Reader)null); - this.xmlSchemaMode = xmlSchemaMode; + this.xmlSchemaMode = xmlSchemaMode; + this.allowDanglingMetacharacters = allowDanglingMetacharacters; } private void yypushstate(int state) { @@ -431,7 +421,14 @@ class _RegExLexer implements FlexLexer { // For Demetra compatibility public void reset(CharSequence buffer, int initialState){ - reset(buffer, 0, buffer.length(), initialState); + zzBuffer = buffer; + zzBufferArray = null; + zzCurrentPos = zzMarkedPos = zzStartRead = 0; + zzPushbackPos = 0; + zzAtEOF = false; + zzAtBOL = true; + zzEndRead = buffer.length(); + yybegin(initialState); } /** @@ -591,7 +588,7 @@ class _RegExLexer implements FlexLexer { while (true) { if (zzCurrentPosL < zzEndReadL) - zzInput = zzBufferL.charAt(zzCurrentPosL++); + zzInput = zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++]:zzBufferL.charAt(zzCurrentPosL++); else if (zzAtEOF) { zzInput = YYEOF; break zzForAction; @@ -613,7 +610,7 @@ class _RegExLexer implements FlexLexer { break zzForAction; } else { - zzInput = zzBufferL.charAt(zzCurrentPosL++); + zzInput = zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++]:zzBufferL.charAt(zzCurrentPosL++); } } int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ]; @@ -840,45 +837,49 @@ class _RegExLexer implements FlexLexer { { return RegExpTT.HEX_CHAR; } case 111: break; - case 20: - { return RegExpTT.BAD_CHARACTER; - } - case 112: break; case 5: { return RegExpTT.GROUP_BEGIN; } - case 113: break; + case 112: break; case 8: { yypushstate(CLASS2); return RegExpTT.CLASS_BEGIN; } - case 114: break; + case 113: break; case 52: { return RegExpTT.NON_CAPT_GROUP; } - case 115: break; + case 114: break; case 9: { return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; } - case 116: break; + case 115: break; case 48: { return RegExpTT.BAD_HEX_VALUE; } - case 117: break; + case 116: break; case 2: { yypopstate(); return RegExpTT.COMMENT; } - case 118: break; + case 117: break; case 47: { yypushstate(QUOTED); return RegExpTT.QUOTE_BEGIN; } - case 119: break; + case 118: break; case 26: { yybegin(CLASS2); return RegExpTT.CHARACTER; } - case 120: break; + case 119: break; case 15: { return RegExpTT.PLUS; } + case 120: break; + case 20: + { if (allowDanglingMetacharacters) { + yypopstate(); yypushback(1); + } else { + return RegExpTT.BAD_CHARACTER; + } + } case 121: break; case 31: { yybegin(YYINITIAL); return RegExpTT.BAD_CHARACTER; diff --git a/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex b/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex index dcd94c284ed0..621fc89cf6e6 100644 --- a/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex +++ b/RegExpSupport/src/org/intellij/lang/regexp/regexp-lexer.flex @@ -26,9 +26,12 @@ import com.intellij.psi.StringEscapesTokenTypes; // as well, but is currently unfinished as it requires to tweak more places than just the lexer. private boolean xmlSchemaMode; - _RegExLexer(boolean xmlSchemaMode) { + private boolean allowDanglingMetacharacters; + + _RegExLexer(boolean xmlSchemaMode, boolean allowDanglingMetacharacters) { this((java.io.Reader)null); - this.xmlSchemaMode = xmlSchemaMode; + this.xmlSchemaMode = xmlSchemaMode; + this.allowDanglingMetacharacters = allowDanglingMetacharacters; } private void yypushstate(int state) { @@ -157,7 +160,12 @@ HEX_CHAR=[0-9a-fA-F] "," { return RegExpTT.COMMA; } {RBRACE} { yypopstate(); return RegExpTT.RBRACE; } - {ANY} { return RegExpTT.BAD_CHARACTER; } + {ANY} { if (allowDanglingMetacharacters) { + yypopstate(); yypushback(1); + } else { + return RegExpTT.BAD_CHARACTER; + } + } } "-" { return RegExpTT.MINUS; }