mirror of
https://gitflic.ru/project/openide/openide.git
synced 2025-12-16 22:51:17 +07:00
749 lines
36 KiB
Java
749 lines
36 KiB
Java
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
|
package org.intellij.lang.regexp;
|
|
|
|
import com.intellij.lexer.Lexer;
|
|
import com.intellij.testFramework.LexerTestCase;
|
|
|
|
import java.io.File;
|
|
import java.util.EnumSet;
|
|
|
|
import static org.intellij.lang.regexp.RegExpCapability.*;
|
|
|
|
/**
|
|
* @author Bas Leijdekkers
|
|
*/
|
|
public class RegExpLexerTest extends LexerTestCase {
|
|
|
|
public void testAtomicGroup() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("(?>atom)", "ATOMIC_GROUP ('(?>')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"CHARACTER ('t')\n" +
|
|
"CHARACTER ('o')\n" +
|
|
"CHARACTER ('m')\n" +
|
|
"GROUP_END (')')", lexer);
|
|
doTest("(?:no)", "NON_CAPT_GROUP ('(?:')\n" +
|
|
"CHARACTER ('n')\n" +
|
|
"CHARACTER ('o')\n" +
|
|
"GROUP_END (')')", lexer);
|
|
}
|
|
|
|
public void testAmpersand() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("[a&&]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"CHARACTER ('&')\n" +
|
|
"CHARACTER ('&')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testQE() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("\\Q\r\n\\E", "QUOTE_BEGIN ('\\Q')\n" +
|
|
"CHARACTER ('\n')\n" +
|
|
"CHARACTER ('\\n')\n" +
|
|
"QUOTE_END ('\\E')", lexer);
|
|
}
|
|
|
|
public void testEditorReplacement() {
|
|
RegExpLexer lexer = new RegExpLexer(EnumSet.of(TRANSFORMATION_ESCAPES));
|
|
final String text = "\\U$1\\E\\u$3\\l$4\\L$2\\E";
|
|
doTest(text, "CHAR_CLASS ('\\U')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"CHAR_CLASS ('\\E')\n" +
|
|
"CHAR_CLASS ('\\u')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"CHAR_CLASS ('\\l')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('4')\n" +
|
|
"CHAR_CLASS ('\\L')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"CHAR_CLASS ('\\E')", lexer);
|
|
|
|
lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
|
|
doTest(text, "INVALID_CHARACTER_ESCAPE_TOKEN ('\\U')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\E')\n" +
|
|
"INVALID_UNICODE_ESCAPE_TOKEN ('\\u')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\l')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('4')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\L')\n" +
|
|
"DOLLAR ('$')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\E')", lexer);
|
|
}
|
|
|
|
public void testIntersection() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(NESTED_CHARACTER_CLASSES));
|
|
doTest("[a&&]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"ANDAND ('&&')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testCarets() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("^\\^[^^]", "CARET ('^')\n" +
|
|
"ESC_CHARACTER ('\\^')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"CARET ('^')\n" +
|
|
"CHARACTER ('^')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testPosixBracketExpression() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(POSIX_BRACKET_EXPRESSIONS));
|
|
doTest("[[:xdigit:]]", "CLASS_BEGIN ('[')\n" +
|
|
"BRACKET_EXPRESSION_BEGIN ('[:')\n" +
|
|
"NAME ('xdigit')\n" +
|
|
"BRACKET_EXPRESSION_END (':]')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testNegatedPosixBracketExpression() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(POSIX_BRACKET_EXPRESSIONS));
|
|
doTest("[[:^xdigit:]]", "CLASS_BEGIN ('[')\n" +
|
|
"BRACKET_EXPRESSION_BEGIN ('[:')\n" +
|
|
"CARET ('^')\n" +
|
|
"NAME ('xdigit')\n" +
|
|
"BRACKET_EXPRESSION_END (':]')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testMysqlCharExpressions() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(MYSQL_BRACKET_EXPRESSIONS));
|
|
doTest("[[.~.][.tilda.][.NUL.][.plus-sign.]]", "CLASS_BEGIN ('[')\n" +
|
|
"MYSQL_CHAR_BEGIN ('[.')\nCHARACTER ('~')\nMYSQL_CHAR_END ('.]')\n" +
|
|
"MYSQL_CHAR_BEGIN ('[.')\nNAME ('tilda')\nMYSQL_CHAR_END ('.]')\n" +
|
|
"MYSQL_CHAR_BEGIN ('[.')\nNAME ('NUL')\nMYSQL_CHAR_END ('.]')\n" +
|
|
"MYSQL_CHAR_BEGIN ('[.')\nNAME ('plus-sign')\nMYSQL_CHAR_END ('.]')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testMysqlCharEqExpressions() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(MYSQL_BRACKET_EXPRESSIONS));
|
|
doTest("[[=.=][=c=]]", "CLASS_BEGIN ('[')\n" +
|
|
"MYSQL_CHAR_EQ_BEGIN ('[=')\nCHARACTER ('.')\nMYSQL_CHAR_EQ_END ('=]')\n" +
|
|
"MYSQL_CHAR_EQ_BEGIN ('[=')\nCHARACTER ('c')\nMYSQL_CHAR_EQ_END ('=]')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
/**
|
|
* \\177 is the maximum valid octal character under Ruby.
|
|
*/
|
|
public void testMaxOctalNoLeadingZero1() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO, MAX_OCTAL_177));
|
|
doTest("\\177\\200", "OCT_CHAR ('\\177')\n" +
|
|
"BAD_OCT_VALUE ('\\20')\n" +
|
|
"CHARACTER ('0')", lexer);
|
|
}
|
|
|
|
/**
|
|
* \\377 is the maximum valid octal character under javascript. \\400 is interpreted as \\40 followed by a 0 character.
|
|
* The BAD_OCT_VALUE token is converted to OCT_CHAR in com.intellij.lang.javascript.inject.JSRegexpParserDefinition
|
|
*/
|
|
public void testMaxOctalNoLeadingZero2() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO, MAX_OCTAL_377));
|
|
doTest("\\177\\200\\377\\400", "OCT_CHAR ('\\177')\n" +
|
|
"OCT_CHAR ('\\200')\n" +
|
|
"OCT_CHAR ('\\377')\n" +
|
|
"BAD_OCT_VALUE ('\\40')\n" +
|
|
"CHARACTER ('0')", lexer);
|
|
}
|
|
|
|
/**
|
|
* \\777 is valid octal character in python regex dialect.
|
|
*/
|
|
public void testMaxOctalNoLeadingZero3() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO));
|
|
doTest("\\177\\200\\377\\400\\777", "OCT_CHAR ('\\177')\n" +
|
|
"OCT_CHAR ('\\200')\n" +
|
|
"OCT_CHAR ('\\377')\n" +
|
|
"OCT_CHAR ('\\400')\n" +
|
|
"OCT_CHAR ('\\777')", lexer);
|
|
}
|
|
|
|
/**
|
|
* \\1 and \\11 valid under js, both inside and outside character class
|
|
*/
|
|
public void testOctalNoLeadingZero1() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO));
|
|
doTest("\\1()\\1\\11[\\1\\11]", "OCT_CHAR ('\\1')\n" +
|
|
"GROUP_BEGIN ('(')\n" +
|
|
"GROUP_END (')')\n" +
|
|
"BACKREF ('\\1')\n" +
|
|
"OCT_CHAR ('\\11')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"OCT_CHAR ('\\1')\n" +
|
|
"OCT_CHAR ('\\11')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
/**
|
|
* \\1 not valid and \\11 valid under ruby, outside character class
|
|
*/
|
|
public void testOctalNoLeadingZero2() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO, MIN_OCTAL_2_DIGITS));
|
|
doTest("\\1()\\1\\11[\\1\\11]", "BAD_OCT_VALUE ('\\1')\n" +
|
|
"GROUP_BEGIN ('(')\n" +
|
|
"GROUP_END (')')\n" +
|
|
"BACKREF ('\\1')\n" +
|
|
"OCT_CHAR ('\\11')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"OCT_CHAR ('\\1')\n" +
|
|
"OCT_CHAR ('\\11')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
/**
|
|
* \\1 and \\11 not valid under python, outside character class
|
|
*/
|
|
public void testOctalNoLeadingZero3() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO, MIN_OCTAL_3_DIGITS));
|
|
doTest("\\1()\\1\\11\\111[\\1\\11\\111]", "BAD_OCT_VALUE ('\\1')\n" +
|
|
"GROUP_BEGIN ('(')\n" +
|
|
"GROUP_END (')')\n" +
|
|
"BACKREF ('\\1')\n" +
|
|
"BAD_OCT_VALUE ('\\11')\n" +
|
|
"OCT_CHAR ('\\111')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"OCT_CHAR ('\\1')\n" +
|
|
"OCT_CHAR ('\\11')\n" +
|
|
"OCT_CHAR ('\\111')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
|
|
/** octal is never a back reference inside a character class, valid under js, ruby, python */
|
|
public void testOctalInsideCharClass() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO));
|
|
doTest("()()()()()()()()()()[\\1\\10\\100]", "GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"GROUP_BEGIN ('(')\nGROUP_END (')')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"OCT_CHAR ('\\1')\n" +
|
|
"OCT_CHAR ('\\10')\n" +
|
|
"OCT_CHAR ('\\100')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
/** \0 always valid under js, ruby, python regex dialects, never a back reference. */
|
|
public void testZeroOctalNoLeadingZero() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO));
|
|
doTest("\\0()\\0[\\0]", "OCT_CHAR ('\\0')\n" +
|
|
"GROUP_BEGIN ('(')\n" +
|
|
"GROUP_END (')')\n" +
|
|
"OCT_CHAR ('\\0')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"OCT_CHAR ('\\0')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
/** three digit octal (\100) always valid, either octal or backreference under js, ruby and python */
|
|
public void testThreeDigitOctalNoLeadingZero() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO));
|
|
doTest("\\100" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()" +
|
|
"()()()()()()()()()()\\100[\\100]", null, lexer);
|
|
}
|
|
|
|
public void testOctalFollowedByDigit() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(OCTAL_NO_LEADING_ZERO));
|
|
doTest("\\39[\\39]", "OCT_CHAR ('\\3')\n" +
|
|
"CHARACTER ('9')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"OCT_CHAR ('\\3')\n" +
|
|
"CHARACTER ('9')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testOctalWithLeadingZero() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("\\0\\123[\\123]", "BAD_OCT_VALUE ('\\0')\n" +
|
|
"BACKREF ('\\1')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\1')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testOctalWithLeadingZero2() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("\\08\\01\\00\\012\\0123\\0377\\0400", "BAD_OCT_VALUE ('\\0')\n" +
|
|
"CHARACTER ('8')\n" +
|
|
"OCT_CHAR ('\\01')\n" +
|
|
"OCT_CHAR ('\\00')\n" +
|
|
"OCT_CHAR ('\\012')\n" +
|
|
"OCT_CHAR ('\\0123')\n" +
|
|
"OCT_CHAR ('\\0377')\n" +
|
|
"OCT_CHAR ('\\040')\n" +
|
|
"CHARACTER ('0')", lexer);
|
|
}
|
|
|
|
public void testBackReference() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)\\105", null, lexer);
|
|
}
|
|
|
|
public void testPcreBackReference() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(PCRE_BACK_REFERENCES));
|
|
doTest("(a)\\g105", null, lexer);
|
|
}
|
|
|
|
public void testPcreRelativeBackReference() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(PCRE_BACK_REFERENCES));
|
|
doTest("(a)\\g{105}", null, lexer);
|
|
}
|
|
|
|
public void testPcreRelativeNegativeBackReference() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(PCRE_BACK_REFERENCES));
|
|
doTest("(a)\\g{-105}", null, lexer);
|
|
}
|
|
|
|
public void testPcreRelativeNegativeInvalidBackReference() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(PCRE_BACK_REFERENCES));
|
|
doTest("(a)\\g-105", null, lexer);
|
|
}
|
|
|
|
public void testNoNestedCharacterClasses1() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("[[\\]]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('[')\n" +
|
|
"ESC_CHARACTER ('\\]')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testNoNestedCharacterClasses2() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("[a-z&&[^aeuoi]]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"MINUS ('-')\n" +
|
|
"CHARACTER ('z')\n" +
|
|
"CHARACTER ('&')\n" +
|
|
"CHARACTER ('&')\n" +
|
|
"CHARACTER ('[')\n" +
|
|
"CHARACTER ('^')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"CHARACTER ('e')\n" +
|
|
"CHARACTER ('u')\n" +
|
|
"CHARACTER ('o')\n" +
|
|
"CHARACTER ('i')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CHARACTER (']')", lexer);
|
|
}
|
|
|
|
public void testNestedCharacterClasses1() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(NESTED_CHARACTER_CLASSES));
|
|
doTest("[a-z&&[^aeuoi]]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"MINUS ('-')\n" +
|
|
"CHARACTER ('z')\n" +
|
|
"ANDAND ('&&')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"CARET ('^')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"CHARACTER ('e')\n" +
|
|
"CHARACTER ('u')\n" +
|
|
"CHARACTER ('o')\n" +
|
|
"CHARACTER ('i')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testNestedCharacterClasses2() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(NESTED_CHARACTER_CLASSES));
|
|
doTest("[]]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER (']')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
doTest("[\\]]", "CLASS_BEGIN ('[')\n" +
|
|
"REDUNDANT_ESCAPE ('\\]')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
doTest("[[]]]", "CLASS_BEGIN ('[')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER (']')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
doTest("[ \\]]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER (' ')\n" +
|
|
"ESC_CHARACTER ('\\]')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
doTest("[\\Q\\E]]", "CLASS_BEGIN ('[')\n" +
|
|
"QUOTE_BEGIN ('\\Q')\n" +
|
|
"QUOTE_END ('\\E')\n" +
|
|
"CHARACTER (']')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
doTest("[\\Q+\\E]]", "CLASS_BEGIN ('[')\n" +
|
|
"QUOTE_BEGIN ('\\Q')\n" +
|
|
"CHARACTER ('+')\n" +
|
|
"QUOTE_END ('\\E')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CHARACTER (']')", lexer);
|
|
doTest("[^\\Q\\E]]", "CLASS_BEGIN ('[')\n" +
|
|
"CARET ('^')\n" +
|
|
"QUOTE_BEGIN ('\\Q')\n" +
|
|
"QUOTE_END ('\\E')\n" +
|
|
"CHARACTER (']')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
doTest("[^\\Q+\\E]]", "CLASS_BEGIN ('[')\n" +
|
|
"CARET ('^')\n" +
|
|
"QUOTE_BEGIN ('\\Q')\n" +
|
|
"CHARACTER ('+')\n" +
|
|
"QUOTE_END ('\\E')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CHARACTER (']')", lexer);
|
|
final RegExpLexer lexer2 = new RegExpLexer(EnumSet.of(COMMENT_MODE, WHITESPACE_IN_CLASS));
|
|
doTest("[ \t\n]]", "CLASS_BEGIN ('[')\n" +
|
|
"WHITE_SPACE (' ')\n" +
|
|
"WHITE_SPACE ('\t')\n" +
|
|
"WHITE_SPACE ('\\n')\n" +
|
|
"CHARACTER (']')\n" +
|
|
"CLASS_END (']')", lexer2);
|
|
doTest("[\\ ]", "CLASS_BEGIN ('[')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\ ')\n" +
|
|
"CLASS_END (']')", lexer2);
|
|
doTest("[#comment\nabc]", "CLASS_BEGIN ('[')\n" +
|
|
"COMMENT ('#comment')\n" +
|
|
"WHITE_SPACE ('\\n')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"CHARACTER ('b')\n" +
|
|
"CHARACTER ('c')\n" +
|
|
"CLASS_END (']')", lexer2);
|
|
doTest("[ ^]]", "CLASS_BEGIN ('[')\n" +
|
|
"WHITE_SPACE (' ')\n" +
|
|
"CHARACTER ('^')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CHARACTER (']')", lexer2);
|
|
doTest("[\\", "CLASS_BEGIN ('[')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\')", lexer2);
|
|
final RegExpLexer lexer3 = new RegExpLexer(EnumSet.of(ALLOW_EMPTY_CHARACTER_CLASS));
|
|
doTest("[]]", "CLASS_BEGIN ('[')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CHARACTER (']')", lexer3);
|
|
doTest("[[]]]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('[')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"CHARACTER (']')\n" +
|
|
"CHARACTER (']')", lexer3);
|
|
doTest("[\\]]", "CLASS_BEGIN ('[')\n" +
|
|
"ESC_CHARACTER ('\\]')\n" +
|
|
"CLASS_END (']')", lexer3);
|
|
doTest("[ \\]]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER (' ')\n" +
|
|
"ESC_CHARACTER ('\\]')\n" +
|
|
"CLASS_END (']')", lexer3);
|
|
final RegExpLexer lexer4 = new RegExpLexer(EnumSet.of(COMMENT_MODE));
|
|
doTest("[ ]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER (' ')\n" +
|
|
"CLASS_END (']')", lexer4);
|
|
doTest("[#]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('#')\n" +
|
|
"CLASS_END (']')", lexer4);
|
|
doTest("[\\ ]", "CLASS_BEGIN ('[')\n" +
|
|
"REDUNDANT_ESCAPE ('\\ ')\n" +
|
|
"CLASS_END (']')", lexer4);
|
|
}
|
|
|
|
public void testBoundaries() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("\\b\\b{g}\\B\\A\\z\\Z\\G[\\b\\b{g}\\B\\A\\z\\Z\\G]", "BOUNDARY ('\\b')\n" +
|
|
"BOUNDARY ('\\b{g}')\n" +
|
|
"BOUNDARY ('\\B')\n" +
|
|
"BOUNDARY ('\\A')\n" +
|
|
"BOUNDARY ('\\z')\n" +
|
|
"BOUNDARY ('\\Z')\n" +
|
|
"BOUNDARY ('\\G')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\b')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\b')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('g')\n" +
|
|
"CHARACTER ('}')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\B')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\A')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\z')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\Z')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\G')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testValidEscapes() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("\\%\\ä", "REDUNDANT_ESCAPE ('\\%')\n" +
|
|
"REDUNDANT_ESCAPE ('\\ä')", lexer);
|
|
|
|
RegExpLexer lexer2 = new RegExpLexer(EnumSet.of(DANGLING_METACHARACTERS));
|
|
doTest("{\\}{33,34\\}{1\\}{1,\\}{,\\}{,2\\}", "CHARACTER ('{')\n" +
|
|
"REDUNDANT_ESCAPE ('\\}')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"CHARACTER ('4')\n" +
|
|
"ESC_CHARACTER ('\\}')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"ESC_CHARACTER ('\\}')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"ESC_CHARACTER ('\\}')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"REDUNDANT_ESCAPE ('\\}')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"REDUNDANT_ESCAPE ('\\}')", lexer2);
|
|
|
|
RegExpLexer lexer3 = new RegExpLexer(EnumSet.of(DANGLING_METACHARACTERS, OMIT_NUMBERS_IN_QUANTIFIERS, OMIT_BOTH_NUMBERS_IN_QUANTIFIERS));
|
|
doTest("{\\}{,\\}{,2\\}", "CHARACTER ('{')\n" +
|
|
"REDUNDANT_ESCAPE ('\\}')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"ESC_CHARACTER ('\\}')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"ESC_CHARACTER ('\\}')", lexer3);
|
|
}
|
|
|
|
public void testEscapesInsideCharClass() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("[\\k<a> (?<t>t)\\g'q'\\R]", "CLASS_BEGIN ('[')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\k')\n" +
|
|
"CHARACTER ('<')\n" +
|
|
"CHARACTER ('a')\n" +
|
|
"CHARACTER ('>')\n" +
|
|
"CHARACTER (' ')\n" +
|
|
"CHARACTER ('(')\n" +
|
|
"CHARACTER ('?')\n" +
|
|
"CHARACTER ('<')\n" +
|
|
"CHARACTER ('t')\n" +
|
|
"CHARACTER ('>')\n" +
|
|
"CHARACTER ('t')\n" +
|
|
"CHARACTER (')')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\g')\n" +
|
|
"CHARACTER (''')\n" +
|
|
"CHARACTER ('q')\n" +
|
|
"CHARACTER (''')\n" +
|
|
"INVALID_CHARACTER_ESCAPE_TOKEN ('\\R')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
|
|
doTest("\\{\\*\\+\\?\\$[\\{\\*\\+\\?\\$]", "ESC_CHARACTER ('\\{')\n" +
|
|
"ESC_CHARACTER ('\\*')\n" +
|
|
"ESC_CHARACTER ('\\+')\n" +
|
|
"ESC_CHARACTER ('\\?')\n" +
|
|
"ESC_CHARACTER ('\\$')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"REDUNDANT_ESCAPE ('\\{')\n" +
|
|
"REDUNDANT_ESCAPE ('\\*')\n" +
|
|
"REDUNDANT_ESCAPE ('\\+')\n" +
|
|
"REDUNDANT_ESCAPE ('\\?')\n" +
|
|
"REDUNDANT_ESCAPE ('\\$')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testUnicode() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(EXTENDED_UNICODE_CHARACTER));
|
|
doTest("\\u{1F680}\\x{1F680}\\u{}\\u{1}\\u{FF}\\x{fff}\\u1234\\u123\\u", "UNICODE_CHAR ('\\u{1F680}')\n" +
|
|
"HEX_CHAR ('\\x{1F680}')\n" +
|
|
"INVALID_UNICODE_ESCAPE_TOKEN ('\\u{}')\n" +
|
|
"UNICODE_CHAR ('\\u{1}')\n" +
|
|
"UNICODE_CHAR ('\\u{FF}')\n" +
|
|
"HEX_CHAR ('\\x{fff}')\n" +
|
|
"UNICODE_CHAR ('\\u1234')\n" +
|
|
"INVALID_UNICODE_ESCAPE_TOKEN ('\\u')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"CHARACTER ('3')\n" +
|
|
"INVALID_UNICODE_ESCAPE_TOKEN ('\\u')", lexer);
|
|
final RegExpLexer lexer2 = new RegExpLexer(EnumSet.of(DANGLING_METACHARACTERS));
|
|
doTest("\\u{1F680}", "INVALID_UNICODE_ESCAPE_TOKEN ('\\u')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"CHARACTER ('F')\n" +
|
|
"CHARACTER ('6')\n" +
|
|
"CHARACTER ('8')\n" +
|
|
"CHARACTER ('0')\n" +
|
|
"CHARACTER ('}')", lexer2);
|
|
}
|
|
|
|
public void testHexChar() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(ONE_HEX_CHAR_ESCAPE));
|
|
doTest("\\x\\x1\\x01", "BAD_HEX_VALUE ('\\x')\n" +
|
|
"HEX_CHAR ('\\x1')\n" +
|
|
"HEX_CHAR ('\\x01')", lexer);
|
|
final RegExpLexer lexer2 = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("\\x\\x1\\x01", "BAD_HEX_VALUE ('\\x')\n" +
|
|
"BAD_HEX_VALUE ('\\x')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"HEX_CHAR ('\\x01')", lexer2);
|
|
}
|
|
|
|
public void testQuantifier() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(DANGLING_METACHARACTERS, OMIT_NUMBERS_IN_QUANTIFIERS));
|
|
doTest("a{,10}", "CHARACTER ('a')\n" +
|
|
"LBRACE ('{')\n" +
|
|
"COMMA (',')\n" +
|
|
"NUMBER ('10')\n" +
|
|
"RBRACE ('}')", lexer);
|
|
|
|
doTest("a{10,}", "CHARACTER ('a')\n" +
|
|
"LBRACE ('{')\n" +
|
|
"NUMBER ('10')\n" +
|
|
"COMMA (',')\n" +
|
|
"RBRACE ('}')", lexer);
|
|
|
|
doTest("a{", "CHARACTER ('a')\n" +
|
|
"CHARACTER ('{')", lexer);
|
|
|
|
doTest("a{1", "CHARACTER ('a')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('1')", lexer);
|
|
|
|
doTest("a{1,", "CHARACTER ('a')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"CHARACTER (',')", lexer);
|
|
|
|
doTest("a{,,}", "CHARACTER ('a')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('}')", lexer);
|
|
|
|
doTest("[{1,2}]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('2')\n" +
|
|
"CHARACTER ('}')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
|
|
doTest("x\\{9}", "CHARACTER ('x')\n" +
|
|
"ESC_CHARACTER ('\\{')\n" +
|
|
"CHARACTER ('9')\n" +
|
|
"CHARACTER ('}')", lexer);
|
|
|
|
doTest("[x\\{9}]", "CLASS_BEGIN ('[')\n" +
|
|
"CHARACTER ('x')\n" +
|
|
"REDUNDANT_ESCAPE ('\\{')\n" +
|
|
"CHARACTER ('9')\n" +
|
|
"CHARACTER ('}')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
|
|
doTest("x\\{}", "CHARACTER ('x')\n" +
|
|
"REDUNDANT_ESCAPE ('\\{')\n" +
|
|
"CHARACTER ('}')", lexer);
|
|
|
|
doTest("x{,}", "CHARACTER ('x')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('}')", lexer);
|
|
|
|
doTest("x\\{,}", "CHARACTER ('x')\n" +
|
|
"REDUNDANT_ESCAPE ('\\{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('}')", lexer);
|
|
}
|
|
|
|
public void testQuantifier2() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(DANGLING_METACHARACTERS));
|
|
doTest("a{,10}", "CHARACTER ('a')\n" +
|
|
"CHARACTER ('{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('1')\n" +
|
|
"CHARACTER ('0')\n" +
|
|
"CHARACTER ('}')", lexer);
|
|
}
|
|
|
|
public void testQuantifier3() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.of(DANGLING_METACHARACTERS, OMIT_NUMBERS_IN_QUANTIFIERS,
|
|
OMIT_BOTH_NUMBERS_IN_QUANTIFIERS));
|
|
doTest("a{,}", "CHARACTER ('a')\n" +
|
|
"LBRACE ('{')\n" +
|
|
"COMMA (',')\n" +
|
|
"RBRACE ('}')", lexer);
|
|
|
|
doTest("x\\{,}", "CHARACTER ('x')\n" +
|
|
"ESC_CHARACTER ('\\{')\n" +
|
|
"CHARACTER (',')\n" +
|
|
"CHARACTER ('}')", lexer);
|
|
}
|
|
|
|
public void testControlCharacters() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("\\n\\b\\t\\r\\f[\\n\\b\\t\\r\\f]", "ESC_CTRL_CHARACTER ('\\n')\n" +
|
|
"BOUNDARY ('\\b')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\t')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\r')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\f')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\n')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\b')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\t')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\r')\n" +
|
|
"ESC_CTRL_CHARACTER ('\\f')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
doTest("\n\t\r\f[\n\t\r\f]", "CTRL_CHARACTER ('\\n')\n" +
|
|
"CTRL_CHARACTER ('\t')\n" +
|
|
"CTRL_CHARACTER ('\n')\n" +
|
|
"CTRL_CHARACTER ('\f')\n" +
|
|
"CLASS_BEGIN ('[')\n" +
|
|
"CTRL_CHARACTER ('\\n')\n" +
|
|
"CTRL_CHARACTER ('\t')\n" +
|
|
"CTRL_CHARACTER ('\n')\n" +
|
|
"CTRL_CHARACTER ('\f')\n" +
|
|
"CLASS_END (']')", lexer);
|
|
}
|
|
|
|
public void testCaret() {
|
|
final RegExpLexer lexer = new RegExpLexer(EnumSet.noneOf(RegExpCapability.class));
|
|
doTest("[\\^\\^]\\^", "CLASS_BEGIN ('[')\n" +
|
|
"ESC_CHARACTER ('\\^')\n" +
|
|
"REDUNDANT_ESCAPE ('\\^')\n" +
|
|
"CLASS_END (']')\n" +
|
|
"ESC_CHARACTER ('\\^')", lexer);
|
|
}
|
|
|
|
@Override
|
|
protected Lexer createLexer() {
|
|
return null;
|
|
}
|
|
|
|
@Override
|
|
protected String getDirPath() {
|
|
if (new File(getHomePath(), "community/RegExpSupport").isDirectory()) {
|
|
return "/community/RegExpSupport/testData/lexer";
|
|
}
|
|
return "/RegExpSupport/testData/lexer";
|
|
}
|
|
}
|