MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
Alexander Lobas
2012-03-10 22:46:20 +04:00
parent 9da59c659b d96aafc266
commit f4f67d59cb
6 changed files with 293 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/ide/highlighter/custom/tokens/KeywordParser.java
+16 -3
View File
@@ -47,19 +47,32 @@ public class KeywordParser extends TokenParser {
if (regex.length() > 0) {
regex.append("|");
}
regex.append(word);
regex.append(escapeSpecialCharacters(word));
}
}
Pattern pattern = null;
try {
pattern = Pattern.compile("(" + regex + ")($|[\\W])", (ignoreCase ? Pattern.CASE_INSENSITIVE : 0) | Pattern.DOTALL);
String pat = "(" + regex + ")($|[^\\w-])";
pattern = Pattern.compile(pat, (ignoreCase ? Pattern.CASE_INSENSITIVE : 0) | Pattern.DOTALL);
}
catch (PatternSyntaxException e) {
LOG.error(e);
LOG.info(e);
}
myPattern = pattern;
}
private static String escapeSpecialCharacters(String word) {
StringBuilder esc = new StringBuilder();
word = word.replace("\\", "\\\\");
for (int i = 0; i < word.length(); i++) {
char ch = word.charAt(i);
if ("-*+?$%^.()".indexOf(ch) >= 0) esc.append('\\');
esc.append(ch);
}
word = esc.toString();
return word;
}
private Set<String> getKeywordSet(Set<String> keywordSet) {
if (!myIgnoreCase) {
return new THashSet<String>(keywordSet);
platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTest.groovy
+276 -14
View File
@@ -15,15 +15,21 @@
*/
package com.intellij.ide.highlighter.custom
import com.intellij.psi.tree.IElementType
import static com.intellij.psi.CustomHighlighterTokenType.*
import org.jetbrains.annotations.NonNls
import org.jetbrains.annotations.Nullable
import com.intellij.testFramework.LexerTestCase
import junit.framework.TestCase
/**
* @author peter
*/
class CustomFileTypeLexerTest extends CustomFileTypeLexerTestBase {
@Override
protected SyntaxTable createSyntaxTable() {
class CustomFileTypeLexerTest extends TestCase {
protected void doTest(SyntaxTable table, @NonNls String text, @Nullable String expected) {
assertEquals(expected, LexerTestCase.printTokens(text, 0, new CustomFileTypeLexer(table)));
}
private SyntaxTable createGenericTable() {
SyntaxTable table = new SyntaxTable();
table.lineComment = ';'
@@ -40,20 +46,276 @@ class CustomFileTypeLexerTest extends CustomFileTypeLexerTestBase {
}
public void testSpacesInsideKeywords() {
checkTypesAndTokens('if length(variable)then return 1',
[KEYWORD_1, WHITESPACE, KEYWORD_1, CHARACTER, IDENTIFIER, CHARACTER, KEYWORD_1, WHITESPACE, KEYWORD_2, WHITESPACE, NUMBER] as IElementType[],
['if', ' ', 'length', '(', 'variable', ')', 'then', ' ', 'return', ' ', '1'] as String[])
doTest createGenericTable(), 'if length(variable)then return 1', '''\
KEYWORD_1 ('if')
WHITESPACE (' ')
KEYWORD_1 ('length')
CHARACTER ('(')
IDENTIFIER ('variable')
CHARACTER (')')
KEYWORD_1 ('then')
WHITESPACE (' ')
KEYWORD_2 ('return')
WHITESPACE (' ')
NUMBER ('1')
'''
}
public void testFortranComments() {
checkTypesAndTokens('''
doTest createGenericTable(), '''
foo;noncomment
;comment
;noncomment
''',
[WHITESPACE, IDENTIFIER, PUNCTUATION, IDENTIFIER, WHITESPACE, LINE_COMMENT, WHITESPACE, PUNCTUATION, IDENTIFIER, WHITESPACE] as IElementType[],
['\n', 'foo', ';', 'noncomment', '\n', ';comment', '\n ', ';', 'noncomment', '\n'] as String[])
''', '''\
WHITESPACE ('\\n')
IDENTIFIER ('foo')
PUNCTUATION (';')
IDENTIFIER ('noncomment')
WHITESPACE ('\\n')
LINE_COMMENT (';comment')
WHITESPACE ('\\n ')
PUNCTUATION (';')
IDENTIFIER ('noncomment')
WHITESPACE ('\\n')
'''
}
private SyntaxTable createJavaSyntaxTable() {
SyntaxTable table = new SyntaxTable();
table.setLineComment("//");
table.setStartComment("/*");
table.setEndComment("*/");
table.setHexPrefix("0x");
table.setNumPostfixChars("cfdle");
table.addKeyword1("package");
table.addKeyword1("import");
table.addKeyword1("this");
table.addKeyword1("super");
table.addKeyword1("public");
table.addKeyword1("private");
table.addKeyword1("protected");
table.addKeyword1("null");
table.addKeyword1("if");
table.addKeyword1("else");
table.addKeyword1("throws");
table.addKeyword1("switch");
table.addKeyword1("case");
table.addKeyword1("break");
table.addKeyword1("default");
table.addKeyword1("continue");
table.addKeyword1("goto");
table.addKeyword1("boolean");
table.addKeyword1("true");
table.addKeyword1("false");
table.addKeyword1("final");
table.addKeyword1("class");
table.addKeyword1("static");
table.addKeyword1("final");
table.addKeyword1("void");
table.addKeyword1("int");
table.addKeyword1("while");
table.addKeyword1("new");
table.addKeyword1("for");
table.addKeyword1("byte");
table.addKeyword1("float");
table.addKeyword1("double");
table.addKeyword1("short");
table.addKeyword1("extends");
table.addKeyword1("implements");
table.addKeyword1("interface");
table.addKeyword1("abstract");
table.addKeyword1("char");
table.addKeyword1("try");
table.addKeyword1("catch");
table.addKeyword1("finally");
table.addKeyword1("synchronized");
return table;
}
public void testParseSampleCode() {
doTest createJavaSyntaxTable(), "private some text f b g\n\n\n// 1\n public static void main(String[] args) {\n}\n-10 - 10\n\"dsfdfdf\"\n/* a\n *bc */", '''\
KEYWORD_1 ('private')
WHITESPACE (' ')
IDENTIFIER ('some')
WHITESPACE (' ')
IDENTIFIER ('text')
WHITESPACE (' ')
IDENTIFIER ('f')
WHITESPACE (' ')
IDENTIFIER ('b')
WHITESPACE (' ')
IDENTIFIER ('g')
WHITESPACE ('\\n\\n\\n')
LINE_COMMENT ('// 1')
WHITESPACE ('\\n ')
KEYWORD_1 ('public')
WHITESPACE (' ')
KEYWORD_1 ('static')
WHITESPACE (' ')
KEYWORD_1 ('void')
WHITESPACE (' ')
IDENTIFIER ('main')
CHARACTER ('(')
IDENTIFIER ('String')
CHARACTER ('[')
CHARACTER (']')
WHITESPACE (' ')
IDENTIFIER ('args')
CHARACTER (')')
WHITESPACE (' ')
CHARACTER ('{')
WHITESPACE ('\\n')
CHARACTER ('}')
WHITESPACE ('\\n')
CHARACTER ('-')
NUMBER ('10')
WHITESPACE (' ')
CHARACTER ('-')
WHITESPACE (' ')
NUMBER ('10')
WHITESPACE ('\\n')
STRING ('"dsfdfdf"')
WHITESPACE ('\\n')
MULTI_LINE_COMMENT ('/* a\\n *bc */')
'''
}
public void testBlockCommentStart() {
doTest createJavaSyntaxTable(), "/*", 'MULTI_LINE_COMMENT (\'/*\')\n'
}
public void testLineCommentStart() {
doTest createJavaSyntaxTable(), "//", 'LINE_COMMENT (\'//\')\n'
}
public void testEmpty() {
doTest createJavaSyntaxTable(), "", ''
}
public void testSpace() {
doTest createJavaSyntaxTable(), " ", 'WHITESPACE (\' \')\n'
}
public void testParseSampleCodeFromTo() {
String sampleCode = " int n=123;\n float z=1;";
def lexer = new CustomFileTypeLexer(createJavaSyntaxTable())
lexer.start(sampleCode, 5, 5);
assertEquals(lexer.getTokenType(), null);
lexer.start(sampleCode, 5, 6);
lexer.getTokenType();
assertEquals(5, lexer.getTokenStart());
assertEquals(6, lexer.getTokenEnd());
assertEquals(6, lexer.getBufferEnd());
}
private SyntaxTable createPropTable() {
SyntaxTable table = new SyntaxTable();
table.setLineComment("#");
table.setIgnoreCase(true);
table.addKeyword1("value");
table.addKeyword2("Value");
table.setNumPostfixChars("LGH");
return table;
}
public void testSimple() {
doTest createPropTable(), "# Comment\n" +
"x.1.a=12.2L\n" +
" y.2.b=13.4 # comment\n" +
"VALUE value VaLuE Value1 17.00h 11.0k", '''\
LINE_COMMENT ('# Comment')
WHITESPACE ('\\n')
IDENTIFIER ('x')
PUNCTUATION ('.')
NUMBER ('1')
PUNCTUATION ('.')
IDENTIFIER ('a')
CHARACTER ('=')
NUMBER ('12.2L')
WHITESPACE ('\\n ')
IDENTIFIER ('y')
PUNCTUATION ('.')
NUMBER ('2')
PUNCTUATION ('.')
IDENTIFIER ('b')
CHARACTER ('=')
NUMBER ('13.4')
WHITESPACE (' ')
LINE_COMMENT ('# comment')
WHITESPACE ('\\n')
KEYWORD_1 ('VALUE')
WHITESPACE (' ')
KEYWORD_1 ('value')
WHITESPACE (' ')
KEYWORD_1 ('VaLuE')
WHITESPACE (' ')
IDENTIFIER ('Value1')
WHITESPACE (' ')
NUMBER ('17.00h')
WHITESPACE (' ')
NUMBER ('11')
PUNCTUATION ('.')
NUMBER ('0')
IDENTIFIER ('k')
'''
}
public void testNumber() {
doTest createPropTable(), "1.23=1.24", '''\
NUMBER ('1.23')
CHARACTER ('=')
NUMBER ('1.24')
'''
}
public void testPostfix() {
doTest createPropTable(), "abc 1.2ltext", '''\
IDENTIFIER ('abc')
WHITESPACE (' ')
NUMBER ('1.2l')
IDENTIFIER ('text')
'''
}
public void testWeird() {
doTest createPropTable(), "test.1.", '''\
IDENTIFIER ('test')
PUNCTUATION ('.')
NUMBER ('1.')
'''
}
public void testParenths() throws Exception {
doTest createPropTable(),"value(255)", '''\
KEYWORD_1 ('value')
CHARACTER ('(')
NUMBER ('255')
CHARACTER (')')
'''
}
public void testSpecialCharactersInKeywords() {
SyntaxTable table = new SyntaxTable()
table.addKeyword1("a*")
table.addKeyword1("b-c")
table.addKeyword2("d#")
table.addKeyword2("e")
doTest table, 'a* b-c d# e-', '''\
KEYWORD_1 ('a*')
WHITESPACE (' ')
KEYWORD_1 ('b-c')
WHITESPACE (' ')
KEYWORD_2 ('d#')
WHITESPACE (' ')
IDENTIFIER ('e-')
'''
}
}
platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/CustomFileTypeLexerTestBase.java
-47
View File
@@ -1,47 +0,0 @@
package com.intellij.ide.highlighter.custom;
import com.intellij.lexer.Lexer;
import com.intellij.psi.tree.IElementType;
import junit.framework.TestCase;
/**
* @author dsl
*/
public abstract class CustomFileTypeLexerTestBase extends TestCase {
protected Lexer lexer;
@Override
protected void setUp() throws Exception {
lexer = new CustomFileTypeLexer(createSyntaxTable());
}
protected abstract SyntaxTable createSyntaxTable();
protected void checkTypesAndTokens(String sampleCode, IElementType[] types, String[] matches) {
lexer.start(sampleCode);
assertEquals(types.length, matches.length);
for (int i = 0; i < types.length; i++) {
assertEquals("Failed at index=" + i, types[i], lexer.getTokenType());
assertEquals("Failed at index=" + i, matches[i], sampleCode.substring(lexer.getTokenStart(), lexer.getTokenEnd()));
lexer.advance();
}
}
protected void checkSameText(String sampleCode) {
lexer.start(sampleCode);
StringBuffer sb = new StringBuffer();
String result;
while (lexer.getTokenType() != null) {
result = sampleCode.substring(lexer.getTokenStart(), lexer.getTokenEnd());
sb.append(result);
lexer.advance();
}
assertEquals("Text created by lexer's output does not match the original text", sampleCode, sb.toString());
}
public void testNothing() throws Exception {
//TODO[dsl] Testcase fails if there's no any testcase. So I've added an empty one.
}
}
platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/JavaCodeTest.java
-130
View File
@@ -1,130 +0,0 @@
package com.intellij.ide.highlighter.custom;
import com.intellij.psi.CustomHighlighterTokenType;
import com.intellij.psi.tree.IElementType;
/**
* @author Yura Cangea
* @version 1.0
*/
public class JavaCodeTest extends CustomFileTypeLexerTestBase {
@Override
public SyntaxTable createSyntaxTable() {
SyntaxTable table = new SyntaxTable();
table.setLineComment("//");
table.setStartComment("/*");
table.setEndComment("*/");
table.setHexPrefix("0x");
table.setNumPostfixChars("cfdle");
table.addKeyword1("package");
table.addKeyword1("import");
table.addKeyword1("this");
table.addKeyword1("super");
table.addKeyword1("public");
table.addKeyword1("private");
table.addKeyword1("protected");
table.addKeyword1("null");
table.addKeyword1("if");
table.addKeyword1("else");
table.addKeyword1("throws");
table.addKeyword1("switch");
table.addKeyword1("case");
table.addKeyword1("break");
table.addKeyword1("default");
table.addKeyword1("continue");
table.addKeyword1("goto");
table.addKeyword1("boolean");
table.addKeyword1("true");
table.addKeyword1("false");
table.addKeyword1("final");
table.addKeyword1("class");
table.addKeyword1("static");
table.addKeyword1("final");
table.addKeyword1("void");
table.addKeyword1("int");
table.addKeyword1("while");
table.addKeyword1("new");
table.addKeyword1("for");
table.addKeyword1("byte");
table.addKeyword1("float");
table.addKeyword1("double");
table.addKeyword1("short");
table.addKeyword1("extends");
table.addKeyword1("implements");
table.addKeyword1("interface");
table.addKeyword1("abstract");
table.addKeyword1("char");
table.addKeyword1("try");
table.addKeyword1("catch");
table.addKeyword1("finally");
table.addKeyword1("synchronized");
return table;
}
public void testParseSampleCode() {
String sampleCode =
"private some text f b g\n\n\n// 1\n public static void main(String[] args) {\n}\n-10 - 10\n\"dsfdfdf\"\n/* a\n *bc */";
String sampleCode2 = "/*";
String sampleCode3 = "//";
String sampleCode4 = "";
String sampleCode5 = " ";
checkSameText(sampleCode);
String result;
IElementType[] types = new IElementType[]{CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.LINE_COMMENT, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.CHARACTER, CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.CHARACTER,
CustomHighlighterTokenType.CHARACTER, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.CHARACTER,
CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.CHARACTER, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.CHARACTER,
CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.CHARACTER, CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.CHARACTER, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.STRING, CustomHighlighterTokenType.WHITESPACE, CustomHighlighterTokenType.MULTI_LINE_COMMENT,
};
String[] matches = new String[]{"private", " ", "some", " ", "text", " ", "f", " ", "b", " ", "g", "\n\n\n", "// 1", "\n ", "public",
" ", "static", " ", "void", " ", "main", "(",
"String", "[", "]", " ", "args", ")", " ", "{", "\n", "}", "\n", "-", "10",
" ", "-", " ", "10", "\n", "\"dsfdfdf\"", "\n", "/* a\n *bc */",
};
checkTypesAndTokens(sampleCode, types, matches);
lexer.start(sampleCode2);
assertEquals(CustomHighlighterTokenType.MULTI_LINE_COMMENT, lexer.getTokenType());
assertEquals("/*", sampleCode2.substring(lexer.getTokenStart(), lexer.getTokenEnd()));
lexer.start(sampleCode3);
assertEquals(CustomHighlighterTokenType.LINE_COMMENT, lexer.getTokenType());
assertEquals("//", sampleCode3.substring(lexer.getTokenStart(), lexer.getTokenEnd()));
lexer.start(sampleCode4);
assertEquals(null, lexer.getTokenType());
assertEquals("", sampleCode4.substring(lexer.getTokenStart(), lexer.getTokenEnd()));
lexer.start(sampleCode5);
assertEquals(CustomHighlighterTokenType.WHITESPACE, lexer.getTokenType());
assertEquals(" ", sampleCode5.substring(lexer.getTokenStart(), lexer.getTokenEnd()));
}
public void testParseSampleCodeFromTo() {
String sampleCode = " int n=123;\n float z=1;";
lexer.start(sampleCode, 5, 5);
assertEquals(lexer.getTokenType(), null);
lexer.start(sampleCode, 5, 6);
lexer.getTokenType();
assertEquals(5, lexer.getTokenStart());
assertEquals(6, lexer.getTokenEnd());
assertEquals(6, lexer.getBufferEnd());
}
}
platform/platform-tests/testSrc/com/intellij/ide/highlighter/custom/PropertiesFileTest.java
-112
View File
@@ -1,112 +0,0 @@
package com.intellij.ide.highlighter.custom;
import com.intellij.psi.CustomHighlighterTokenType;
import com.intellij.psi.tree.IElementType;
/**
* @author dsl
*/
public class PropertiesFileTest extends CustomFileTypeLexerTestBase {
@Override
protected void setUp() throws Exception {
lexer = new CustomFileTypeLexer(createSyntaxTable());
}
@Override
protected SyntaxTable createSyntaxTable() {
SyntaxTable table = new SyntaxTable();
table.setLineComment("#");
table.setIgnoreCase(true);
table.addKeyword1("value");
table.addKeyword2("Value");
table.setNumPostfixChars("LGH");
return table;
}
public void testSimple() {
String sampleFile =
"# Comment\n" +
"x.1.a=12.2L\n" +
" y.2.b=13.4 # comment\n" +
"VALUE value VaLuE Value1 17.00h 11.0k";
checkSameText(sampleFile);
IElementType[] types = {
CustomHighlighterTokenType.LINE_COMMENT, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.PUNCTUATION,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.PUNCTUATION,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.CHARACTER,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.PUNCTUATION,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.PUNCTUATION,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.CHARACTER,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.LINE_COMMENT, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.PUNCTUATION,
CustomHighlighterTokenType.NUMBER,
CustomHighlighterTokenType.IDENTIFIER
};
String[] matches = {
"# Comment", "\n",
"x", ".", "1", ".", "a", "=", "12.2L", "\n ",
"y", ".", "2", ".", "b", "=", "13.4", " ", "# comment", "\n",
"VALUE", " ", "value", " ", "VaLuE", " ", "Value1", " ", "17.00h", " ", "11", ".", "0", "k"
};
checkTypesAndTokens(sampleFile, types, matches);
}
public void testNumber() {
String sample = "1.23=1.24";
checkSameText(sample);
IElementType[] types = {
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.CHARACTER, CustomHighlighterTokenType.NUMBER
};
String[] matches = { "1.23", "=", "1.24" };
checkTypesAndTokens(sample, types, matches);
}
public void testPostfix() {
String sample = "abc 1.2ltext";
checkSameText(sample);
IElementType[] types = {
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.WHITESPACE,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.IDENTIFIER
};
String[] matches = { "abc", " ", "1.2l", "text" };
checkTypesAndTokens(sample, types, matches);
}
public void testWeird() {
String sample = "test.1.";
checkSameText(sample);
IElementType[] types = {
CustomHighlighterTokenType.IDENTIFIER, CustomHighlighterTokenType.PUNCTUATION, CustomHighlighterTokenType.NUMBER
};
String[] matches = { "test", ".", "1." };
checkTypesAndTokens(sample, types, matches);
}
public void testParenths() throws Exception {
String sample = "value(255)";
checkSameText(sample);
IElementType[] types = {
CustomHighlighterTokenType.KEYWORD_1, CustomHighlighterTokenType.CHARACTER,
CustomHighlighterTokenType.NUMBER, CustomHighlighterTokenType.CHARACTER
};
String[] matches = { "value", "(", "255", ")" };
checkTypesAndTokens(sample, types, matches);
}
}
platform/testFramework/src/com/intellij/testFramework/LexerTestCase.java
+1 -1
View File
@@ -72,7 +72,7 @@ public abstract class LexerTestCase extends UsefulTestCase {
return printTokens(text, start, createLexer());
}
private static String printTokens(String text, int start, Lexer lexer) {
public static String printTokens(String text, int start, Lexer lexer) {
lexer.start(text, start, text.length());
String result = "";
while (true) {