java: lexers united; lexer creation extracted

This commit is contained in:
Roman Shevchenko
2013-06-07 12:00:15 +04:00
parent 03e5528a24
commit a0f88a0987
31 changed files with 1606 additions and 840 deletions
@@ -15,7 +15,7 @@
*/
package com.intellij.lang.java.lexer;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.lexer.Lexer;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.testFramework.LexerTestCase;
@@ -158,7 +158,7 @@ public class JavaLexerTest extends LexerTestCase {
@Override
protected Lexer createLexer() {
return new JavaLexer(LanguageLevel.HIGHEST);
return JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
}
@Override
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2011 JetBrains s.r.o.
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -81,5 +81,5 @@ public class JavadocParsingTest extends JavaParsingTestCase {
public void testTypeParam() { doTest(true); }
public void testParameterlessTag() { doTest(true); }
public void testIDEADEV_41403() {doTest(true);}
public void testIDEADEV_41403() { doTest(true); }
}
@@ -1,8 +1,23 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lang.psi;
import com.intellij.lang.*;
import com.intellij.lang.impl.PsiBuilderImpl;
import com.intellij.lexer.JavaLexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.openapi.fileTypes.StdFileTypes;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
@@ -17,7 +32,7 @@ import com.intellij.testFramework.LightIdeaTestCase;
import junit.framework.AssertionFailedError;
/**
* Date: Jan 21, 2005
* @since Jan 21, 2005
* @author max
*/
public class PsiBuilderTest extends LightIdeaTestCase {
@@ -66,7 +81,7 @@ public class PsiBuilderTest extends LightIdeaTestCase {
final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
assertNotNull(parserDefinition);
PsiFile psiFile = createFile("x.java", text);
return new PsiBuilderImpl(getProject(), psiFile, parserDefinition, new JavaLexer(LanguageLevel.JDK_1_5),
return new PsiBuilderImpl(getProject(), psiFile, parserDefinition, JavaParserDefinition.createLexer(LanguageLevel.JDK_1_5),
SharedImplUtil.findCharTableByTree(psiFile.getNode()), text, originalTree, null);
}
@@ -152,8 +167,8 @@ public class PsiBuilderTest extends LightIdeaTestCase {
public void testAssertionFailureOnUnbalancedMarkers() {
myBuilder = createBuilder("foo");
myBuilder.setDebugMode(true);
final PsiBuilder.Marker m = myBuilder.mark();
final PsiBuilder.Marker m1 = myBuilder.mark();
PsiBuilder.Marker m = myBuilder.mark();
@SuppressWarnings("UnusedDeclaration") PsiBuilder.Marker m1 = myBuilder.mark();
myBuilder.getTokenType();
myBuilder.advanceLexer();
try {
@@ -227,6 +242,4 @@ public class PsiBuilderTest extends LightIdeaTestCase {
myBuilder.advanceLexer();
root.done(JavaStubElementTypes.ENUM_CONSTANT_INITIALIZER);
}
}
@@ -1,5 +1,21 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.tree.IElementType;
@@ -91,17 +107,16 @@ public class LayeredLexerTest extends TestCase {
assertEquals(null, lexer.getTokenType());
}
private Lexer setupLexer(String text) {
LayeredLexer lexer = new LayeredLexer(new JavaLexer(LanguageLevel.JDK_1_3));
private static Lexer setupLexer(String text) {
LayeredLexer lexer = new LayeredLexer(JavaParserDefinition.createLexer(LanguageLevel.JDK_1_3));
lexer.registerSelfStoppingLayer(new StringLiteralLexer('\"', JavaTokenType.STRING_LITERAL),
new IElementType[]{JavaTokenType.STRING_LITERAL},
IElementType.EMPTY_ARRAY);
lexer.start(text);
return lexer;
}
private String nextToken(Lexer lexer) {
private static String nextToken(Lexer lexer) {
assertTrue(lexer.getTokenType() != null);
final String s = lexer.getBufferSequence().subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
lexer.advance();