MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

New Java parser (work in progress)

Browse Source
This commit is contained in:
Roman Shevchenko
2010-07-30 20:26:28 +04:00
parent 160631140d
commit a847515b92
16 changed files with 257 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/java-impl/src/com/intellij/lang/java/parser/DeclarationParser.java
+3 -5
View File
@@ -56,7 +56,7 @@ public class DeclarationParser {
expectOrError(builder, JavaTokenType.RBRACE, JavaErrorMessages.message("expected.rbrace"));
}
@NotNull
@Nullable
private static PsiBuilder.Marker parseClassFromKeyword(final PsiBuilder builder, final PsiBuilder.Marker declaration,
final boolean isAnnotation) {
final IElementType keywordTokenType = builder.getTokenType();
@@ -67,7 +67,7 @@ public class DeclarationParser {
if (!expect(builder, JavaTokenType.IDENTIFIER)) {
error(builder, JavaErrorMessages.message("expected.identifier"));
declaration.drop();
return declaration;
return null;
}
ReferenceParser.parseTypeParameters(builder);
@@ -195,10 +195,8 @@ public class DeclarationParser {
if (context == Context.FILE) return null;
}
else if (tokenType instanceof ILazyParseableElementType) {
final PsiBuilder.Marker declaration = builder.mark();
declaration.drop();
builder.advanceLexer();
return declaration;
return null;
}
else if (!ElementType.MODIFIER_BIT_SET.contains(tokenType) &&
!ElementType.CLASS_KEYWORD_BIT_SET.contains(tokenType) &&
java/java-impl/src/com/intellij/lang/java/parser/ReferenceParser.java
+2 -1
View File
@@ -198,6 +198,7 @@ public class ReferenceParser {
}
dotPos.drop();
final PsiBuilder.Marker prevElement = refElement;
refElement = refElement.precede();
if (!hasIdentifier) {
@@ -205,7 +206,7 @@ public class ReferenceParser {
if (isImport) {
error(builder, JavaErrorMessages.message("import.statement.identifier.or.asterisk.expected."));
refElement.drop();
return refElement;
return prevElement;
}
else {
error(builder, JavaErrorMessages.message("expected.identifier"));
java/java-impl/src/com/intellij/lang/java/parser/StatementParser.java
+3 -4
View File
@@ -34,7 +34,7 @@ import static com.intellij.lang.java.parser.JavaParserUtil.*;
public class StatementParser {
private static final boolean DEEP_PARSE_BLOCKS_IN_STATEMENTS = true; // todo: reset after testing done
private static final boolean DEEP_PARSE_BLOCKS_IN_STATEMENTS = false;
private enum BraceMode {
TILL_FIRST, TILL_LAST
@@ -199,10 +199,8 @@ public class StatementParser {
return parseBlockStatement(builder);
}
else if (tokenType instanceof ILazyParseableElementType) {
final PsiBuilder.Marker marker = builder.mark();
marker.drop();
builder.advanceLexer();
return marker;
return null;
}
else if (tokenType == JavaTokenType.SEMICOLON) {
final PsiBuilder.Marker empty = builder.mark();
@@ -220,6 +218,7 @@ public class StatementParser {
final PsiBuilder.Marker declStatement = builder.mark();
final PsiBuilder.Marker decl = DeclarationParser.parse(builder, DeclarationParser.Context.CODE_BLOCK);
if (decl == null) {
ReferenceParser.parseType(builder, false, false);
error(builder, JavaErrorMessages.message("expected.identifier"));
}
declStatement.done(JavaElementType.DECLARATION_STATEMENT);
java/java-tests/testData/psi/parser-partial/files/BindBefore0.txt
+29
View File
@@ -0,0 +1,29 @@
PsiJavaFile:BindBefore0.java
PsiImportList
<empty list>
PsiClass:A
PsiModifierList:
<empty list>
PsiKeyword:class('class')
PsiWhiteSpace(' ')
PsiIdentifier:A('A')
PsiTypeParameterList
<empty list>
PsiReferenceList
<empty list>
PsiReferenceList
<empty list>
PsiJavaToken:LBRACE('{')
PsiWhiteSpace('\n ')
PsiField:field
PsiComment(END_OF_LINE_COMMENT)('// comment')
PsiWhiteSpace('\n ')
PsiModifierList:
<empty list>
PsiTypeElement:int
PsiKeyword:int('int')
PsiWhiteSpace(' ')
PsiIdentifier:field('field')
PsiJavaToken:SEMICOLON(';')
PsiWhiteSpace('\n')
PsiJavaToken:RBRACE('}')
java/java-tests/testData/psi/parser-partial/files/BindBefore1.txt
+29
View File
@@ -0,0 +1,29 @@
PsiJavaFile:BindBefore1.java
PsiImportList
<empty list>
PsiClass:A
PsiModifierList:
<empty list>
PsiKeyword:class('class')
PsiWhiteSpace(' ')
PsiIdentifier:A('A')
PsiTypeParameterList
<empty list>
PsiReferenceList
<empty list>
PsiReferenceList
<empty list>
PsiJavaToken:LBRACE('{')
PsiWhiteSpace('\n ')
PsiComment(END_OF_LINE_COMMENT)('// comment')
PsiWhiteSpace('\n\n ')
PsiField:field
PsiModifierList:
<empty list>
PsiTypeElement:int
PsiKeyword:int('int')
PsiWhiteSpace(' ')
PsiIdentifier:field('field')
PsiJavaToken:SEMICOLON(';')
PsiWhiteSpace('\n')
PsiJavaToken:RBRACE('}')
java/java-tests/testData/psi/parser-partial/files/BindBefore2.txt
+29
View File
@@ -0,0 +1,29 @@
PsiJavaFile:BindBefore2.java
PsiImportList
<empty list>
PsiClass:A
PsiModifierList:
<empty list>
PsiKeyword:class('class')
PsiWhiteSpace(' ')
PsiIdentifier:A('A')
PsiTypeParameterList
<empty list>
PsiReferenceList
<empty list>
PsiReferenceList
<empty list>
PsiJavaToken:LBRACE('{')
PsiWhiteSpace(' ')
PsiComment(END_OF_LINE_COMMENT)('// comment')
PsiWhiteSpace('\n ')
PsiField:field
PsiModifierList:
<empty list>
PsiTypeElement:int
PsiKeyword:int('int')
PsiWhiteSpace(' ')
PsiIdentifier:field('field')
PsiJavaToken:SEMICOLON(';')
PsiWhiteSpace('\n')
PsiJavaToken:RBRACE('}')
java/java-tests/testData/psi/parser-partial/files/BindBefore3.txt
+28
View File
@@ -0,0 +1,28 @@
PsiJavaFile:BindBefore3.java
PsiImportList
<empty list>
PsiClass:A
PsiModifierList:
<empty list>
PsiKeyword:class('class')
PsiWhiteSpace(' ')
PsiIdentifier:A('A')
PsiTypeParameterList
<empty list>
PsiReferenceList
<empty list>
PsiReferenceList
<empty list>
PsiJavaToken:LBRACE('{')
PsiComment(END_OF_LINE_COMMENT)('// comment')
PsiWhiteSpace('\n ')
PsiField:field
PsiModifierList:
<empty list>
PsiTypeElement:int
PsiKeyword:int('int')
PsiWhiteSpace(' ')
PsiIdentifier:field('field')
PsiJavaToken:SEMICOLON(';')
PsiWhiteSpace('\n')
PsiJavaToken:RBRACE('}')
java/java-tests/testData/psi/parser-partial/files/BindBefore4.txt
+33
View File
@@ -0,0 +1,33 @@
PsiJavaFile:BindBefore4.java
PsiImportList
<empty list>
PsiClass:A
PsiModifierList:
<empty list>
PsiKeyword:class('class')
PsiWhiteSpace(' ')
PsiIdentifier:A('A')
PsiTypeParameterList
<empty list>
PsiReferenceList
<empty list>
PsiReferenceList
<empty list>
PsiJavaToken:LBRACE('{')
PsiWhiteSpace('\n ')
PsiField:field
PsiComment(END_OF_LINE_COMMENT)('// comment 1')
PsiWhiteSpace('\n ')
PsiComment(END_OF_LINE_COMMENT)('// comment 2')
PsiWhiteSpace('\n ')
PsiComment(END_OF_LINE_COMMENT)('// comment 3')
PsiWhiteSpace('\n ')
PsiModifierList:
<empty list>
PsiTypeElement:int
PsiKeyword:int('int')
PsiWhiteSpace(' ')
PsiIdentifier:field('field')
PsiJavaToken:SEMICOLON(';')
PsiWhiteSpace('\n')
PsiJavaToken:RBRACE('}')
java/java-tests/testSrc/com/intellij/JavaTestUtil.java
-8
View File
@@ -20,12 +20,4 @@ public class JavaTestUtil {
return PathManagerEx.getTestDataPath();
}
public static PsiBuilder getJavaBuilder(final CharSequence source) {
final LanguageLevel languageLevel = LanguageLevel.HIGHEST;
final JavaLexer lexer = new JavaLexer(languageLevel);
final PsiBuilder builder = PsiBuilderFactory.getInstance().createBuilder(lexer, StdLanguages.JAVA, source);
builder.setDebugMode(true);
JavaParserUtil.setLanguageLevel(builder, languageLevel);
return builder;
}
}
java/java-tests/testSrc/com/intellij/lang/java/parser/JavaParsingTestCase.java
+65 -17
View File
@@ -15,17 +15,29 @@
*/
package com.intellij.lang.java.parser;
import com.intellij.lang.ASTNode;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.PsiBuilderFactory;
import com.intellij.lang.StdLanguages;
import com.intellij.openapi.fileTypes.StdFileTypes;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.roots.LanguageLevelProjectExtension;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.FileViewProvider;
import com.intellij.psi.PsiManager;
import com.intellij.psi.SingleRootFileViewProvider;
import com.intellij.psi.impl.DebugUtil;
import com.intellij.psi.impl.source.tree.JavaElementType;
import com.intellij.psi.impl.source.PsiJavaFileImpl;
import com.intellij.psi.impl.source.tree.FileElement;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.IFileElementType;
import com.intellij.testFramework.IdeaTestCase;
import com.intellij.testFramework.LightVirtualFile;
import com.intellij.testFramework.ParsingTestCase;
import org.jetbrains.annotations.NonNls;
import java.io.IOException;
import static com.intellij.JavaTestUtil.getJavaBuilder;
public abstract class JavaParsingTestCase extends ParsingTestCase {
@SuppressWarnings({"JUnitTestCaseWithNonTrivialConstructors"})
@@ -34,33 +46,69 @@ public abstract class JavaParsingTestCase extends ParsingTestCase {
IdeaTestCase.initPlatformPrefix();
}
protected interface Parser {
protected static void withLevel(final LanguageLevel level, final Runnable r) {
final LanguageLevel current = LanguageLevelProjectExtension.getInstance(getProject()).getLanguageLevel();
try {
LanguageLevelProjectExtension.getInstance(getProject()).setLanguageLevel(level);
r.run();
}
finally {
LanguageLevelProjectExtension.getInstance(getProject()).setLanguageLevel(current);
}
}
protected interface TestParser {
void parse(PsiBuilder builder);
}
protected void doParserTest(final String text, final Parser parser) {
protected void doParserTest(final String source, final TestParser parser) {
final String name = getTestName(false);
final PsiBuilder builder = getJavaBuilder(text);
final PsiBuilder.Marker root = builder.mark();
final IElementType fileElementType = new IFileElementType("test.java.file", StdLanguages.JAVA) {
@Override
public ASTNode parseContents(final ASTNode chameleon) {
final PsiBuilder builder = createBuilder(chameleon);
parser.parse(builder);
final PsiBuilder.Marker root = builder.mark();
parser.parse(builder);
if (!builder.eof()) {
final PsiBuilder.Marker unparsed = builder.mark();
while (!builder.eof()) builder.advanceLexer();
unparsed.error("Unparsed tokens");
}
root.done(this);
if (builder.getTokenType() != null) {
final PsiBuilder.Marker unparsed = builder.mark();
while (builder.getTokenType() != null) builder.advanceLexer();
unparsed.error("Unparsed tokens");
}
return builder.getTreeBuilt().getFirstChildNode();
}
};
root.done(JavaElementType.JAVA_FILE);
final LightVirtualFile virtualFile = new LightVirtualFile(name + '.' + myFileExt, StdFileTypes.JAVA, source, -1);
final FileViewProvider viewProvider = new SingleRootFileViewProvider(PsiManager.getInstance(getProject()), virtualFile, true);
final PsiJavaFileImpl psiFile = new PsiJavaFileImpl(viewProvider) {
@Override
protected FileElement createFileElement(final CharSequence text) {
return new FileElement(fileElementType, text);
}
};
final String raw = DebugUtil.treeToString(builder.getTreeBuilt(), false);
final String tree = raw.replaceFirst("com.intellij.psi.util.PsiUtilBase\\S+", "PsiJavaFile:" + name + ".java");
try {
checkResult(name + ".txt", tree);
checkResult(name + ".txt", DebugUtil.psiToString(psiFile, false));
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
private static PsiBuilder createBuilder(final ASTNode chameleon) {
final Project project = chameleon.getPsi().getProject();
final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
final PsiBuilder builder = factory.createBuilder(project, chameleon, null, chameleon.getElementType().getLanguage(), chameleon.getChars());
builder.setDebugMode(true);
final LanguageLevel level = LanguageLevelProjectExtension.getInstance(project).getLanguageLevel();
JavaParserUtil.setLanguageLevel(builder, level);
return builder;
}
}
java/java-tests/testSrc/com/intellij/lang/java/parser/partial/AnnotationParserTest.java
+1 -1
View File
@@ -38,7 +38,7 @@ public class AnnotationParserTest extends JavaParsingTestCase {
public void testFirstNameMissed() { doParserTest("@Anno(value1, param2=value2)"); }
private void doParserTest(final String text) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
DeclarationParser.parseAnnotations(builder);
}
java/java-tests/testSrc/com/intellij/lang/java/parser/partial/DeclarationParserTest.java
+15 -9
View File
@@ -18,6 +18,7 @@ package com.intellij.lang.java.parser.partial;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.java.parser.DeclarationParser;
import com.intellij.lang.java.parser.JavaParsingTestCase;
import com.intellij.pom.java.LanguageLevel;
// todo: fix parser and uncomment tests
@@ -40,14 +41,19 @@ public class DeclarationParserTest extends JavaParsingTestCase {
public void testEnumSmartTypeCompletion() { doParserTest("{ @Preliminary(A.B\n#) public class TimeTravel {}\n" +
" @Preliminary(a=A.B\n#) public class TimeTravel {}\n" +
" @Preliminary(a=A.B\n#, b=c) public class TimeTravel {} }", false, false); }
public void testTypeAnno() { doParserTest("{ class C<@D T extends @F Object> extends @F Object {\n" +
" @F int @F[] method() @F throws @F Exception {\n" +
" a = this instanceof @F C;\n" +
" C<@F @G C> c = new @Q C<@F C>();\n" +
" c = (@F Object)c;\n" +
" Class c = @TA String.class;\n" +
" @F C.field++;\n" +
" }\n} }", false, false); }
public void testTypeAnno() {
withLevel(LanguageLevel.JDK_1_7,
new Runnable() { public void run() {
doParserTest("{ class C<@D T extends @F Object> extends @F Object {\n" +
" @F int @F[] method() @F throws @F Exception {\n" +
" a = this instanceof @F C;\n" +
" C<@F @G C> c = new @Q C<@F C>();\n" +
" c = (@F Object)c;\n" +
" Class c = @TA String.class;\n" +
" @F C.field++;\n" +
" }\n} }", false, false);
} });
}
public void testEnumBody0() { doParserTest("{ ; }", false, true); }
public void testEnumBody1() { doParserTest("{ RED, GREEN, BLUE; }", false, true); }
@@ -95,7 +101,7 @@ public class DeclarationParserTest extends JavaParsingTestCase {
public void testParameterizedMethod() { doParserTest("{ @Nullable <T> T bar() {} }", false, false); }
private void doParserTest(final String text, final boolean isAnnotation, final boolean isEnum) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
DeclarationParser.parseClassBodyWithBraces(builder, isAnnotation, isEnum);
}
java/java-tests/testSrc/com/intellij/lang/java/parser/partial/ExpressionParserTest.java
+1 -1
View File
@@ -101,7 +101,7 @@ public class ExpressionParserTest extends JavaParsingTestCase {
public void testPrimitiveClassObjectAccess() { doParserTest("int.class"); }
private void doParserTest(final String text) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
ExpressionParser.parse(builder);
}
java/java-tests/testSrc/com/intellij/lang/java/parser/partial/FileParserTest.java
+10 -1
View File
@@ -20,6 +20,7 @@ import com.intellij.lang.java.parser.FileParser;
import com.intellij.lang.java.parser.JavaParsingTestCase;
// todo: fix comment binding, uncomment tests
public class FileParserTest extends JavaParsingTestCase {
public FileParserTest() {
super("parser-partial/files");
@@ -45,8 +46,16 @@ public class FileParserTest extends JavaParsingTestCase {
"public class C { }\n" +
"class D { }"); }
//public void testBindBefore0() { doParserTest("class A{\n // comment\n int field;\n}"); }
public void testBindBefore1() { doParserTest("class A{\n // comment\n\n int field;\n}"); }
public void testBindBefore2() { doParserTest("class A{ // comment\n int field;\n}"); }
public void testBindBefore3() { doParserTest("class A{// comment\n int field;\n}"); }
//public void testBindBefore4() { doParserTest("class A{\n // comment 1\n // comment 2\n // comment 3\n int field;\n}"); }
// todo: doc comment binding tests
private void doParserTest(final String text) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
FileParser.parse(builder);
}
java/java-tests/testSrc/com/intellij/lang/java/parser/partial/ReferenceParserTest.java
+8 -4
View File
@@ -18,6 +18,7 @@ package com.intellij.lang.java.parser.partial;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.java.parser.JavaParsingTestCase;
import com.intellij.lang.java.parser.ReferenceParser;
import com.intellij.pom.java.LanguageLevel;
public class ReferenceParserTest extends JavaParsingTestCase {
@@ -35,7 +36,10 @@ public class ReferenceParserTest extends JavaParsingTestCase {
public void testType3() { doTypeParserTest("int[][", false); }
public void testType4() { doTypeParserTest("Map<String,List<String>>", false); }
public void testType5() { doTypeParserTest("Object[]...", false); }
public void testType6() { doTypeParserTest("@English String @NonEmpty []", false); }
public void testType6() {
withLevel(LanguageLevel.JDK_1_7,
new Runnable() { public void run() { doTypeParserTest("@English String @NonEmpty []", false); } });
}
public void testTypeParams0() { doTypeParamsParserTest("<T>"); }
public void testTypeParams1() { doTypeParamsParserTest("<T, U>"); }
@@ -47,7 +51,7 @@ public class ReferenceParserTest extends JavaParsingTestCase {
public void testTypeParams7() { doTypeParamsParserTest("<T extends X, Y>"); }
private void doRefParserTest(final String text, final boolean incomplete) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
ReferenceParser.parseJavaCodeReference(builder, incomplete, false, false);
}
@@ -55,7 +59,7 @@ public class ReferenceParserTest extends JavaParsingTestCase {
}
private void doTypeParserTest(final String text, final boolean incomplete) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
ReferenceParser.parseTypeWithEllipsis(builder, incomplete, false);
}
@@ -63,7 +67,7 @@ public class ReferenceParserTest extends JavaParsingTestCase {
}
private void doTypeParamsParserTest(final String text) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
ReferenceParser.parseTypeParameters(builder);
}
java/java-tests/testSrc/com/intellij/lang/java/parser/partial/StatementParserTest.java
+1 -1
View File
@@ -133,7 +133,7 @@ public class StatementParserTest extends JavaParsingTestCase {
public void testWhileIncomplete5() { doParserTest("{ while() foo(); }"); }
private void doParserTest(final String text) {
doParserTest(text, new Parser() {
doParserTest(text, new TestParser() {
public void parse(final PsiBuilder builder) {
StatementParser.parseCodeBlockDeep(builder, true);
}