New Java parser (work in progress)

java/java-impl/src/com/intellij/lang/java/parser/DeclarationParser.java

@@ -18,6 +18,7 @@ package com.intellij.lang.java.parser;
 import com.intellij.codeInsight.daemon.JavaErrorMessages;
 import com.intellij.lang.LighterASTNode;
 import com.intellij.lang.PsiBuilder;
+import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.psi.JavaTokenType;
 import com.intellij.psi.impl.source.tree.ElementType;
 import com.intellij.psi.impl.source.tree.JavaElementType;
@@ -36,6 +37,8 @@ public class DeclarationParser {
     FILE, CLASS, CODE_BLOCK, ANNOTATION_INTERFACE
   }
 
+  private static final Logger LOG = Logger.getInstance("#com.intellij.lang.java.parser.DeclarationParser");
+
   private static final TokenSet AFTER_END_DECLARATION_SET = TokenSet.create(JavaElementType.FIELD, JavaElementType.METHOD);
 
   private DeclarationParser() { }
@@ -48,10 +51,11 @@ public class DeclarationParser {
     marker.drop();
     builder.advanceLexer();
 
+    final PsiBuilder builderWrapper = braceMatchingBuilder(builder);
     if (isEnum) {
-      parseEnumConstants(builder);
+      parseEnumConstants(builderWrapper);
     }
-    parseClassBodyDeclarations(builder, isAnnotation);
+    parseClassBodyDeclarations(builderWrapper, isAnnotation);
 
     expectOrError(builder, JavaTokenType.RBRACE, JavaErrorMessages.message("expected.rbrace"));
 
@@ -189,7 +193,7 @@ public class DeclarationParser {
     final PsiBuilder.Marker modList = parseModifierList(builder);
 
     if (expect(builder, JavaTokenType.AT)) {
-      if (tokenType == JavaTokenType.INTERFACE_KEYWORD) {
+      if (builder.getTokenType() == JavaTokenType.INTERFACE_KEYWORD) {
         return parseClassFromKeyword(builder, declaration, true);
       }
       else {
@@ -197,9 +201,8 @@ public class DeclarationParser {
         return null;
       }
     }
-    else if (ElementType.CLASS_KEYWORD_BIT_SET.contains(tokenType)) {
+    else if (ElementType.CLASS_KEYWORD_BIT_SET.contains(builder.getTokenType())) {
       final PsiBuilder.Marker root = parseClassFromKeyword(builder, declaration, false);
-
       if (context == Context.FILE) {
         // todo: append following declarations to root
         boolean declarationsAfterEnd = false;
@@ -233,18 +236,100 @@ public class DeclarationParser {
     }
 
     if (context == Context.FILE) {
-      if (typeParams == null) {
-        error(builder, JavaErrorMessages.message("expected.class.or.interface"));
-      }
-      else {
-        typeParams.precede().errorBefore(JavaErrorMessages.message("expected.class.or.interface"), typeParams);
-      }
+      error(builder, JavaErrorMessages.message("expected.class.or.interface"), typeParams);
       declaration.drop();
       return modList;
     }
 
-    // todo: implement
-    throw new UnsupportedOperationException(builder.toString() + context);
+    PsiBuilder.Marker type;
+    if (ElementType.PRIMITIVE_TYPE_BIT_SET.contains(builder.getTokenType())) {
+      type = parseTypeNotNull(builder);
+    }
+    else if (builder.getTokenType() == JavaTokenType.IDENTIFIER) {
+      final PsiBuilder.Marker idPos = builder.mark();
+      type = parseTypeNotNull(builder);
+      if (builder.getTokenType() == JavaTokenType.LPARENTH) {  // constructor
+        if (context == Context.CODE_BLOCK) {
+          declaration.rollbackTo();
+          return null;
+        }
+        idPos.rollbackTo();
+        if (typeParams == null) {
+          emptyElement(type, JavaElementType.TYPE_PARAMETER_LIST);
+        }
+        builder.advanceLexer();
+        if (builder.getTokenType() != JavaTokenType.LPARENTH) {
+          declaration.drop();
+          return null;
+        }
+        parseMethodFromLeftParenth(builder, declaration, false);
+        return declaration;
+      }
+      idPos.drop();
+    }
+    else if (builder.getTokenType() == JavaTokenType.LBRACE) {
+      if (context == Context.CODE_BLOCK) {
+        error(builder, JavaErrorMessages.message("expected.identifier.or.type"), typeParams);
+        declaration.drop();
+        return modList;
+      }
+
+      final PsiBuilder.Marker codeBlock = StatementParser.parseCodeBlock(builder);
+      LOG.assertTrue(codeBlock != null);
+
+      if (typeParams != null) {
+        final PsiBuilder.Marker error = typeParams.precede();
+        error.errorBefore(JavaErrorMessages.message("unexpected.token"), codeBlock);
+      }
+      declaration.done(JavaElementType.CLASS_INITIALIZER);
+      return declaration;
+    }
+    else {
+      final PsiBuilder.Marker error;
+      if (typeParams != null) {
+        error = typeParams.precede();
+      }
+      else {
+        error = builder.mark();
+      }
+      error.error(JavaErrorMessages.message("expected.identifier.or.type"));
+      return modList;
+    }
+
+    if (!expect(builder, JavaTokenType.IDENTIFIER)) {
+      if (context == Context.CODE_BLOCK /* todo: && modifierList.getFirstChildNode() == null */) {
+        declaration.rollbackTo();
+        return null;
+      }
+      else {
+        if (typeParams != null) {
+          final PsiBuilder.Marker error = typeParams.precede();
+          error.errorBefore(JavaErrorMessages.message("unexpected.token"), type);
+        }
+        builder.error(JavaErrorMessages.message("expected.identifier"));
+        declaration.drop();
+        return modList;
+      }
+    }
+
+    if (builder.getTokenType() == JavaTokenType.LPARENTH) {
+      if (context == Context.CLASS || context == Context.ANNOTATION_INTERFACE) {  // method
+        if (typeParams == null) {
+          emptyElement(type, JavaElementType.TYPE_PARAMETER_LIST);
+        }
+        parseMethodFromLeftParenth(builder, declaration, (context == Context.ANNOTATION_INTERFACE));
+        return declaration;
+      }
+    }
+
+    return parseFieldOrLocalVariable(builder, declaration);
+  }
+
+  @NotNull
+  private static PsiBuilder.Marker parseTypeNotNull(final PsiBuilder builder) {
+    final ReferenceParser.TypeInfo typeInfo = ReferenceParser.parseType(builder);
+    assert typeInfo != null : builder.getOriginalText();
+    return typeInfo.marker;
   }
 
   @NotNull
@@ -276,6 +361,17 @@ public class DeclarationParser {
     return modList;
   }
 
+  private static void parseMethodFromLeftParenth(final PsiBuilder builder, final PsiBuilder.Marker declaration, final boolean anno) {
+    // todo: implement
+    throw new UnsupportedOperationException(builder.toString() + declaration + anno);
+  }
+
+  @Nullable
+  private static PsiBuilder.Marker parseFieldOrLocalVariable(final PsiBuilder builder, final PsiBuilder.Marker declaration) {
+    // todo: implement
+    throw new UnsupportedOperationException(builder.toString() + declaration);
+  }
+
   @Nullable
   public static PsiBuilder.Marker parseAnnotations(final PsiBuilder builder) {
     PsiBuilder.Marker firstAnno = null;

java/java-impl/src/com/intellij/lang/java/parser/JavaParserUtil.java

@@ -15,12 +15,16 @@
  */
 package com.intellij.lang.java.parser;
 
-import com.intellij.lang.PsiBuilder;
-import com.intellij.lang.PsiBuilderUtil;
+import com.intellij.lang.*;
 import com.intellij.openapi.util.Key;
 import com.intellij.pom.java.LanguageLevel;
+import com.intellij.psi.JavaTokenType;
 import com.intellij.psi.tree.IElementType;
+import com.intellij.psi.tree.TokenSet;
+import com.intellij.util.diff.FlyweightCapableTreeStructure;
+import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 
 public class JavaParserUtil {
@@ -49,6 +53,15 @@ public class JavaParserUtil {
     builder.mark().error(message);
   }
 
+  public static void error(final PsiBuilder builder, final String message, @Nullable final PsiBuilder.Marker before) {
+    if (before == null) {
+      error(builder, message);
+    }
+    else {
+      before.precede().errorBefore(message, before);
+    }
+  }
+
   public static boolean expectOrError(final PsiBuilder builder, final IElementType expectedType, final String errorMessage) {
     if (!PsiBuilderUtil.expect(builder, expectedType)) {
       error(builder, errorMessage);
@@ -60,4 +73,106 @@ public class JavaParserUtil {
   public static void emptyElement(final PsiBuilder builder, final IElementType type) {
     builder.mark().done(type);
   }
+
+  public static void emptyElement(final PsiBuilder.Marker before, final IElementType type) {
+    before.precede().done(type);
+  }
+
+  public static PsiBuilder braceMatchingBuilder(final PsiBuilder builder) {
+    return new PsiBuilderAdapter(builder) {
+      private int braceCount = 1;
+      private int lastOffset = -1;
+
+      @Override
+      public IElementType getTokenType() {
+        final IElementType tokenType = super.getTokenType();
+        if (getCurrentOffset() != lastOffset) {
+          if (tokenType == JavaTokenType.LBRACE) {
+            braceCount++;
+          }
+          else if (tokenType == JavaTokenType.RBRACE) {
+            braceCount--;
+          }
+          lastOffset = getCurrentOffset();
+        }
+        return (braceCount == 0 ? null : tokenType);
+      }
+    };
+  }
+
+  public static class PsiBuilderAdapter implements PsiBuilder {
+    protected final PsiBuilder myDelegate;
+
+    public PsiBuilderAdapter(final PsiBuilder delegate) {
+      myDelegate = delegate;
+    }
+
+    public CharSequence getOriginalText() {
+      return myDelegate.getOriginalText();
+    }
+
+    public void advanceLexer() {
+      myDelegate.advanceLexer();
+    }
+
+    @Nullable
+    public IElementType getTokenType() {
+      return myDelegate.getTokenType();
+    }
+
+    public void setTokenTypeRemapper(final ITokenTypeRemapper remapper) {
+      myDelegate.setTokenTypeRemapper(remapper);
+    }
+
+    @Nullable @NonNls
+    public String getTokenText() {
+      return myDelegate.getTokenText();
+    }
+
+    public int getCurrentOffset() {
+      return myDelegate.getCurrentOffset();
+    }
+
+    public Marker mark() {
+      return myDelegate.mark();
+    }
+
+    public void error(final String messageText) {
+      myDelegate.error(messageText);
+    }
+
+    public boolean eof() {
+      return myDelegate.eof();
+    }
+
+    public ASTNode getTreeBuilt() {
+      return myDelegate.getTreeBuilt();
+    }
+
+    public FlyweightCapableTreeStructure<LighterASTNode> getLightTree() {
+      return myDelegate.getLightTree();
+    }
+
+    public void setDebugMode(final boolean dbgMode) {
+      myDelegate.setDebugMode(dbgMode);
+    }
+
+    public void enforceCommentTokens(final TokenSet tokens) {
+      myDelegate.enforceCommentTokens(tokens);
+    }
+
+    @Nullable
+    public LighterASTNode getLatestDoneMarker() {
+      return myDelegate.getLatestDoneMarker();
+    }
+
+    @Nullable
+    public <T> T getUserData(@NotNull final Key<T> key) {
+      return myDelegate.getUserData(key);
+    }
+
+    public <T> void putUserData(@NotNull final Key<T> key, @Nullable final T value) {
+      myDelegate.putUserData(key, value);
+    }
+  }
 }

java/java-impl/src/com/intellij/lang/java/parser/StatementParser.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2000-2010 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.lang.java.parser;
+
+import com.intellij.lang.PsiBuilder;
+import org.jetbrains.annotations.Nullable;
+
+
+public class StatementParser {
+  private StatementParser() { }
+
+  @Nullable
+  public static PsiBuilder.Marker parseCodeBlock(final PsiBuilder builder) {
+    // todo: implement
+    throw new UnsupportedOperationException(builder.toString());
+  }
+}

java/java-impl/src/com/intellij/psi/impl/source/tree/JavaElementType.java

@@ -93,7 +93,7 @@ public interface JavaElementType {
   IElementType CLASS_OBJECT_ACCESS_EXPRESSION = new IJavaElementType("CLASS_OBJECT_ACCESS_EXPRESSION");
   IElementType EMPTY_EXPRESSION = new IJavaElementType("EMPTY_EXPRESSION");
 
-  IElementType EXPRESSION_LIST = new IJavaElementType("EXPRESSION_LIST");
+  IElementType EXPRESSION_LIST = new IJavaElementType("EXPRESSION_LIST", true);
 
   IElementType EMPTY_STATEMENT = new IJavaElementType("EMPTY_STATEMENT");
   IElementType BLOCK_STATEMENT = new IJavaElementType("BLOCK_STATEMENT");

java/java-tests/testData/psi/parser-partial/declarations/EmptyBody0.txt

@@ -0,0 +1,4 @@
+PsiJavaFile:EmptyBody0.java
+  PsiJavaToken:LBRACE('{')
+  PsiWhiteSpace(' ')
+  PsiJavaToken:RBRACE('}')

java/java-tests/testData/psi/parser-partial/declarations/EmptyBody1.txt

@@ -0,0 +1,5 @@
+PsiJavaFile:EmptyBody1.java
+  PsiJavaToken:LBRACE('{')
+  PsiErrorElement:'}' expected
+    <empty list>
+  PsiWhiteSpace(' ')

java/java-tests/testData/psi/parser-partial/declarations/EnumBody0.txt

@@ -0,0 +1,6 @@
+PsiJavaFile:EnumBody0.java
+  PsiJavaToken:LBRACE('{')
+  PsiWhiteSpace(' ')
+  PsiJavaToken:SEMICOLON(';')
+  PsiWhiteSpace(' ')
+  PsiJavaToken:RBRACE('}')

java/java-tests/testData/psi/parser-partial/declarations/EnumBody1.txt

@@ -0,0 +1,28 @@
+PsiJavaFile:EnumBody1.java
+  PsiJavaToken:LBRACE('{')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:RED
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:RED('RED')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:GREEN
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:GREEN('GREEN')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:BLUE
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:BLUE('BLUE')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:SEMICOLON(';')
+  PsiWhiteSpace(' ')
+  PsiJavaToken:RBRACE('}')

java/java-tests/testData/psi/parser-partial/declarations/EnumBody2.txt

@@ -0,0 +1,27 @@
+PsiJavaFile:EnumBody2.java
+  PsiJavaToken:LBRACE('{')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:RED
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:RED('RED')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:GREEN
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:GREEN('GREEN')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:BLUE
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:BLUE('BLUE')
+    PsiExpressionList
+      <empty list>
+  PsiWhiteSpace(' ')
+  PsiJavaToken:RBRACE('}')

java/java-tests/testData/psi/parser-partial/declarations/EnumBody3.txt

@@ -0,0 +1,28 @@
+PsiJavaFile:EnumBody3.java
+  PsiJavaToken:LBRACE('{')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:RED
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:RED('RED')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:GREEN
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:GREEN('GREEN')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:BLUE
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:BLUE('BLUE')
+    PsiExpressionList
+      <empty list>
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiJavaToken:RBRACE('}')

java/java-tests/testData/psi/parser-partial/declarations/EnumBody4.txt

@@ -0,0 +1,37 @@
+PsiJavaFile:EnumBody4.java
+  PsiJavaToken:LBRACE('{')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:RED
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:RED('RED')
+    PsiExpressionList
+      PsiJavaToken:LPARENTH('(')
+      PsiLiteralExpression:0
+        PsiJavaToken:INTEGER_LITERAL('0')
+      PsiJavaToken:RPARENTH(')')
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:GREEN
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:GREEN('GREEN')
+    PsiExpressionList
+      PsiJavaToken:LPARENTH('(')
+      PsiLiteralExpression:1
+        PsiJavaToken:INTEGER_LITERAL('1')
+      PsiJavaToken:RPARENTH(')')
+  PsiJavaToken:COMMA(',')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:BLUE
+    PsiModifierList:
+      <empty list>
+    PsiIdentifier:BLUE('BLUE')
+    PsiExpressionList
+      PsiJavaToken:LPARENTH('(')
+      PsiLiteralExpression:2
+        PsiJavaToken:INTEGER_LITERAL('2')
+      PsiJavaToken:RPARENTH(')')
+  PsiJavaToken:SEMICOLON(';')
+  PsiWhiteSpace(' ')
+  PsiJavaToken:RBRACE('}')

java/java-tests/testData/psi/parser-partial/declarations/EnumBody5.txt

@@ -0,0 +1,22 @@
+PsiJavaFile:EnumBody5.java
+  PsiJavaToken:LBRACE('{')
+  PsiWhiteSpace(' ')
+  PsiEnumConstant:A
+    PsiModifierList:@ANNOTATION
+      PsiAnnotation
+        PsiJavaToken:AT('@')
+        PsiJavaCodeReferenceElement:ANNOTATION
+          PsiIdentifier:ANNOTATION('ANNOTATION')
+          PsiReferenceParameterList
+            <empty list>
+        PsiAnnotationParameterList
+          <empty list>
+    PsiWhiteSpace(' ')
+    PsiIdentifier:A('A')
+    PsiExpressionList
+      PsiJavaToken:LPARENTH('(')
+      PsiLiteralExpression:10
+        PsiJavaToken:INTEGER_LITERAL('10')
+      PsiJavaToken:RPARENTH(')')
+  PsiWhiteSpace(' ')
+  PsiJavaToken:RBRACE('}')

java/java-tests/testSrc/com/intellij/lang/java/parser/partial/DeclarationParserTest.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2000-2010 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.lang.java.parser.partial;
+
+import com.intellij.lang.PsiBuilder;
+import com.intellij.lang.java.parser.DeclarationParser;
+import com.intellij.lang.java.parser.JavaParsingTestCase;
+
+
+public class DeclarationParserTest extends JavaParsingTestCase {
+  public DeclarationParserTest() {
+    super("parser-partial/declarations");
+  }
+
+  public void testEmptyBody0() { doParserTest("{ }", false, false); }
+  public void testEmptyBody1() { doParserTest("{ ", false, false); }
+
+  public void testEnumBody0() { doParserTest("{ ; }", false, true); }
+  public void testEnumBody1() { doParserTest("{ RED, GREEN, BLUE; }", false, true); }
+  public void testEnumBody2() { doParserTest("{ RED, GREEN, BLUE }", false, true); }
+  public void testEnumBody3() { doParserTest("{ RED, GREEN, BLUE, }", false, true); }
+  public void testEnumBody4() { doParserTest("{ RED(0), GREEN(1), BLUE(2); }", false, true); }
+  public void testEnumBody5() { doParserTest("{ @ANNOTATION A(10) }", false, true); }
+
+  private void doParserTest(final String text, final boolean isAnnotation, final boolean isEnum) {
+    doParserTest(text, new Parser() {
+      public void parse(final PsiBuilder builder) {
+        DeclarationParser.parseClassBodyWithBraces(builder, isAnnotation, isEnum);
+      }
+    });
+  }
+}