Collapsible nodes in PSI builder

platform/lang-api/src/com/intellij/lang/PsiBuilder.java

@@ -53,7 +53,7 @@ public interface PsiBuilder extends UserDataHolder {
   IElementType getTokenType();
 
   /**
-   * Sets optional remapper that can changs the type of freshly lexed tokens.
+   * Sets optional remapper that can change the type of freshly lexed tokens.
    * Output of getTokenType() is affected by it.
    * @param remapper the remapper object, or null.
    */
@@ -110,17 +110,25 @@ public interface PsiBuilder extends UserDataHolder {
      */
     void done(IElementType type);
 
+    /**
+     * Like done(), but collapses all tokens between start and end markers into single leaf node of given type.
+     *
+     * @param type the type of the node in the AST tree.
+     */
+    void collapse(IElementType type);
+
     /**
      * TODO doc
-     * @param type
-     * @param before
+     * @param type the type of the node in the AST tree.
+     * @param before marker to complete this one before.
      */
     void doneBefore(IElementType type, Marker before);
 
     /**
      * TODO doc
-     * @param type
-     * @param before
+     * @param type the type of the node in the AST tree.
+     * @param before marker to complete this one before.
+     * @param errorMessage for error element.
      */
     void doneBefore(IElementType type, Marker before, String errorMessage);
 
@@ -128,7 +136,7 @@ public interface PsiBuilder extends UserDataHolder {
      * Completes this marker and labels it as error element with specified message. Before calling this method,
      * all markers added after the beginning of this marker must be either dropped or completed.
      *
-     * @param message for error element
+     * @param message for error element.
      */
     void error(String message);
   }
@@ -162,7 +170,7 @@ public interface PsiBuilder extends UserDataHolder {
   ASTNode getTreeBuilt();
 
   /**
-   * Same as {@link #getTreeBuilt()} but returns a light tree, which is build faster, produces less garbage but is uncapable of creating a PSI over.
+   * Same as {@link #getTreeBuilt()} but returns a light tree, which is build faster, produces less garbage but is incapable of creating a PSI over.
    * @return the light tree built.
    */
   FlyweightCapableTreeStructure<LighterASTNode> getLightTree();

platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java

@@ -277,6 +277,11 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
       myBuilder.done(this);
     }
 
+    public void collapse(IElementType type) {
+      myType = type;
+      myBuilder.collapse(this);
+    }
+
     public void doneBefore(IElementType type, Marker before) {
       myType = type;
       myBuilder.doneBefore(this, before);
@@ -373,6 +378,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
 
   private static class DoneMarker extends ProductionMarker {
     public StartMarker myStart;
+    public boolean myCollapse = false;
 
     public DoneMarker() {}
 
@@ -593,10 +599,9 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
     myProduction.add(doneMarker);
   }
 
-  public void done(Marker marker) {
+  public void done(final Marker marker) {
     doValidityChecks(marker);
 
-
     DoneMarker doneMarker = DONE_MARKERS.alloc();
     doneMarker.myStart = (StartMarker)marker;
     doneMarker.myLexemeIndex = myCurrentLexeme;
@@ -605,6 +610,11 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
     myProduction.add(doneMarker);
   }
 
+  public void collapse(final Marker marker) {
+    done(marker);
+    ((StartMarker)marker).myDoneMarker.myCollapse = true;
+  }
+
   @SuppressWarnings({"UseOfSystemOutOrSystemErr", "SuspiciousMethodCalls"})
   private void doValidityChecks(final Marker marker) {
     if (myDebugMode) {
@@ -828,9 +838,14 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
 
         lexIndex = insertLeafs(lexIndex, childMarker.myLexemeIndex, ast);
 
-        CompositeElement childNode = createComposite(childMarker);
-        ast.rawAddChildren(childNode);
-        lexIndex = bind(childNode, childMarker, lexIndex);
+        if (!childMarker.myDoneMarker.myCollapse) {
+          CompositeElement childNode = createComposite(childMarker);
+          ast.rawAddChildren(childNode);
+          lexIndex = bind(childNode, childMarker, lexIndex);
+        }
+        else {
+          lexIndex = collapseLeafs(ast, childMarker);
+        }
 
         lexIndex = insertLeafs(lexIndex, childMarker.myDoneMarker.myLexemeIndex, ast);
       }
@@ -863,6 +878,14 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
     return curToken;
   }
 
+  private int collapseLeafs(CompositeElement ast, StartMarker startMarker) {
+    final int start = myLexStarts[startMarker.myLexemeIndex];
+    final int end = myLexStarts[startMarker.myDoneMarker.myLexemeIndex];
+    final TreeElement leaf = createLeaf(startMarker.myType, start, end);
+    ast.rawAddChildren(leaf);
+    return startMarker.myDoneMarker.myLexemeIndex;
+  }
+
   private static CompositeElement createComposite(final StartMarker marker) {
     final IElementType type = marker.myType;
     if (type == TokenType.ERROR_ELEMENT) {

platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2000-2010 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.lang;
+
+import com.intellij.lang.impl.PsiBuilderImpl;
+import com.intellij.lexer.LexerBase;
+import com.intellij.psi.impl.DebugUtil;
+import com.intellij.psi.tree.IElementType;
+import com.intellij.psi.tree.TokenSet;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class LightPsiBuilderTest {
+
+  @Test
+  public void testPlain() {
+    doTest(
+      "a<<b",
+      new PsiParser() {
+        @NotNull
+        public ASTNode parse(IElementType root, PsiBuilder builder) {
+          final PsiBuilder.Marker rootMarker = builder.mark();
+          while (builder.getTokenType() != null) {
+            builder.advanceLexer();
+          }
+          rootMarker.done(root);
+          return builder.getTreeBuilt();
+        }
+      },
+      "Element(ROOT)\n" +
+      "  PsiElement(TOKEN)('a')\n" +
+      "  PsiElement(TOKEN)('<')\n" +
+      "  PsiElement(TOKEN)('<')\n" +
+      "  PsiElement(TOKEN)('b')\n"
+    );
+  }
+
+  @Test
+  public void testCollapse() {
+    doTest(
+      "a<<b",
+      new PsiParser() {
+        @NotNull
+        public ASTNode parse(IElementType root, PsiBuilder builder) {
+          final PsiBuilder.Marker rootMarker = builder.mark();
+          PsiBuilder.Marker inner = null;
+          while (builder.getTokenType() != null) {
+            if ("<".equals(builder.getTokenText()) && inner == null) inner = builder.mark();
+            builder.advanceLexer();
+            if (!"<".equals(builder.getTokenText()) && inner != null) { inner.collapse(new IElementType("COLLAPSE", Language.ANY)); inner = null; }
+          }
+          rootMarker.done(root);
+          return builder.getTreeBuilt();
+        }
+      },
+      "Element(ROOT)\n" +
+      "  PsiElement(TOKEN)('a')\n" +
+      "  PsiElement(COLLAPSE)('<<')\n" +
+      "  PsiElement(TOKEN)('b')\n"
+    );
+  }
+
+
+  private static void doTest(final String text, final PsiParser parser, final String expected) {
+    final PsiBuilder builder = new PsiBuilderImpl(new MyTestLexer(), TokenSet.EMPTY, TokenSet.EMPTY, text);
+    final ASTNode root = parser.parse(new IElementType("ROOT", Language.ANY), builder);
+    assertEquals(expected, DebugUtil.nodeTreeToString(root, true));
+  }
+
+  private static class MyTestLexer extends LexerBase {
+    public static IElementType TOKEN = new IElementType("TOKEN", Language.ANY);
+
+    private CharSequence myBuffer = "";
+    private int myIndex = 0;
+    private int myBufferEnd = 1;
+
+    public void start(CharSequence buffer, int startOffset, int endOffset, int initialState) {
+      myBuffer = buffer.subSequence(startOffset, endOffset);
+      myIndex = 0;
+      myBufferEnd = myBuffer.length();
+    }
+
+    public int getState() {
+      return 0;
+    }
+
+    public IElementType getTokenType() {
+      return myIndex < myBufferEnd ? TOKEN : null;
+    }
+
+    public int getTokenStart() {
+      return myIndex;
+    }
+
+    public int getTokenEnd() {
+      return myIndex + 1;
+    }
+
+    public void advance() {
+      if (myIndex < myBufferEnd) myIndex++;
+    }
+
+    public CharSequence getBufferSequence() {
+      return myBuffer;
+    }
+
+    public int getBufferEnd() {
+      return myBufferEnd;
+    }
+  }
+}