diff --git a/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java b/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java index 3f0d6f04386b..de5fb9cb590a 100644 --- a/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java +++ b/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java @@ -203,7 +203,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { private abstract static class ProductionMarker extends Node { public int myLexemeIndex; public WhitespacesAndCommentsProcessor myEdgeProcessor; - ProductionMarker next; + public ProductionMarker next; public void clean() { myLexemeIndex = 0; @@ -729,7 +729,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { public ASTNode getTreeBuilt() { try { - StartMarker rootMarker = prepareLightTree(); + final StartMarker rootMarker = prepareLightTree(); if (myOriginalTree != null) { merge(myOriginalTree, rootMarker); @@ -756,7 +756,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { } public FlyweightCapableTreeStructure getLightTree() { - StartMarker rootMarker = prepareLightTree(); + final StartMarker rootMarker = prepareLightTree(); return new MyTreeStructure(rootMarker); } @@ -823,6 +823,8 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { } private StartMarker prepareLightTree() { + final StartMarker rootMarker = (StartMarker)myProduction.get(0); + markTokenTypeChecked(); for (int i = 1; i < myProduction.size() - 1; i++) { @@ -843,20 +845,20 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { item.myLexemeIndex = wsStartIndex + item.myEdgeProcessor.process(wsTokens); } - StartMarker rootMarker = (StartMarker)myProduction.get(0); + rootMarker.firstChild = rootMarker.lastChild = rootMarker.next = null; StartMarker curNode = rootMarker; - - Stack nodes = new Stack(); + final Stack nodes = new Stack(); nodes.push(rootMarker); int lastErrorIndex = -1; for (int i = 1; i < myProduction.size(); i++) { - ProductionMarker item = myProduction.get(i); + final ProductionMarker item = myProduction.get(i); if (curNode == null) LOG.error("Unexpected end of the production"); if (item instanceof StartMarker) { StartMarker marker = (StartMarker)item; + marker.firstChild = marker.lastChild = marker.next = null; curNode.addChild(marker); nodes.push(curNode); curNode = marker; @@ -1084,6 +1086,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { } private int count; + public int getChildren(@NotNull final LighterASTNode item, @NotNull final Ref into) { if (item instanceof Token || item instanceof ErrorItem) return 0; StartMarker marker = (StartMarker)item; @@ -1091,11 +1094,26 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { count = 0; ProductionMarker child = marker.firstChild; + ProductionMarker prevChild = null; int lexIndex = marker.myLexemeIndex; while (child != null) { lexIndex = insertLeafs(lexIndex, child.myLexemeIndex, into); - ensureCapacity(into); - into.get()[count++] = child; + + if (child instanceof StartMarker && ((StartMarker)child).myDoneMarker.myCollapse) { + final int start = myLexStarts[child.myLexemeIndex]; + final int end = myLexStarts[((StartMarker)child).myDoneMarker.myLexemeIndex]; + insertLeaf(into, start, end, child.getTokenType()); + + if (prevChild != null) prevChild.next = child.next; + if (marker.firstChild == child) marker.firstChild = child.next; + if (marker.lastChild == child) marker.lastChild = prevChild; + } + else { + ensureCapacity(into); + into.get()[count++] = child; + prevChild = child; + } + if (child instanceof StartMarker) { lexIndex = ((StartMarker)child).myDoneMarker.myLexemeIndex; } @@ -1119,7 +1137,6 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { } } - private int insertLeafs(int curToken, int lastIdx, Ref into) { lastIdx = Math.min(lastIdx, myLexemeCount); while (curToken < lastIdx) { @@ -1127,18 +1144,21 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { final int end = myLexStarts[curToken + 1]; final IElementType type = myLexTypes[curToken]; if (start < end || type instanceof ILeafElementType) { // Empty token. Most probably a parser directive like indent/dedent in Python - Token lexeme = myPool.alloc(); - lexeme.myTokenType = type; - lexeme.myTokenStart = start; - lexeme.myTokenEnd = end; - ensureCapacity(into); - into.get()[count++] = lexeme; + insertLeaf(into, start, end, type); } curToken++; } - return curToken; } + + private void insertLeaf(Ref into, int start, int end, IElementType type) { + Token lexeme = myPool.alloc(); + lexeme.myTokenType = type; + lexeme.myTokenStart = start; + lexeme.myTokenEnd = end; + ensureCapacity(into); + into.get()[count++] = lexeme; + } } private class ASTConverter implements Convertor { diff --git a/platform/lang-impl/src/com/intellij/psi/impl/DebugUtil.java b/platform/lang-impl/src/com/intellij/psi/impl/DebugUtil.java index afe54a7f0da1..bfb06a8cd5b2 100644 --- a/platform/lang-impl/src/com/intellij/psi/impl/DebugUtil.java +++ b/platform/lang-impl/src/com/intellij/psi/impl/DebugUtil.java @@ -31,6 +31,7 @@ import com.intellij.psi.impl.source.SourceTreeToPsiMap; import com.intellij.psi.impl.source.tree.CompositeElement; import com.intellij.psi.impl.source.tree.SharedImplUtil; import com.intellij.psi.impl.source.tree.TreeElement; +import com.intellij.psi.tree.IElementType; import com.intellij.util.ArrayUtil; import com.intellij.util.CharTable; import com.intellij.util.diff.FlyweightCapableTreeStructure; @@ -108,10 +109,7 @@ public class DebugUtil { } } else { - String text = root.getText(); - text = StringUtil.replace(text, "\n", "\\n"); - text = StringUtil.replace(text, "\r", "\\r"); - text = StringUtil.replace(text, "\t", "\\t"); + final String text = fixWhiteSpaces(root.getText()); buffer.append(root.toString()).append("('").append(text).append("')"); } if (showRanges) buffer.append(root.getTextRange()); @@ -132,6 +130,57 @@ public class DebugUtil { } } + public static String lightTreeToString(@NotNull final FlyweightCapableTreeStructure tree, + @NotNull final String source, + final boolean skipWhitespaces) { + StringBuilder buffer = new StringBuilder(); + lightTreeToBuffer(tree, source, buffer, tree.getRoot(), 0, skipWhitespaces); + return buffer.toString(); + } + + public static void lightTreeToBuffer(@NotNull final FlyweightCapableTreeStructure tree, + @NotNull final String source, + @NotNull final StringBuilder buffer, + @NotNull final LighterASTNode root, + final int indent, + final boolean skipWhiteSpaces) { + final IElementType tokenType = root.getTokenType(); + if (skipWhiteSpaces && tokenType == TokenType.WHITE_SPACE) return; + + final Ref kids = new Ref(); + final int numKids = tree.getChildren(tree.prepareForGetChildren(root), kids); + final boolean composite = numKids > 0 || root.getStartOffset() == root.getEndOffset(); + + StringUtil.repeatSymbol(buffer, ' ', indent); + if (tokenType == TokenType.ERROR_ELEMENT) { + buffer.append("PsiErrorElement"); // todo: error message text + } + else if (tokenType == TokenType.WHITE_SPACE) { + buffer.append("PsiWhiteSpace"); + } + else { + buffer.append(composite ? "Element" : "PsiElement").append('(').append(tokenType).append(')'); + } + + if (!composite) { + final String text = source.substring(root.getStartOffset(), root.getEndOffset()); + buffer.append("('").append(fixWhiteSpaces(text)).append("')"); + } + buffer.append("\n"); + + if (composite) { + if (numKids == 0) { + StringUtil.repeatSymbol(buffer, ' ', indent + 2); + buffer.append("\n"); + } + else { + for (int i = 0; i < numKids; i++) { + lightTreeToBuffer(tree, source, buffer, kids.get()[i], indent + 2, skipWhiteSpaces); + } + } + } + } + private static void treeToBufferWithUserData(StringBuilder buffer, TreeElement root, int indent, boolean skipWhiteSpaces) { if (skipWhiteSpaces && root.getElementType() == TokenType.WHITE_SPACE) return; @@ -140,10 +189,7 @@ public class DebugUtil { buffer.append(SourceTreeToPsiMap.treeElementToPsi(root).toString()); } else { - String text = root.getText(); - text = StringUtil.replace(text, "\n", "\\n"); - text = StringUtil.replace(text, "\r", "\\r"); - text = StringUtil.replace(text, "\t", "\\t"); + final String text = fixWhiteSpaces(root.getText()); buffer.append(root.toString()).append("('").append(text).append("')"); } buffer.append(root.getUserDataString()); @@ -170,10 +216,7 @@ public class DebugUtil { buffer.append(root); } else { - String text = root.getText(); - text = StringUtil.replace(text, "\n", "\\n"); - text = StringUtil.replace(text, "\r", "\\r"); - text = StringUtil.replace(text, "\t", "\\t"); + final String text = fixWhiteSpaces(root.getText()); buffer.append(root.toString()).append("('").append(text).append("')"); } buffer.append(((UserDataHolderBase)root).getUserDataString()); @@ -264,11 +307,12 @@ public class DebugUtil { boolean skipWhiteSpaces, boolean showRanges) { final StringBuilder result = new StringBuilder(); - if (root.getNode() == null) { + final ASTNode node = root.getNode(); + if (node == null) { psiToBuffer(result, root, 0, skipWhiteSpaces, showRanges, showRanges); } else { - treeToBuffer(result, root.getNode(), 0, skipWhiteSpaces, showRanges, showRanges, true); + treeToBuffer(result, node, 0, skipWhiteSpaces, showRanges, showRanges, true); } return result.toString(); } @@ -286,14 +330,9 @@ public class DebugUtil { buffer.append(rootStr); PsiElement child = root.getFirstChild(); if (child == null) { - String text = root.getText(); + final String text = root.getText(); assert text != null : "text is null for <" + root + ">"; - text = StringUtil.replace(text, "\n", "\\n"); - text = StringUtil.replace(text, "\r", "\\r"); - text = StringUtil.replace(text, "\t", "\\t"); - buffer.append("('"); - buffer.append(text); - buffer.append("')"); + buffer.append("('").append(fixWhiteSpaces(text)).append("')"); } if (showRanges) buffer.append(root.getTextRange()); @@ -304,6 +343,12 @@ public class DebugUtil { } } + public static String fixWhiteSpaces(String text) { + text = StringUtil.replace(text, "\n", "\\n"); + text = StringUtil.replace(text, "\r", "\\r"); + text = StringUtil.replace(text, "\t", "\\t"); + return text; + } public static class IncorrectTreeStructureException extends RuntimeException { private final ASTNode myElement; diff --git a/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java b/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java index b78ae338b436..54bfdcb7e85c 100644 --- a/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java +++ b/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java @@ -17,9 +17,11 @@ package com.intellij.lang; import com.intellij.lang.impl.PsiBuilderImpl; import com.intellij.lexer.LexerBase; +import com.intellij.psi.TokenType; import com.intellij.psi.impl.DebugUtil; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; +import com.intellij.util.diff.FlyweightCapableTreeStructure; import com.sun.tools.internal.xjc.util.NullStream; import org.junit.Test; @@ -34,7 +36,6 @@ public class LightPsiBuilderTest { private static final IElementType ROOT = new IElementType("ROOT", Language.ANY); private static final IElementType LETTER = new IElementType("LETTER", Language.ANY); private static final IElementType DIGIT = new IElementType("DIGIT", Language.ANY); - private static final IElementType WHITESPACE = new IElementType("WHITESPACE", Language.ANY); private static final IElementType OTHER = new IElementType("OTHER", Language.ANY); private static final IElementType COLLAPSED = new IElementType("COLLAPSED", Language.ANY); private static final IElementType LEFT_BOUND = new IElementType("LEFT_BOUND", Language.ANY) { @@ -42,7 +43,7 @@ public class LightPsiBuilderTest { }; private static final IElementType COMMENT = new IElementType("COMMENT", Language.ANY); - private static final TokenSet WHITESPACE_SET = TokenSet.create(WHITESPACE); + private static final TokenSet WHITESPACE_SET = TokenSet.create(TokenType.WHITE_SPACE); private static final TokenSet COMMENT_SET = TokenSet.create(COMMENT); @Test @@ -65,20 +66,23 @@ public class LightPsiBuilderTest { @Test public void testCollapse() { - doTest("a<>b", new Parser() { public void parse(PsiBuilder builder) { - PsiBuilder.Marker inner = null; - while (builder.getTokenType() != null) { - if (builder.getTokenType() == OTHER && inner == null) inner = builder.mark(); - builder.advanceLexer(); - if (builder.getTokenType() != OTHER && inner != null) { inner.collapse(COLLAPSED); inner = null; } - } + PsiBuilderUtil.advance(builder, 1); + final PsiBuilder.Marker marker1 = builder.mark(); + PsiBuilderUtil.advance(builder, 2); + marker1.collapse(COLLAPSED); + final PsiBuilder.Marker marker2 = builder.mark(); + PsiBuilderUtil.advance(builder, 2); + marker2.collapse(COLLAPSED); + PsiBuilderUtil.advance(builder, 1); } }, "Element(ROOT)\n" + " PsiElement(LETTER)('a')\n" + " PsiElement(COLLAPSED)('<<')\n" + + " PsiElement(COLLAPSED)('>>')\n" + " PsiElement(LETTER)('b')\n" ); } @@ -333,6 +337,11 @@ public class LightPsiBuilderTest { final PsiBuilder.Marker rootMarker = builder.mark(); parser.parse(builder); rootMarker.done(ROOT); + + final FlyweightCapableTreeStructure lightTree = builder.getLightTree(); + final String lightExpected = expected.replaceAll("PsiErrorElement:.*\n", "PsiErrorElement\n"); + assertEquals(lightExpected, DebugUtil.lightTreeToString(lightTree, text, false)); + final ASTNode root = builder.getTreeBuilt(); assertEquals(expected, DebugUtil.nodeTreeToString(root, false)); } @@ -376,7 +385,7 @@ public class LightPsiBuilderTest { if (myIndex >= myBufferEnd) return null; else if (Character.isLetter(myBuffer.charAt(myIndex))) return LETTER; else if (Character.isDigit(myBuffer.charAt(myIndex))) return DIGIT; - else if (Character.isWhitespace(myBuffer.charAt(myIndex))) return WHITESPACE; + else if (Character.isWhitespace(myBuffer.charAt(myIndex))) return TokenType.WHITE_SPACE; else if (myBuffer.charAt(myIndex) == '#') return COMMENT; else return OTHER; }