MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Proper handling for collapsed nodes in PSI builder

Browse Source
This commit is contained in:
Roman Shevchenko
2010-08-18 19:45:46 +04:00
parent ecd9754936
commit 0f588f1a4e
3 changed files with 122 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java
+37 -17
View File
@@ -203,7 +203,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
private abstract static class ProductionMarker extends Node {
public int myLexemeIndex;
public WhitespacesAndCommentsProcessor myEdgeProcessor;
ProductionMarker next;
public ProductionMarker next;
public void clean() {
myLexemeIndex = 0;
@@ -729,7 +729,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
public ASTNode getTreeBuilt() {
try {
StartMarker rootMarker = prepareLightTree();
final StartMarker rootMarker = prepareLightTree();
if (myOriginalTree != null) {
merge(myOriginalTree, rootMarker);
@@ -756,7 +756,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
}
public FlyweightCapableTreeStructure<LighterASTNode> getLightTree() {
StartMarker rootMarker = prepareLightTree();
final StartMarker rootMarker = prepareLightTree();
return new MyTreeStructure(rootMarker);
}
@@ -823,6 +823,8 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
}
private StartMarker prepareLightTree() {
final StartMarker rootMarker = (StartMarker)myProduction.get(0);
markTokenTypeChecked();
for (int i = 1; i < myProduction.size() - 1; i++) {
@@ -843,20 +845,20 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
item.myLexemeIndex = wsStartIndex + item.myEdgeProcessor.process(wsTokens);
}
StartMarker rootMarker = (StartMarker)myProduction.get(0);
rootMarker.firstChild = rootMarker.lastChild = rootMarker.next = null;
StartMarker curNode = rootMarker;
Stack<StartMarker> nodes = new Stack<StartMarker>();
final Stack<StartMarker> nodes = new Stack<StartMarker>();
nodes.push(rootMarker);
int lastErrorIndex = -1;
for (int i = 1; i < myProduction.size(); i++) {
ProductionMarker item = myProduction.get(i);
final ProductionMarker item = myProduction.get(i);
if (curNode == null) LOG.error("Unexpected end of the production");
if (item instanceof StartMarker) {
StartMarker marker = (StartMarker)item;
marker.firstChild = marker.lastChild = marker.next = null;
curNode.addChild(marker);
nodes.push(curNode);
curNode = marker;
@@ -1084,6 +1086,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
}
private int count;
public int getChildren(@NotNull final LighterASTNode item, @NotNull final Ref<LighterASTNode[]> into) {
if (item instanceof Token || item instanceof ErrorItem) return 0;
StartMarker marker = (StartMarker)item;
@@ -1091,11 +1094,26 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
count = 0;
ProductionMarker child = marker.firstChild;
ProductionMarker prevChild = null;
int lexIndex = marker.myLexemeIndex;
while (child != null) {
lexIndex = insertLeafs(lexIndex, child.myLexemeIndex, into);
ensureCapacity(into);
into.get()[count++] = child;
if (child instanceof StartMarker && ((StartMarker)child).myDoneMarker.myCollapse) {
final int start = myLexStarts[child.myLexemeIndex];
final int end = myLexStarts[((StartMarker)child).myDoneMarker.myLexemeIndex];
insertLeaf(into, start, end, child.getTokenType());
if (prevChild != null) prevChild.next = child.next;
if (marker.firstChild == child) marker.firstChild = child.next;
if (marker.lastChild == child) marker.lastChild = prevChild;
}
else {
ensureCapacity(into);
into.get()[count++] = child;
prevChild = child;
}
if (child instanceof StartMarker) {
lexIndex = ((StartMarker)child).myDoneMarker.myLexemeIndex;
}
@@ -1119,7 +1137,6 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
}
}
private int insertLeafs(int curToken, int lastIdx, Ref<LighterASTNode[]> into) {
lastIdx = Math.min(lastIdx, myLexemeCount);
while (curToken < lastIdx) {
@@ -1127,18 +1144,21 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
final int end = myLexStarts[curToken + 1];
final IElementType type = myLexTypes[curToken];
if (start < end || type instanceof ILeafElementType) { // Empty token. Most probably a parser directive like indent/dedent in Python
Token lexeme = myPool.alloc();
lexeme.myTokenType = type;
lexeme.myTokenStart = start;
lexeme.myTokenEnd = end;
ensureCapacity(into);
into.get()[count++] = lexeme;
insertLeaf(into, start, end, type);
}
curToken++;
}
return curToken;
}
private void insertLeaf(Ref<LighterASTNode[]> into, int start, int end, IElementType type) {
Token lexeme = myPool.alloc();
lexeme.myTokenType = type;
lexeme.myTokenStart = start;
lexeme.myTokenEnd = end;
ensureCapacity(into);
into.get()[count++] = lexeme;
}
}
private class ASTConverter implements Convertor<Node, ASTNode> {
platform/lang-impl/src/com/intellij/psi/impl/DebugUtil.java
+66 -21
View File
@@ -31,6 +31,7 @@ import com.intellij.psi.impl.source.SourceTreeToPsiMap;
import com.intellij.psi.impl.source.tree.CompositeElement;
import com.intellij.psi.impl.source.tree.SharedImplUtil;
import com.intellij.psi.impl.source.tree.TreeElement;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.ArrayUtil;
import com.intellij.util.CharTable;
import com.intellij.util.diff.FlyweightCapableTreeStructure;
@@ -108,10 +109,7 @@ public class DebugUtil {
}
}
else {
String text = root.getText();
text = StringUtil.replace(text, "\n", "\\n");
text = StringUtil.replace(text, "\r", "\\r");
text = StringUtil.replace(text, "\t", "\\t");
final String text = fixWhiteSpaces(root.getText());
buffer.append(root.toString()).append("('").append(text).append("')");
}
if (showRanges) buffer.append(root.getTextRange());
@@ -132,6 +130,57 @@ public class DebugUtil {
}
}
public static String lightTreeToString(@NotNull final FlyweightCapableTreeStructure<LighterASTNode> tree,
@NotNull final String source,
final boolean skipWhitespaces) {
StringBuilder buffer = new StringBuilder();
lightTreeToBuffer(tree, source, buffer, tree.getRoot(), 0, skipWhitespaces);
return buffer.toString();
}
public static void lightTreeToBuffer(@NotNull final FlyweightCapableTreeStructure<LighterASTNode> tree,
@NotNull final String source,
@NotNull final StringBuilder buffer,
@NotNull final LighterASTNode root,
final int indent,
final boolean skipWhiteSpaces) {
final IElementType tokenType = root.getTokenType();
if (skipWhiteSpaces && tokenType == TokenType.WHITE_SPACE) return;
final Ref<LighterASTNode[]> kids = new Ref<LighterASTNode[]>();
final int numKids = tree.getChildren(tree.prepareForGetChildren(root), kids);
final boolean composite = numKids > 0 || root.getStartOffset() == root.getEndOffset();
StringUtil.repeatSymbol(buffer, ' ', indent);
if (tokenType == TokenType.ERROR_ELEMENT) {
buffer.append("PsiErrorElement"); // todo: error message text
}
else if (tokenType == TokenType.WHITE_SPACE) {
buffer.append("PsiWhiteSpace");
}
else {
buffer.append(composite ? "Element" : "PsiElement").append('(').append(tokenType).append(')');
}
if (!composite) {
final String text = source.substring(root.getStartOffset(), root.getEndOffset());
buffer.append("('").append(fixWhiteSpaces(text)).append("')");
}
buffer.append("\n");
if (composite) {
if (numKids == 0) {
StringUtil.repeatSymbol(buffer, ' ', indent + 2);
buffer.append("<empty list>\n");
}
else {
for (int i = 0; i < numKids; i++) {
lightTreeToBuffer(tree, source, buffer, kids.get()[i], indent + 2, skipWhiteSpaces);
}
}
}
}
private static void treeToBufferWithUserData(StringBuilder buffer, TreeElement root, int indent, boolean skipWhiteSpaces) {
if (skipWhiteSpaces && root.getElementType() == TokenType.WHITE_SPACE) return;
@@ -140,10 +189,7 @@ public class DebugUtil {
buffer.append(SourceTreeToPsiMap.treeElementToPsi(root).toString());
}
else {
String text = root.getText();
text = StringUtil.replace(text, "\n", "\\n");
text = StringUtil.replace(text, "\r", "\\r");
text = StringUtil.replace(text, "\t", "\\t");
final String text = fixWhiteSpaces(root.getText());
buffer.append(root.toString()).append("('").append(text).append("')");
}
buffer.append(root.getUserDataString());
@@ -170,10 +216,7 @@ public class DebugUtil {
buffer.append(root);
}
else {
String text = root.getText();
text = StringUtil.replace(text, "\n", "\\n");
text = StringUtil.replace(text, "\r", "\\r");
text = StringUtil.replace(text, "\t", "\\t");
final String text = fixWhiteSpaces(root.getText());
buffer.append(root.toString()).append("('").append(text).append("')");
}
buffer.append(((UserDataHolderBase)root).getUserDataString());
@@ -264,11 +307,12 @@ public class DebugUtil {
boolean skipWhiteSpaces,
boolean showRanges) {
final StringBuilder result = new StringBuilder();
if (root.getNode() == null) {
final ASTNode node = root.getNode();
if (node == null) {
psiToBuffer(result, root, 0, skipWhiteSpaces, showRanges, showRanges);
}
else {
treeToBuffer(result, root.getNode(), 0, skipWhiteSpaces, showRanges, showRanges, true);
treeToBuffer(result, node, 0, skipWhiteSpaces, showRanges, showRanges, true);
}
return result.toString();
}
@@ -286,14 +330,9 @@ public class DebugUtil {
buffer.append(rootStr);
PsiElement child = root.getFirstChild();
if (child == null) {
String text = root.getText();
final String text = root.getText();
assert text != null : "text is null for <" + root + ">";
text = StringUtil.replace(text, "\n", "\\n");
text = StringUtil.replace(text, "\r", "\\r");
text = StringUtil.replace(text, "\t", "\\t");
buffer.append("('");
buffer.append(text);
buffer.append("')");
buffer.append("('").append(fixWhiteSpaces(text)).append("')");
}
if (showRanges) buffer.append(root.getTextRange());
@@ -304,6 +343,12 @@ public class DebugUtil {
}
}
public static String fixWhiteSpaces(String text) {
text = StringUtil.replace(text, "\n", "\\n");
text = StringUtil.replace(text, "\r", "\\r");
text = StringUtil.replace(text, "\t", "\\t");
return text;
}
public static class IncorrectTreeStructureException extends RuntimeException {
private final ASTNode myElement;
platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java
+19 -10
View File
@@ -17,9 +17,11 @@ package com.intellij.lang;
import com.intellij.lang.impl.PsiBuilderImpl;
import com.intellij.lexer.LexerBase;
import com.intellij.psi.TokenType;
import com.intellij.psi.impl.DebugUtil;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.diff.FlyweightCapableTreeStructure;
import com.sun.tools.internal.xjc.util.NullStream;
import org.junit.Test;
@@ -34,7 +36,6 @@ public class LightPsiBuilderTest {
private static final IElementType ROOT = new IElementType("ROOT", Language.ANY);
private static final IElementType LETTER = new IElementType("LETTER", Language.ANY);
private static final IElementType DIGIT = new IElementType("DIGIT", Language.ANY);
private static final IElementType WHITESPACE = new IElementType("WHITESPACE", Language.ANY);
private static final IElementType OTHER = new IElementType("OTHER", Language.ANY);
private static final IElementType COLLAPSED = new IElementType("COLLAPSED", Language.ANY);
private static final IElementType LEFT_BOUND = new IElementType("LEFT_BOUND", Language.ANY) {
@@ -42,7 +43,7 @@ public class LightPsiBuilderTest {
};
private static final IElementType COMMENT = new IElementType("COMMENT", Language.ANY);
private static final TokenSet WHITESPACE_SET = TokenSet.create(WHITESPACE);
private static final TokenSet WHITESPACE_SET = TokenSet.create(TokenType.WHITE_SPACE);
private static final TokenSet COMMENT_SET = TokenSet.create(COMMENT);
@Test
@@ -65,20 +66,23 @@ public class LightPsiBuilderTest {
@Test
public void testCollapse() {
doTest("a<<b",
doTest("a<<>>b",
new Parser() {
public void parse(PsiBuilder builder) {
PsiBuilder.Marker inner = null;
while (builder.getTokenType() != null) {
if (builder.getTokenType() == OTHER && inner == null) inner = builder.mark();
builder.advanceLexer();
if (builder.getTokenType() != OTHER && inner != null) { inner.collapse(COLLAPSED); inner = null; }
}
PsiBuilderUtil.advance(builder, 1);
final PsiBuilder.Marker marker1 = builder.mark();
PsiBuilderUtil.advance(builder, 2);
marker1.collapse(COLLAPSED);
final PsiBuilder.Marker marker2 = builder.mark();
PsiBuilderUtil.advance(builder, 2);
marker2.collapse(COLLAPSED);
PsiBuilderUtil.advance(builder, 1);
}
},
"Element(ROOT)\n" +
" PsiElement(LETTER)('a')\n" +
" PsiElement(COLLAPSED)('<<')\n" +
" PsiElement(COLLAPSED)('>>')\n" +
" PsiElement(LETTER)('b')\n"
);
}
@@ -333,6 +337,11 @@ public class LightPsiBuilderTest {
final PsiBuilder.Marker rootMarker = builder.mark();
parser.parse(builder);
rootMarker.done(ROOT);
final FlyweightCapableTreeStructure<LighterASTNode> lightTree = builder.getLightTree();
final String lightExpected = expected.replaceAll("PsiErrorElement:.*\n", "PsiErrorElement\n");
assertEquals(lightExpected, DebugUtil.lightTreeToString(lightTree, text, false));
final ASTNode root = builder.getTreeBuilt();
assertEquals(expected, DebugUtil.nodeTreeToString(root, false));
}
@@ -376,7 +385,7 @@ public class LightPsiBuilderTest {
if (myIndex >= myBufferEnd) return null;
else if (Character.isLetter(myBuffer.charAt(myIndex))) return LETTER;
else if (Character.isDigit(myBuffer.charAt(myIndex))) return DIGIT;
else if (Character.isWhitespace(myBuffer.charAt(myIndex))) return WHITESPACE;
else if (Character.isWhitespace(myBuffer.charAt(myIndex))) return TokenType.WHITE_SPACE;
else if (myBuffer.charAt(myIndex) == '#') return COMMENT;
else return OTHER;
}