diff --git a/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java b/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java index 7a55ed3f5969..86cb6a17fbb6 100644 --- a/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java +++ b/platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java @@ -384,6 +384,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { private static class DoneMarker extends ProductionMarker { public StartMarker myStart; public boolean myCollapse = false; + public boolean myTieToTheLeft = false; public DoneMarker() {} @@ -567,20 +568,6 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { START_MARKERS.recycle((StartMarker)marker); } - @SuppressWarnings({"SuspiciousMethodCalls"}) - public void doneBefore(Marker marker, Marker before) { - doValidityChecks(marker, before); - - int beforeIndex = myProduction.lastIndexOf(before); - - DoneMarker doneMarker = DONE_MARKERS.alloc(); - doneMarker.myLexemeIndex = ((StartMarker)before).myLexemeIndex; - doneMarker.myStart = (StartMarker)marker; - - ((StartMarker)marker).myDoneMarker = doneMarker; - myProduction.add(beforeIndex, doneMarker); - } - @SuppressWarnings({"SuspiciousMethodCalls"}) public void drop(Marker marker) { final boolean removed = myProduction.remove(myProduction.lastIndexOf(marker)) == marker; @@ -594,6 +581,8 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { doValidityChecks(marker, null); DoneWithErrorMarker doneMarker = new DoneWithErrorMarker((StartMarker)marker, myCurrentLexeme, message); + doneMarker.myTieToTheLeft = isEmpty(((StartMarker)marker).myLexemeIndex, myCurrentLexeme); + ((StartMarker)marker).myDoneMarker = doneMarker; myProduction.add(doneMarker); } @@ -604,8 +593,9 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { int beforeIndex = myProduction.lastIndexOf(before); - DoneWithErrorMarker doneMarker = new DoneWithErrorMarker((StartMarker)marker, myCurrentLexeme, message); - doneMarker.myLexemeIndex = ((StartMarker)before).myLexemeIndex; + DoneWithErrorMarker doneMarker = new DoneWithErrorMarker((StartMarker)marker, ((StartMarker)before).myLexemeIndex, message); + doneMarker.myTieToTheLeft = isEmpty(((StartMarker)marker).myLexemeIndex, ((StartMarker)before).myLexemeIndex); + ((StartMarker)marker).myDoneMarker = doneMarker; myProduction.add(beforeIndex, doneMarker); } @@ -616,11 +606,37 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { DoneMarker doneMarker = DONE_MARKERS.alloc(); doneMarker.myStart = (StartMarker)marker; doneMarker.myLexemeIndex = myCurrentLexeme; + doneMarker.myTieToTheLeft = doneMarker.myStart.myType.isLeftBound() && + isEmpty(((StartMarker)marker).myLexemeIndex, myCurrentLexeme); ((StartMarker)marker).myDoneMarker = doneMarker; myProduction.add(doneMarker); } + @SuppressWarnings({"SuspiciousMethodCalls"}) + public void doneBefore(Marker marker, Marker before) { + doValidityChecks(marker, before); + + int beforeIndex = myProduction.lastIndexOf(before); + + DoneMarker doneMarker = DONE_MARKERS.alloc(); + doneMarker.myLexemeIndex = ((StartMarker)before).myLexemeIndex; + doneMarker.myStart = (StartMarker)marker; + doneMarker.myTieToTheLeft = doneMarker.myStart.myType.isLeftBound() && + isEmpty(((StartMarker)marker).myLexemeIndex, ((StartMarker)before).myLexemeIndex); + + ((StartMarker)marker).myDoneMarker = doneMarker; + myProduction.add(beforeIndex, doneMarker); + } + + private boolean isEmpty(final int startIdx, final int endIdx) { + for (int i = startIdx; i < endIdx; i++) { + final IElementType token = myLexTypes[i]; + if (!myWhitespaces.contains(token) && !myComments.contains(token)) return false; + } + return true; + } + public void collapse(final Marker marker) { done(marker); ((StartMarker)marker).myDoneMarker.myCollapse = true; @@ -779,25 +795,36 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder { for (int i = 1; i < fProduction.size() - 1; i++) { ProductionMarker item = fProduction.get(i); - if (item instanceof StartMarker) { + if (item instanceof StartMarker && ((StartMarker)item).myDoneMarker == null) { + LOG.error(UNBALANCED_MESSAGE); + } + + if (item instanceof StartMarker && ((StartMarker)item).myDoneMarker.myTieToTheLeft) { + int prevProductionLexIndex = fProduction.get(i - 1).myLexemeIndex; + IElementType prevTokenType; + while (item.myLexemeIndex > prevProductionLexIndex && + item.myLexemeIndex - 1 < myLexemeCount && + (myWhitespaces.contains(prevTokenType = myLexTypes[item.myLexemeIndex - 1]) || + myComments.contains(prevTokenType))) { + item.myLexemeIndex--; + } + ((StartMarker)item).myDoneMarker.myLexemeIndex = item.myLexemeIndex; + } + else if (item instanceof StartMarker) { IElementType nextTokenType; while (item.myLexemeIndex < myLexemeCount && - ( myWhitespaces.contains(nextTokenType = myLexTypes[item.myLexemeIndex]) || - myComments.contains(nextTokenType) - ) - ) { + (myWhitespaces.contains(nextTokenType = myLexTypes[item.myLexemeIndex]) || + myComments.contains(nextTokenType))) { item.myLexemeIndex++; } } else if (item instanceof DoneMarker || item instanceof ErrorItem) { int prevProductionLexIndex = fProduction.get(i - 1).myLexemeIndex; IElementType prevTokenType; - - while (item.myLexemeIndex > prevProductionLexIndex && item.myLexemeIndex - 1 < myLexemeCount && - ( myWhitespaces.contains(prevTokenType = myLexTypes[item.myLexemeIndex - 1]) || - myComments.contains(prevTokenType) - ) - ) { + while (item.myLexemeIndex > prevProductionLexIndex && + item.myLexemeIndex - 1 < myLexemeCount && + (myWhitespaces.contains(prevTokenType = myLexTypes[item.myLexemeIndex - 1]) || + myComments.contains(prevTokenType))) { item.myLexemeIndex--; } } diff --git a/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java b/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java index 111228759fff..0a4b82ab3e45 100644 --- a/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java +++ b/platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java @@ -33,8 +33,13 @@ public class LightPsiBuilderTest { private static final IElementType ROOT = new IElementType("ROOT", Language.ANY); private static final IElementType LETTER = new IElementType("LETTER", Language.ANY); private static final IElementType DIGIT = new IElementType("DIGIT", Language.ANY); + private static final IElementType WHITESPACE = new IElementType("WHITESPACE", Language.ANY); private static final IElementType OTHER = new IElementType("OTHER", Language.ANY); private static final IElementType COLLAPSED = new IElementType("COLLAPSED", Language.ANY); + private static final IElementType LEFT_BOUND = new IElementType("LEFT_BOUND", Language.ANY) { + public boolean isLeftBound() { return true; } + }; + private static final TokenSet WHITESPACE_SET = TokenSet.create(WHITESPACE); @Test public void testPlain() { @@ -154,7 +159,8 @@ public class LightPsiBuilderTest { builder.mark(); first.done(LETTER); } - }); + }, + "Another not done marker added after this one. Must be done before this."); } @Test @@ -168,7 +174,8 @@ public class LightPsiBuilderTest { second.precede(); first.doneBefore(LETTER, second); } - }); + }, + "Another not done marker added after this one. Must be done before this."); } @Test @@ -181,7 +188,95 @@ public class LightPsiBuilderTest { final PsiBuilder.Marker second = builder.mark(); second.doneBefore(LETTER, first); } - }); + }, + "'Before' marker precedes this one."); + } + + @Test + public void testWhitespaceTrimming() throws Exception { + doTest(" a b ", + new Parser() { + public void parse(PsiBuilder builder) { + PsiBuilder.Marker marker = builder.mark(); + builder.advanceLexer(); + marker.done(OTHER); + marker = builder.mark(); + builder.advanceLexer(); + marker.done(OTHER); + builder.advanceLexer(); + } + }, + "Element(ROOT)\n" + + " PsiWhiteSpace(' ')\n" + + " Element(OTHER)\n" + + " PsiElement(LETTER)('a')\n" + + " PsiWhiteSpace(' ')\n" + + " Element(OTHER)\n" + + " PsiElement(LETTER)('b')\n" + + " PsiWhiteSpace(' ')\n"); + } + + @Test + public void testWhitespaceBalancingByErrors() throws Exception { + doTest("a b c", + new Parser() { + public void parse(PsiBuilder builder) { + PsiBuilder.Marker marker = builder.mark(); + builder.advanceLexer(); + builder.error("error 1"); + marker.done(OTHER); + marker = builder.mark(); + builder.advanceLexer(); + builder.mark().error("error 2"); + marker.done(OTHER); + marker = builder.mark(); + builder.advanceLexer(); + marker.error("error 3"); + } + }, + "Element(ROOT)\n" + + " Element(OTHER)\n" + + " PsiElement(LETTER)('a')\n" + + " PsiErrorElement:error 1\n" + + " \n" + + " PsiWhiteSpace(' ')\n" + + " Element(OTHER)\n" + + " PsiElement(LETTER)('b')\n" + + " PsiErrorElement:error 2\n" + + " \n" + + " PsiWhiteSpace(' ')\n" + + " PsiErrorElement:error 3\n" + + " PsiElement(LETTER)('c')\n"); + } + + @Test + public void testWhitespaceBalancingByEmptyComposites() throws Exception { + doTest("a b c", + new Parser() { + public void parse(PsiBuilder builder) { + PsiBuilder.Marker marker = builder.mark(); + builder.advanceLexer(); + builder.mark().done(OTHER); + marker.done(OTHER); + marker = builder.mark(); + builder.advanceLexer(); + builder.mark().done(LEFT_BOUND); + marker.done(OTHER); + builder.advanceLexer(); + } + }, + "Element(ROOT)\n" + + " Element(OTHER)\n" + + " PsiElement(LETTER)('a')\n" + + " PsiWhiteSpace(' ')\n" + + " Element(OTHER)\n" + + " \n" + + " Element(OTHER)\n" + + " PsiElement(LETTER)('b')\n" + + " Element(LEFT_BOUND)\n" + + " \n" + + " PsiWhiteSpace(' ')\n" + + " PsiElement(LETTER)('c')\n"); } private interface Parser { @@ -189,15 +284,15 @@ public class LightPsiBuilderTest { } private static void doTest(final String text, final Parser parser, final String expected) { - final PsiBuilder builder = new PsiBuilderImpl(new MyTestLexer(), TokenSet.EMPTY, TokenSet.EMPTY, text); + final PsiBuilder builder = new PsiBuilderImpl(new MyTestLexer(), WHITESPACE_SET, TokenSet.EMPTY, text); final PsiBuilder.Marker rootMarker = builder.mark(); parser.parse(builder); rootMarker.done(ROOT); final ASTNode root = builder.getTreeBuilt(); - assertEquals(expected, DebugUtil.nodeTreeToString(root, true)); + assertEquals(expected, DebugUtil.nodeTreeToString(root, false)); } - private static void doFailTest(final String text, final Parser parser) { + private static void doFailTest(final String text, final Parser parser, final String expected) { final PrintStream std = System.err; //noinspection IOResourceOpenedButNotSafelyClosed System.setErr(new PrintStream(new NullStream())); @@ -209,8 +304,7 @@ public class LightPsiBuilderTest { fail("should fail"); } catch (AssertionError e) { - //System.out.println("caught: " + e); - if ("should fail".equals(e.getMessage())) throw e; + assertEquals(expected, e.getMessage()); } } finally { @@ -234,18 +328,11 @@ public class LightPsiBuilderTest { } public IElementType getTokenType() { - if (myIndex >= myBufferEnd) { - return null; - } - else if (Character.isDigit(myBuffer.charAt(myIndex))) { - return DIGIT; - } - else if (Character.isLetter(myBuffer.charAt(myIndex))) { - return LETTER; - } - else { - return OTHER; - } + if (myIndex >= myBufferEnd) return null; + else if (Character.isLetter(myBuffer.charAt(myIndex))) return LETTER; + else if (Character.isDigit(myBuffer.charAt(myIndex))) return DIGIT; + else if (Character.isWhitespace(myBuffer.charAt(myIndex))) return WHITESPACE; + else return OTHER; } public int getTokenStart() { diff --git a/platform/platform-api/src/com/intellij/psi/tree/IElementType.java b/platform/platform-api/src/com/intellij/psi/tree/IElementType.java index eac5aee9b638..3815860c1331 100644 --- a/platform/platform-api/src/com/intellij/psi/tree/IElementType.java +++ b/platform/platform-api/src/com/intellij/psi/tree/IElementType.java @@ -32,7 +32,6 @@ import java.util.List; * @see com.intellij.lexer.Lexer#getTokenType() * @see com.intellij.lang.ASTNode#getElementType() */ - public class IElementType { private static final Logger LOG = Logger.getInstance("#com.intellij.psi.tree.IElementType"); @@ -46,9 +45,8 @@ public class IElementType { /** * Default enumeration predicate which matches all token types. * - * @see #enumerate(com.intellij.psi.tree.IElementType.Predicate) + * @see #enumerate(Predicate) */ - public static final Predicate TRUE = new Predicate() { public boolean matches(IElementType type) { return true; @@ -89,7 +87,6 @@ public class IElementType { * @param debugName the name of the element type, used for debugging purposes. * @param language the language with which the element type is associated. */ - public IElementType(@NotNull @NonNls String debugName, @Nullable Language language) { this(debugName, language, true); } @@ -98,6 +95,7 @@ public class IElementType { myDebugName = debugName; myLanguage = language == null ? Language.ANY : language; if (register) { + //noinspection AssignmentToStaticFieldFromInstanceMethod myIndex = ourCounter++; LOG.assertTrue(ourCounter < MAX_INDEXED_TYPES, "Too many element types registered. Out of (short) range."); synchronized (ourRegistry) { @@ -114,7 +112,6 @@ public class IElementType { * * @return the associated language. */ - @NotNull public Language getLanguage() { return myLanguage; @@ -126,7 +123,6 @@ public class IElementType { * * @return the element type index. */ - public final short getIndex() { return myIndex; } @@ -138,10 +134,9 @@ public class IElementType { /** * Returns the element type registered at the specified index. * - * @param idx the indx for which the element type should be returned. + * @param idx the index for which the element type should be returned. * @return the element type at the specified index. */ - public static IElementType find(short idx) { synchronized (ourRegistry) { if (idx == 0) return ourRegistry.get(0); // We've changed FIRST_TOKEN_INDEX from 0 to 1. This is just for old plugins to avoid crashes. @@ -152,10 +147,34 @@ public class IElementType { /** * Predicate for matching element types. * - * @see IElementType#enumerate(com.intellij.psi.tree.IElementType.Predicate) + * @see IElementType#enumerate(Predicate) */ - public interface Predicate { boolean matches(IElementType type); } + + /** + * Controls whitespace balancing behavior of PsiBuilder. + *

By default, empty composite elements (containing no children) are bounded to the right neighbour, forming following tree: + * 

+   *  [previous_element]
+   *  [whitespace]
+   *  [empty_element]
+   *    <empty>
+   *  [next_element]
+   *
+ *

Left-bound elements are bounded to the left neighbour instead: + * 

+   *  [previous_element]
+   *  [empty_element]
+   *    <empty>
+   *  [whitespace]
+   *  [next_element]
+   *
+ *

See {@linkplain com.intellij.lang.impl.PsiBuilderImpl#prepareLightTree()} for details. + * @return true if empty elements of this type should be bound to the left. + */ + public boolean isLeftBound() { + return false; + } } \ No newline at end of file