MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

PSI builder: whitespaces balancing

Browse Source 
1. empty error elements (builder.mark().error()) are bounded to the left - like builder.error() ones
2. empty composites (builder.mark().done(type)) can be bounded to the left - if type.isLeftBound() is true
This commit is contained in:
Roman Shevchenko
2010-07-22 13:43:51 +04:00
parent deeb9d932f
commit a1bc7207e2
3 changed files with 190 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/lang-impl/src/com/intellij/lang/impl/PsiBuilderImpl.java
+54 -27
View File
@@ -384,6 +384,7 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
private static class DoneMarker extends ProductionMarker {
public StartMarker myStart;
public boolean myCollapse = false;
public boolean myTieToTheLeft = false;
public DoneMarker() {}
@@ -567,20 +568,6 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
START_MARKERS.recycle((StartMarker)marker);
}
@SuppressWarnings({"SuspiciousMethodCalls"})
public void doneBefore(Marker marker, Marker before) {
doValidityChecks(marker, before);
int beforeIndex = myProduction.lastIndexOf(before);
DoneMarker doneMarker = DONE_MARKERS.alloc();
doneMarker.myLexemeIndex = ((StartMarker)before).myLexemeIndex;
doneMarker.myStart = (StartMarker)marker;
((StartMarker)marker).myDoneMarker = doneMarker;
myProduction.add(beforeIndex, doneMarker);
}
@SuppressWarnings({"SuspiciousMethodCalls"})
public void drop(Marker marker) {
final boolean removed = myProduction.remove(myProduction.lastIndexOf(marker)) == marker;
@@ -594,6 +581,8 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
doValidityChecks(marker, null);
DoneWithErrorMarker doneMarker = new DoneWithErrorMarker((StartMarker)marker, myCurrentLexeme, message);
doneMarker.myTieToTheLeft = isEmpty(((StartMarker)marker).myLexemeIndex, myCurrentLexeme);
((StartMarker)marker).myDoneMarker = doneMarker;
myProduction.add(doneMarker);
}
@@ -604,8 +593,9 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
int beforeIndex = myProduction.lastIndexOf(before);
DoneWithErrorMarker doneMarker = new DoneWithErrorMarker((StartMarker)marker, myCurrentLexeme, message);
doneMarker.myLexemeIndex = ((StartMarker)before).myLexemeIndex;
DoneWithErrorMarker doneMarker = new DoneWithErrorMarker((StartMarker)marker, ((StartMarker)before).myLexemeIndex, message);
doneMarker.myTieToTheLeft = isEmpty(((StartMarker)marker).myLexemeIndex, ((StartMarker)before).myLexemeIndex);
((StartMarker)marker).myDoneMarker = doneMarker;
myProduction.add(beforeIndex, doneMarker);
}
@@ -616,11 +606,37 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
DoneMarker doneMarker = DONE_MARKERS.alloc();
doneMarker.myStart = (StartMarker)marker;
doneMarker.myLexemeIndex = myCurrentLexeme;
doneMarker.myTieToTheLeft = doneMarker.myStart.myType.isLeftBound() &&
isEmpty(((StartMarker)marker).myLexemeIndex, myCurrentLexeme);
((StartMarker)marker).myDoneMarker = doneMarker;
myProduction.add(doneMarker);
}
@SuppressWarnings({"SuspiciousMethodCalls"})
public void doneBefore(Marker marker, Marker before) {
doValidityChecks(marker, before);
int beforeIndex = myProduction.lastIndexOf(before);
DoneMarker doneMarker = DONE_MARKERS.alloc();
doneMarker.myLexemeIndex = ((StartMarker)before).myLexemeIndex;
doneMarker.myStart = (StartMarker)marker;
doneMarker.myTieToTheLeft = doneMarker.myStart.myType.isLeftBound() &&
isEmpty(((StartMarker)marker).myLexemeIndex, ((StartMarker)before).myLexemeIndex);
((StartMarker)marker).myDoneMarker = doneMarker;
myProduction.add(beforeIndex, doneMarker);
}
private boolean isEmpty(final int startIdx, final int endIdx) {
for (int i = startIdx; i < endIdx; i++) {
final IElementType token = myLexTypes[i];
if (!myWhitespaces.contains(token) && !myComments.contains(token)) return false;
}
return true;
}
public void collapse(final Marker marker) {
done(marker);
((StartMarker)marker).myDoneMarker.myCollapse = true;
@@ -779,25 +795,36 @@ public class PsiBuilderImpl extends UserDataHolderBase implements PsiBuilder {
for (int i = 1; i < fProduction.size() - 1; i++) {
ProductionMarker item = fProduction.get(i);
if (item instanceof StartMarker) {
if (item instanceof StartMarker && ((StartMarker)item).myDoneMarker == null) {
LOG.error(UNBALANCED_MESSAGE);
}
if (item instanceof StartMarker && ((StartMarker)item).myDoneMarker.myTieToTheLeft) {
int prevProductionLexIndex = fProduction.get(i - 1).myLexemeIndex;
IElementType prevTokenType;
while (item.myLexemeIndex > prevProductionLexIndex &&
item.myLexemeIndex - 1 < myLexemeCount &&
(myWhitespaces.contains(prevTokenType = myLexTypes[item.myLexemeIndex - 1]) ||
myComments.contains(prevTokenType))) {
item.myLexemeIndex--;
}
((StartMarker)item).myDoneMarker.myLexemeIndex = item.myLexemeIndex;
}
else if (item instanceof StartMarker) {
IElementType nextTokenType;
while (item.myLexemeIndex < myLexemeCount &&
( myWhitespaces.contains(nextTokenType = myLexTypes[item.myLexemeIndex]) ||
myComments.contains(nextTokenType)
)
) {
(myWhitespaces.contains(nextTokenType = myLexTypes[item.myLexemeIndex]) ||
myComments.contains(nextTokenType))) {
item.myLexemeIndex++;
}
}
else if (item instanceof DoneMarker || item instanceof ErrorItem) {
int prevProductionLexIndex = fProduction.get(i - 1).myLexemeIndex;
IElementType prevTokenType;
while (item.myLexemeIndex > prevProductionLexIndex && item.myLexemeIndex - 1 < myLexemeCount &&
( myWhitespaces.contains(prevTokenType = myLexTypes[item.myLexemeIndex - 1]) ||
myComments.contains(prevTokenType)
)
) {
while (item.myLexemeIndex > prevProductionLexIndex &&
item.myLexemeIndex - 1 < myLexemeCount &&
(myWhitespaces.contains(prevTokenType = myLexTypes[item.myLexemeIndex - 1]) ||
myComments.contains(prevTokenType))) {
item.myLexemeIndex--;
}
}
platform/lang-impl/testSrc/com/intellij/lang/LightPsiBuilderTest.java
+107 -20
View File
@@ -33,8 +33,13 @@ public class LightPsiBuilderTest {
private static final IElementType ROOT = new IElementType("ROOT", Language.ANY);
private static final IElementType LETTER = new IElementType("LETTER", Language.ANY);
private static final IElementType DIGIT = new IElementType("DIGIT", Language.ANY);
private static final IElementType WHITESPACE = new IElementType("WHITESPACE", Language.ANY);
private static final IElementType OTHER = new IElementType("OTHER", Language.ANY);
private static final IElementType COLLAPSED = new IElementType("COLLAPSED", Language.ANY);
private static final IElementType LEFT_BOUND = new IElementType("LEFT_BOUND", Language.ANY) {
public boolean isLeftBound() { return true; }
};
private static final TokenSet WHITESPACE_SET = TokenSet.create(WHITESPACE);
@Test
public void testPlain() {
@@ -154,7 +159,8 @@ public class LightPsiBuilderTest {
builder.mark();
first.done(LETTER);
}
});
},
"Another not done marker added after this one. Must be done before this.");
}
@Test
@@ -168,7 +174,8 @@ public class LightPsiBuilderTest {
second.precede();
first.doneBefore(LETTER, second);
}
});
},
"Another not done marker added after this one. Must be done before this.");
}
@Test
@@ -181,7 +188,95 @@ public class LightPsiBuilderTest {
final PsiBuilder.Marker second = builder.mark();
second.doneBefore(LETTER, first);
}
});
},
"'Before' marker precedes this one.");
}
@Test
public void testWhitespaceTrimming() throws Exception {
doTest(" a b ",
new Parser() {
public void parse(PsiBuilder builder) {
PsiBuilder.Marker marker = builder.mark();
builder.advanceLexer();
marker.done(OTHER);
marker = builder.mark();
builder.advanceLexer();
marker.done(OTHER);
builder.advanceLexer();
}
},
"Element(ROOT)\n" +
" PsiWhiteSpace(' ')\n" +
" Element(OTHER)\n" +
" PsiElement(LETTER)('a')\n" +
" PsiWhiteSpace(' ')\n" +
" Element(OTHER)\n" +
" PsiElement(LETTER)('b')\n" +
" PsiWhiteSpace(' ')\n");
}
@Test
public void testWhitespaceBalancingByErrors() throws Exception {
doTest("a b c",
new Parser() {
public void parse(PsiBuilder builder) {
PsiBuilder.Marker marker = builder.mark();
builder.advanceLexer();
builder.error("error 1");
marker.done(OTHER);
marker = builder.mark();
builder.advanceLexer();
builder.mark().error("error 2");
marker.done(OTHER);
marker = builder.mark();
builder.advanceLexer();
marker.error("error 3");
}
},
"Element(ROOT)\n" +
" Element(OTHER)\n" +
" PsiElement(LETTER)('a')\n" +
" PsiErrorElement:error 1\n" +
" <empty list>\n" +
" PsiWhiteSpace(' ')\n" +
" Element(OTHER)\n" +
" PsiElement(LETTER)('b')\n" +
" PsiErrorElement:error 2\n" +
" <empty list>\n" +
" PsiWhiteSpace(' ')\n" +
" PsiErrorElement:error 3\n" +
" PsiElement(LETTER)('c')\n");
}
@Test
public void testWhitespaceBalancingByEmptyComposites() throws Exception {
doTest("a b c",
new Parser() {
public void parse(PsiBuilder builder) {
PsiBuilder.Marker marker = builder.mark();
builder.advanceLexer();
builder.mark().done(OTHER);
marker.done(OTHER);
marker = builder.mark();
builder.advanceLexer();
builder.mark().done(LEFT_BOUND);
marker.done(OTHER);
builder.advanceLexer();
}
},
"Element(ROOT)\n" +
" Element(OTHER)\n" +
" PsiElement(LETTER)('a')\n" +
" PsiWhiteSpace(' ')\n" +
" Element(OTHER)\n" +
" <empty list>\n" +
" Element(OTHER)\n" +
" PsiElement(LETTER)('b')\n" +
" Element(LEFT_BOUND)\n" +
" <empty list>\n" +
" PsiWhiteSpace(' ')\n" +
" PsiElement(LETTER)('c')\n");
}
private interface Parser {
@@ -189,15 +284,15 @@ public class LightPsiBuilderTest {
}
private static void doTest(final String text, final Parser parser, final String expected) {
final PsiBuilder builder = new PsiBuilderImpl(new MyTestLexer(), TokenSet.EMPTY, TokenSet.EMPTY, text);
final PsiBuilder builder = new PsiBuilderImpl(new MyTestLexer(), WHITESPACE_SET, TokenSet.EMPTY, text);
final PsiBuilder.Marker rootMarker = builder.mark();
parser.parse(builder);
rootMarker.done(ROOT);
final ASTNode root = builder.getTreeBuilt();
assertEquals(expected, DebugUtil.nodeTreeToString(root, true));
assertEquals(expected, DebugUtil.nodeTreeToString(root, false));
}
private static void doFailTest(final String text, final Parser parser) {
private static void doFailTest(final String text, final Parser parser, final String expected) {
final PrintStream std = System.err;
//noinspection IOResourceOpenedButNotSafelyClosed
System.setErr(new PrintStream(new NullStream()));
@@ -209,8 +304,7 @@ public class LightPsiBuilderTest {
fail("should fail");
}
catch (AssertionError e) {
//System.out.println("caught: " + e);
if ("should fail".equals(e.getMessage())) throw e;
assertEquals(expected, e.getMessage());
}
}
finally {
@@ -234,18 +328,11 @@ public class LightPsiBuilderTest {
}
public IElementType getTokenType() {
if (myIndex >= myBufferEnd) {
return null;
}
else if (Character.isDigit(myBuffer.charAt(myIndex))) {
return DIGIT;
}
else if (Character.isLetter(myBuffer.charAt(myIndex))) {
return LETTER;
}
else {
return OTHER;
}
if (myIndex >= myBufferEnd) return null;
else if (Character.isLetter(myBuffer.charAt(myIndex))) return LETTER;
else if (Character.isDigit(myBuffer.charAt(myIndex))) return DIGIT;
else if (Character.isWhitespace(myBuffer.charAt(myIndex))) return WHITESPACE;
else return OTHER;
}
public int getTokenStart() {
platform/platform-api/src/com/intellij/psi/tree/IElementType.java
+29 -10
View File
@@ -32,7 +32,6 @@ import java.util.List;
* @see com.intellij.lexer.Lexer#getTokenType()
* @see com.intellij.lang.ASTNode#getElementType()
*/
public class IElementType {
private static final Logger LOG = Logger.getInstance("#com.intellij.psi.tree.IElementType");
@@ -46,9 +45,8 @@ public class IElementType {
/**
* Default enumeration predicate which matches all token types.
*
* @see #enumerate(com.intellij.psi.tree.IElementType.Predicate)
* @see #enumerate(Predicate)
*/
public static final Predicate TRUE = new Predicate() {
public boolean matches(IElementType type) {
return true;
@@ -89,7 +87,6 @@ public class IElementType {
* @param debugName the name of the element type, used for debugging purposes.
* @param language the language with which the element type is associated.
*/
public IElementType(@NotNull @NonNls String debugName, @Nullable Language language) {
this(debugName, language, true);
}
@@ -98,6 +95,7 @@ public class IElementType {
myDebugName = debugName;
myLanguage = language == null ? Language.ANY : language;
if (register) {
//noinspection AssignmentToStaticFieldFromInstanceMethod
myIndex = ourCounter++;
LOG.assertTrue(ourCounter < MAX_INDEXED_TYPES, "Too many element types registered. Out of (short) range.");
synchronized (ourRegistry) {
@@ -114,7 +112,6 @@ public class IElementType {
*
* @return the associated language.
*/
@NotNull
public Language getLanguage() {
return myLanguage;
@@ -126,7 +123,6 @@ public class IElementType {
*
* @return the element type index.
*/
public final short getIndex() {
return myIndex;
}
@@ -138,10 +134,9 @@ public class IElementType {
/**
* Returns the element type registered at the specified index.
*
* @param idx the indx for which the element type should be returned.
* @param idx the index for which the element type should be returned.
* @return the element type at the specified index.
*/
public static IElementType find(short idx) {
synchronized (ourRegistry) {
if (idx == 0) return ourRegistry.get(0); // We've changed FIRST_TOKEN_INDEX from 0 to 1. This is just for old plugins to avoid crashes.
@@ -152,10 +147,34 @@ public class IElementType {
/**
* Predicate for matching element types.
*
* @see IElementType#enumerate(com.intellij.psi.tree.IElementType.Predicate)
* @see IElementType#enumerate(Predicate)
*/
public interface Predicate {
boolean matches(IElementType type);
}
/**
* Controls whitespace balancing behavior of PsiBuilder.
* <p>By default, empty composite elements (containing no children) are bounded to the right neighbour, forming following tree:
* <pre>
* [previous_element]
* [whitespace]
* [empty_element]
* &lt;empty&gt;
* [next_element]
* </pre>
* <p>Left-bound elements are bounded to the left neighbour instead:
* <pre>
* [previous_element]
* [empty_element]
* &lt;empty&gt;
* [whitespace]
* [next_element]
* </pre>
* <p>See {@linkplain com.intellij.lang.impl.PsiBuilderImpl#prepareLightTree()} for details.
* @return true if empty elements of this type should be bound to the left.
*/
public boolean isLeftBound() {
return false;
}
}