diff --git a/platform/core-api/src/com/intellij/psi/tree/IElementType.java b/platform/core-api/src/com/intellij/psi/tree/IElementType.java index 1726c27832ed..0219e5805713 100644 --- a/platform/core-api/src/com/intellij/psi/tree/IElementType.java +++ b/platform/core-api/src/com/intellij/psi/tree/IElementType.java @@ -144,6 +144,7 @@ public class IElementType { public static IElementType find(short idx) { synchronized (ourRegistry) { if (idx == 0) return ourRegistry.get(0); // We've changed FIRST_TOKEN_INDEX from 0 to 1. This is just for old plugins to avoid crashes. + if (idx >= ourRegistry.size() + FIRST_TOKEN_INDEX) return null; return ourRegistry.get(idx - FIRST_TOKEN_INDEX); } } diff --git a/platform/core-api/src/com/intellij/psi/tree/TokenSet.java b/platform/core-api/src/com/intellij/psi/tree/TokenSet.java index 700bad28faa4..30ce34b211c5 100644 --- a/platform/core-api/src/com/intellij/psi/tree/TokenSet.java +++ b/platform/core-api/src/com/intellij/psi/tree/TokenSet.java @@ -20,7 +20,9 @@ import com.intellij.util.ArrayUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** * A set of element types. @@ -30,7 +32,9 @@ public class TokenSet { @Override public boolean contains(IElementType t) { return false; } }; - private final short myShift, myMax, myTop; + private final short myShift; + private final short myMax; + private final short myTop; private final long[] myWords; private volatile IElementType[] myTypes; @@ -44,7 +48,7 @@ public class TokenSet { private boolean get(int index) { final int wordIndex = (index >> 6) - myShift; - return wordIndex >= 0 && wordIndex < myWords.length && (myWords[wordIndex] & (1l << index)) != 0; + return wordIndex >= 0 && wordIndex < myWords.length && (myWords[wordIndex] & (1L << index)) != 0; } /** @@ -69,24 +73,20 @@ public class TokenSet { IElementType[] types = myTypes; if (types == null) { - if (myWords.length > 0) { - int elementCount = 0; - for (long word : myWords) { - elementCount += Long.bitCount(word); - } - - types = new IElementType[elementCount]; - int count = 0; - for (short i = (short)(myShift << 6); i <= myMax; i++) { - if (get(i)) { - types[count++] = IElementType.find(i); - } - } - } - else { + if (myWords.length == 0) { types = IElementType.EMPTY_ARRAY; } - + else { + List list = new ArrayList(); + for (short i = (short)Math.max(1, myShift << 6); i <= myMax; i++) { + if (!get(i)) continue; + IElementType type = IElementType.find(i); + if (type != null) { + list.add(type); + } + } + types = list.toArray(new IElementType[list.size()]); + } myTypes = types; } @@ -105,10 +105,11 @@ public class TokenSet { * @return the new token set. */ @NotNull - public static TokenSet create(IElementType... types) { + public static TokenSet create(@NotNull IElementType... types) { if (types.length == 0) return EMPTY; - short min = Short.MAX_VALUE, max = 0; + short min = Short.MAX_VALUE; + short max = 0; for (IElementType type : types) { if (type != null) { final short index = type.getIndex(); @@ -124,7 +125,7 @@ public class TokenSet { if (type != null) { final short index = type.getIndex(); final int wordIndex = (index >> 6) - shift; - set.myWords[wordIndex] |= (1l << index); + set.myWords[wordIndex] |= 1L << index; } } return set; @@ -140,7 +141,8 @@ public class TokenSet { public static TokenSet orSet(@NotNull TokenSet... sets) { if (sets.length == 0) return EMPTY; - short shift = sets[0].myShift, max = sets[0].myMax; + short shift = sets[0].myShift; + short max = sets[0].myMax; for (int i = 1; i < sets.length; i++) { if (shift > sets[i].myShift) shift = sets[i].myShift; if (max < sets[i].myMax) max = sets[i].myMax; @@ -167,7 +169,8 @@ public class TokenSet { public static TokenSet andSet(@NotNull TokenSet a, @NotNull TokenSet b) { final TokenSet newSet = new TokenSet((short)Math.min(a.myShift, b.myShift), (short)Math.max(a.myMax, b.myMax)); for (int i = 0; i < newSet.myWords.length; i++) { - final int ai = newSet.myShift - a.myShift + i, bi = newSet.myShift - b.myShift + i; + final int ai = newSet.myShift - a.myShift + i; + final int bi = newSet.myShift - b.myShift + i; newSet.myWords[i] = (0 <= ai && ai < a.myWords.length ? a.myWords[ai] : 0l) & (0 <= bi && bi < b.myWords.length ? b.myWords[bi] : 0l); } return newSet; @@ -184,7 +187,8 @@ public class TokenSet { public static TokenSet andNot(@NotNull TokenSet a, @NotNull TokenSet b) { final TokenSet newSet = new TokenSet((short)Math.min(a.myShift, b.myShift), (short)Math.max(a.myMax, b.myMax)); for (int i = 0; i < newSet.myWords.length; i++) { - final int ai = newSet.myShift - a.myShift + i, bi = newSet.myShift - b.myShift + i; + final int ai = newSet.myShift - a.myShift + i; + final int bi = newSet.myShift - b.myShift + i; newSet.myWords[i] = (0 <= ai && ai < a.myWords.length ? a.myWords[ai] : 0l) & ~(0 <= bi && bi < b.myWords.length ? b.myWords[bi] : 0l); } return newSet; @@ -192,16 +196,19 @@ public class TokenSet { /** @deprecated please use {@linkplain #andNot(TokenSet, TokenSet)} (to remove in IDEA 13) */ @SuppressWarnings("UnusedDeclaration") + @NotNull public TokenSet minus(@NotNull TokenSet t) { return andNot(this, t); } /** @deprecated please use {@linkplain IElementType#enumerate(com.intellij.psi.tree.IElementType.Predicate)} (to remove in IDEA 13) */ @SuppressWarnings("UnusedDeclaration") + @NotNull public static TokenSet not(@NotNull TokenSet set) { final TokenSet newSet = new TokenSet((short)0, set.myTop); for (int i = 0; i < newSet.myWords.length; i++) { - final long word = i >= set.myShift ? set.myWords[(i - set.myShift)] : 0l; + int wordIndex = i - set.myShift; + long word = wordIndex >= 0 && wordIndex < set.myWords.length ? set.myWords[wordIndex] : 0; newSet.myWords[i] = ~word; } return newSet; diff --git a/platform/platform-tests/testSrc/com/intellij/psi/tree/TokenSetTest.java b/platform/platform-tests/testSrc/com/intellij/psi/tree/TokenSetTest.java index 3dd6fcfaa0ff..9dfc0c34eba4 100644 --- a/platform/platform-tests/testSrc/com/intellij/psi/tree/TokenSetTest.java +++ b/platform/platform-tests/testSrc/com/intellij/psi/tree/TokenSetTest.java @@ -17,36 +17,50 @@ package com.intellij.psi.tree; import com.intellij.lang.Language; import com.intellij.testFramework.PlatformTestUtil; +import com.intellij.util.ArrayUtil; import com.intellij.util.ThrowableRunnable; import com.intellij.util.containers.ContainerUtil; +import org.jetbrains.annotations.NotNull; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; -import java.util.Arrays; -import java.util.Random; -import java.util.Set; +import java.util.*; import static org.junit.Assert.*; +@RunWith(Parameterized.class) public class TokenSetTest { - private IElementType T1, T2, T3, T4, T5, T6; + @Parameterized.Parameters + public static List data() { + return Collections.nCopies(10, ArrayUtil.EMPTY_OBJECT_ARRAY); + } + + private static IElementType T1, T2, T3, T4, T5, T6; private TokenSet S1, S12, S3, S34, S5; - @Before - public void setUp() { + @BeforeClass + public static void setUp() { T1 = new IElementType("T1", Language.ANY); T2 = new IElementType("T2", Language.ANY); - S1 = TokenSet.create(T1); - S12 = TokenSet.create(T1, T2); fakeElements(1, 128); T3 = new IElementType("T3", Language.ANY); T4 = new IElementType("T4", Language.ANY); - S3 = TokenSet.create(T3); - S34 = TokenSet.create(T3, T4); fakeElements(201, 204); T5 = new IElementType("T5", Language.ANY); T6 = new IElementType("T6", Language.ANY); + } + + @Before + public void isetup() { + S1 = TokenSet.create(T1); + S12 = TokenSet.create(T1, T2); + S3 = TokenSet.create(T3); + S34 = TokenSet.create(T3, T4); S5 = TokenSet.create(T5); + } @Test @@ -92,9 +106,9 @@ public class TokenSetTest { @SuppressWarnings("deprecation") @Test public void not() throws Exception { - check(TokenSet.not(S12)); - check(TokenSet.not(S34), T1, T2); - check(TokenSet.not(S5), T1, T2, T3, T4, T6); + checkNot(TokenSet.not(S12), T1, T2); + checkNot(TokenSet.not(S34), T3, T4); + checkNot(TokenSet.not(S5), T5); } private static void fakeElements(int from, int to) { @@ -103,7 +117,7 @@ public class TokenSetTest { } } - private void check(TokenSet set, IElementType... elements) { + private static void check(@NotNull TokenSet set, @NotNull IElementType... elements) { final Set expected = ContainerUtil.newHashSet(elements); for (IElementType t : Arrays.asList(T1, T2, T3, T4, T5, T6)) { if (expected.contains(t)) { @@ -115,6 +129,19 @@ public class TokenSetTest { } } + private static void checkNot(@NotNull TokenSet set, @NotNull IElementType... elements) { + final Set expected = ContainerUtil.newHashSet(elements); + for (IElementType t : Arrays.asList(T1, T2, T3, T4, T5, T6)) { + if (!expected.contains(t)) { + assertTrue("missed: " + t, set.contains(t)); + } + else { + assertFalse("unexpected: " + t, set.contains(t)); + } + } + } + + @Test public void performance() throws Exception { final IElementType[] elementTypes = IElementType.enumerate(IElementType.TRUE);