MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Bitmap-based token set

Browse Source
This commit is contained in:
Roman Shevchenko
2012-09-21 15:11:54 +02:00
parent 7b0a1d6257
commit 4c6d9bde01
2 changed files with 241 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/core-api/src/com/intellij/psi/tree/TokenSet.java
+109 -56
View File
@@ -16,6 +16,9 @@
package com.intellij.psi.tree;
import com.intellij.openapi.diagnostic.LogUtil;
import com.intellij.util.ArrayUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Arrays;
@@ -23,30 +26,65 @@ import java.util.Arrays;
* A set of element types.
*/
public class TokenSet {
public static final TokenSet EMPTY = new TokenSet();
public static final TokenSet EMPTY = new TokenSet(Short.MAX_VALUE, (short)0) {
@Override public boolean contains(IElementType t) { return false; }
};
private final boolean[] mySet = new boolean[IElementType.getAllocatedTypesCount()];
private final short myShift, myMax, myTop;
private final long[] myWords;
private volatile IElementType[] myTypes;
private TokenSet(short shift, short max) {
myShift = shift;
myMax = max;
myTop = IElementType.getAllocatedTypesCount();
final int size = (max >> 6) + 1 - shift;
myWords = size > 0 ? new long[size] : ArrayUtil.EMPTY_LONG_ARRAY;
}
private boolean get(int index) {
final int wordIndex = (index >> 6) - myShift;
return wordIndex >= 0 && wordIndex < myWords.length && (myWords[wordIndex] & (1l << index)) != 0;
}
/**
* Checks if the specified element type is contained in the set.
*
* @param t the element type to search for.
* @return true if the element type is found in the set, false otherwise.
*/
public boolean contains(@Nullable IElementType t) {
if (t == null) return false;
final short i = t.getIndex();
return 0 <= i && i <= myMax && get(i);
}
/**
* Returns the array of element types contained in the set.
*
* @return the contents of the set.
*/
@NotNull
public IElementType[] getTypes() {
IElementType[] types = myTypes;
if (types == null) {
int elementCount = 0;
for (boolean bit : mySet) {
if (bit) elementCount++;
}
types = new IElementType[elementCount];
int count = 0;
for (short i = IElementType.FIRST_TOKEN_INDEX; i < mySet.length; i++) {
if (mySet[i]) {
types[count++] = IElementType.find(i);
if (types == null) {
if (myWords.length > 0) {
int elementCount = 0;
for (long word : myWords) {
elementCount += Long.bitCount(word);
}
types = new IElementType[elementCount];
int count = 0;
for (short i = (short)(myShift << 6); i <= myMax; i++) {
if (get(i)) {
types[count++] = IElementType.find(i);
}
}
}
else {
types = IElementType.EMPTY_ARRAY;
}
myTypes = types;
@@ -55,19 +93,38 @@ public class TokenSet {
return types;
}
@Override
public String toString() {
return Arrays.toString(getTypes());
}
/**
* Returns a new token set containing the specified element types.
*
* @param types the element types contained in the set.
* @return the new token set.
*/
@NotNull
public static TokenSet create(IElementType... types) {
TokenSet set = new TokenSet();
if (types.length == 0) return EMPTY;
short min = Short.MAX_VALUE, max = 0;
for (IElementType type : types) {
if (type != null) {
final short index = type.getIndex();
assert index >= 0 : "Unregistered elements are not allowed here: " + LogUtil.objectAndClass(type);
set.mySet[index] = true;
if (min > index) min = index;
if (max < index) max = index;
}
}
final short shift = (short)(min >> 6);
final TokenSet set = new TokenSet(shift, max);
for (IElementType type : types) {
if (type != null) {
final short index = type.getIndex();
final int wordIndex = (index >> 6) - shift;
set.myWords[wordIndex] |= (1l << index);
}
}
return set;
@@ -79,12 +136,21 @@ public class TokenSet {
* @param sets the token sets to unite.
* @return the new token set.
*/
public static TokenSet orSet(TokenSet... sets) {
TokenSet newSet = new TokenSet();
@NotNull
public static TokenSet orSet(@NotNull TokenSet... sets) {
if (sets.length == 0) return EMPTY;
short shift = sets[0].myShift, max = sets[0].myMax;
for (int i = 1; i < sets.length; i++) {
if (shift > sets[i].myShift) shift = sets[i].myShift;
if (max < sets[i].myMax) max = sets[i].myMax;
}
final TokenSet newSet = new TokenSet(shift, max);
for (TokenSet set : sets) {
for (int i = 0; i < newSet.mySet.length; i++) {
if (i >= set.mySet.length) break;
newSet.mySet[i] |= set.mySet[i];
final int shiftDiff = set.myShift - newSet.myShift;
for (int i = 0; i < set.myWords.length; i++) {
newSet.myWords[i + shiftDiff] |= set.myWords[i];
}
}
return newSet;
@@ -97,51 +163,38 @@ public class TokenSet {
* @param b the second token set to intersect.
* @return the new token set.
*/
public static TokenSet andSet(TokenSet a, TokenSet b) {
TokenSet set = new TokenSet();
final boolean[] aSet = a.mySet;
final boolean[] bSet = b.mySet;
final boolean[] newSet = set.mySet;
final int aLen = aSet.length;
final int bLen = bSet.length;
final int andSize = Math.max(newSet.length, Math.max(aLen, bLen));
for (int i = 0; i < andSize; i++) {
newSet[i] = (i < aLen && aSet[i]) && (i < bLen && bSet[i]);
@NotNull
public static TokenSet andSet(@NotNull TokenSet a, @NotNull TokenSet b) {
final TokenSet newSet = new TokenSet((short)Math.min(a.myShift, b.myShift), (short)Math.max(a.myMax, b.myMax));
for (int i = 0; i < newSet.myWords.length; i++) {
final int ai = newSet.myShift - a.myShift + i, bi = newSet.myShift - b.myShift + i;
newSet.myWords[i] = (0 <= ai && ai < a.myWords.length ? a.myWords[ai] : 0l) & (0 <= bi && bi < b.myWords.length ? b.myWords[bi] : 0l);
}
return set;
return newSet;
}
/**
* Checks if the specified element type is contained in the set.
*
* @param t the element type to search for.
* @return true if the element type is found in the set, false otherwise.
*/
public boolean contains(IElementType t) {
if (t == null) return false;
final short i = t.getIndex();
return 0 <= i && i < mySet.length && mySet[i];
@NotNull
public TokenSet minus(@NotNull TokenSet t) {
return andNot(this, t);
}
public TokenSet minus(TokenSet t) {
TokenSet set = new TokenSet();
for (int i = 0; i < mySet.length; i++) {
set.mySet[i] = mySet[i] && (i >= t.mySet.length || !t.mySet[i]);
@NotNull
public static TokenSet andNot(@NotNull TokenSet a, @NotNull TokenSet b) {
final TokenSet newSet = new TokenSet((short)Math.min(a.myShift, b.myShift), (short)Math.max(a.myMax, b.myMax));
for (int i = 0; i < newSet.myWords.length; i++) {
final int ai = newSet.myShift - a.myShift + i, bi = newSet.myShift - b.myShift + i;
newSet.myWords[i] = (0 <= ai && ai < a.myWords.length ? a.myWords[ai] : 0l) & ~(0 <= bi && bi < b.myWords.length ? b.myWords[bi] : 0l);
}
return set;
return newSet;
}
public static TokenSet not(TokenSet t) {
TokenSet set = new TokenSet();
for (int i = 0; i < t.mySet.length; i++) {
set.mySet[i] = !t.mySet[i];
@NotNull
public static TokenSet not(@NotNull TokenSet set) {
final TokenSet newSet = new TokenSet((short)0, set.myTop);
for (int i = 0; i < newSet.myWords.length; i++) {
final long word = i >= set.myShift ? set.myWords[(i - set.myShift)] : 0l;
newSet.myWords[i] = ~word;
}
return set;
}
@Override
public String toString() {
return Arrays.asList(getTypes()).toString();
return newSet;
}
}
platform/platform-tests/testSrc/com/intellij/psi/tree/TokenSetTest.java
+132
View File
@@ -0,0 +1,132 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.psi.tree;
import com.intellij.lang.Language;
import com.intellij.testFramework.PlatformTestUtil;
import com.intellij.util.ThrowableRunnable;
import com.intellij.util.containers.ContainerUtil;
import org.junit.Before;
import org.junit.Test;
import java.util.Arrays;
import java.util.Random;
import java.util.Set;
import static org.junit.Assert.*;
public class TokenSetTest {
private IElementType T1, T2, T3, T4, T5, T6;
private TokenSet S1, S12, S3, S34, S5;
@Before
public void setUp() {
T1 = new IElementType("T1", Language.ANY);
T2 = new IElementType("T2", Language.ANY);
S1 = TokenSet.create(T1);
S12 = TokenSet.create(T1, T2);
fakeElements(1, 128);
T3 = new IElementType("T3", Language.ANY);
T4 = new IElementType("T4", Language.ANY);
S3 = TokenSet.create(T3);
S34 = TokenSet.create(T3, T4);
fakeElements(201, 204);
T5 = new IElementType("T5", Language.ANY);
T6 = new IElementType("T6", Language.ANY);
S5 = TokenSet.create(T5);
}
@Test
public void create() {
check(S1, T1);
check(S12, T1, T2);
check(S3, T3);
check(S34, T3, T4);
}
@Test
public void getTypes() throws Exception {
assertArrayEquals(IElementType.EMPTY_ARRAY, TokenSet.EMPTY.getTypes());
assertArrayEquals(new IElementType[]{T1, T2}, S12.getTypes());
assertArrayEquals(new IElementType[]{T3, T4}, S34.getTypes());
assertEquals("[]", TokenSet.EMPTY.toString());
assertEquals("[T1, T2]", S12.toString());
assertEquals("[T3, T4]", S34.toString());
}
@Test
public void orSet() {
check(TokenSet.orSet(S1, S12, S3), T1, T2, T3);
check(TokenSet.orSet(S1, S3), T1, T3);
}
@Test
public void andSet() {
check(TokenSet.andSet(S1, S12), T1);
check(TokenSet.andSet(S12, S34));
}
@Test
public void andNot() throws Exception {
final TokenSet S123 = TokenSet.orSet(S12, S3);
check(S123.minus(S12), T3);
check(S123.minus(S5), T1, T2, T3);
check(TokenSet.andNot(S123, S12), T3);
check(TokenSet.andNot(S123, S5), T1, T2, T3);
}
@Test
public void not() throws Exception {
check(TokenSet.not(S12));
check(TokenSet.not(S34), T1, T2);
check(TokenSet.not(S5), T1, T2, T3, T4, T6);
}
private static void fakeElements(int from, int to) {
for (int i = from; i <= to; i++) {
new IElementType("Test element #" + i, Language.ANY);
}
}
private void check(TokenSet set, IElementType... elements) {
final Set<IElementType> expected = ContainerUtil.newHashSet(elements);
for (IElementType t : Arrays.asList(T1, T2, T3, T4, T5, T6)) {
if (expected.contains(t)) {
assertTrue("missed: " + t, set.contains(t));
}
else {
assertFalse("unexpected: " + t, set.contains(t));
}
}
}
@Test
public void performance() throws Exception {
final IElementType[] elementTypes = IElementType.enumerate(IElementType.TRUE);
final TokenSet set = TokenSet.create();
final int shift = new Random().nextInt(500000);
PlatformTestUtil.startPerformanceTest("TokenSet.contains() performance", 25, new ThrowableRunnable() {
@Override
public void run() throws Throwable {
for (int i = 0; i < 1000000; i++) {
final IElementType next = elementTypes[((i + shift) % elementTypes.length)];
assertFalse(set.contains(next));
}
}
}).cpuBound().assertTiming();
}
}