From 33a4463ec4f2b0a17f674c28217012bc7533110a Mon Sep 17 00:00:00 2001 From: Alexey Kudravtsev Date: Mon, 29 Apr 2013 18:49:49 +0400 Subject: [PATCH] dictionary compressed by storing alphabet indices in packed bit strings --- .../compress/CompressedDictionary.java | 63 +++---- .../spellchecker/compress/Encoder.java | 51 +++--- .../spellchecker/compress/UnitBitSet.java | 87 ++++++++-- .../spellchecker/engine/Suggestion.java | 12 +- .../spellchecker/compress/DictionaryTest.java | 61 ++++++- .../compress/EncodeAndCompressTest.java | 8 +- .../spellchecker/compress/EncoderTest.java | 157 ++++++++---------- .../spellchecker/compress/UnitBitSetTest.java | 40 +---- 8 files changed, 263 insertions(+), 216 deletions(-) diff --git a/spellchecker/src/com/intellij/spellchecker/compress/CompressedDictionary.java b/spellchecker/src/com/intellij/spellchecker/compress/CompressedDictionary.java index 2e7587ff2c9f..9e7b66dd04cd 100644 --- a/spellchecker/src/com/intellij/spellchecker/compress/CompressedDictionary.java +++ b/spellchecker/src/com/intellij/spellchecker/compress/CompressedDictionary.java @@ -18,7 +18,9 @@ package com.intellij.spellchecker.compress; import com.intellij.spellchecker.dictionary.Dictionary; import com.intellij.spellchecker.dictionary.Loader; import com.intellij.spellchecker.engine.Transformation; +import com.intellij.util.ArrayUtil; import com.intellij.util.Consumer; +import gnu.trove.THashSet; import gnu.trove.TIntObjectHashMap; import gnu.trove.TIntObjectProcedure; import org.jetbrains.annotations.NonNls; @@ -44,13 +46,13 @@ public final class CompressedDictionary implements Dictionary { } }; - CompressedDictionary(@NotNull Alphabet alphabet, @NotNull Encoder encoder, @NotNull String name) { + private CompressedDictionary(@NotNull Alphabet alphabet, @NotNull Encoder encoder, @NotNull String name) { this.alphabet = alphabet; this.encoder = encoder; this.name = name; } - void addToDictionary(@NotNull byte[] word) { + private void addToDictionary(@NotNull byte[] word) { SortedSet set = rawData.get(word.length); if (set == null) { set = createSet(); @@ -60,20 +62,21 @@ public final class CompressedDictionary implements Dictionary { wordsCount++; } - void pack() { + private void pack() { lengths = new int[rawData.size()]; words = new byte[rawData.size()][]; rawData.forEachEntry(new TIntObjectProcedure>() { int row = 0; @Override - public boolean execute(int l, SortedSet value) { - lengths[row] = l; - words[row] = new byte[value.size() * l]; + public boolean execute(int length, SortedSet value) { + lengths[row] = length; + words[row] = new byte[value.size() * length]; int k = 0; + byte[] wordBytes = words[row]; for (byte[] bytes : value) { - for (byte aByte : bytes) { - words[row][k++] = aByte; - } + assert bytes.length == length; + System.arraycopy(bytes, 0, wordBytes, k, bytes.length); + k += bytes.length; } row++; return true; @@ -97,15 +100,14 @@ public final class CompressedDictionary implements Dictionary { int i = 0; for (byte[] data : words) { int length = lengths[i]; + if (length < minLength || length > maxLength) continue; for (int x = 0; x < data.length; x += length) { - byte[] toTest = new byte[length]; - System.arraycopy(data, x, toTest, 0, length); - if (toTest[1] != index || toTest[0] > maxLength || toTest[0] < minLength) { - continue; + if (encoder.getFirstLetterIndex(data[x]) == index) { + byte[] toTest = new byte[length]; + System.arraycopy(data, x, toTest, 0, length); + String decoded = encoder.decode(toTest); + result.add(decoded); } - UnitBitSet set = UnitBitSet.create(toTest); - String decoded = encoder.decode(set); - if(decoded!=null) result.add(decoded); } i++; } @@ -127,20 +129,12 @@ public final class CompressedDictionary implements Dictionary { @Nullable public Boolean contains(@NotNull String word) { UnitBitSet bs = encoder.encode(word, false); - if (bs == Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS) - return null; + if (bs == Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS) return null; if (bs == null) return false; //TODO throw new EncodingException("WORD_WITH_SOME_UNKNOWN_LETTERS"); - byte[] compressed = UnitBitSet.getBytes(bs); - int index = -1; - for (int i = 0; i < lengths.length; i++) { - if (lengths[i] == compressed.length) { - index = i; - break; - } - } + byte[] compressed = bs.pack(); + int index = ArrayUtil.indexOf(lengths, compressed.length); return index != -1 && contains(compressed, words[index]); - } @Override @@ -155,7 +149,12 @@ public final class CompressedDictionary implements Dictionary { @Override public Set getWords() { - throw new UnsupportedOperationException(); + Set words = new THashSet(); + for (int i=0; i<=alphabet.getLastIndexUsed();i++) { + char letter = alphabet.getLetter(i); + words.addAll(getWords(letter)); + } + return words; } @Override @@ -178,6 +177,7 @@ public final class CompressedDictionary implements Dictionary { Alphabet alphabet = new Alphabet(); final Encoder encoder = new Encoder(alphabet); final CompressedDictionary dictionary = new CompressedDictionary(alphabet, encoder, loader.getName()); + final List bss = new ArrayList(); loader.load(new Consumer() { @Override public void consume(String s) { @@ -185,11 +185,14 @@ public final class CompressedDictionary implements Dictionary { if (transformed != null) { UnitBitSet bs = encoder.encode(transformed, true); if (bs == null) return; - byte[] compressed = UnitBitSet.getBytes(bs); - dictionary.addToDictionary(compressed); + bss.add(bs); } } }); + for (UnitBitSet bs : bss) { + byte[] compressed = bs.pack(); + dictionary.addToDictionary(compressed); + } dictionary.pack(); return dictionary; } diff --git a/spellchecker/src/com/intellij/spellchecker/compress/Encoder.java b/spellchecker/src/com/intellij/spellchecker/compress/Encoder.java index 2f4025f36efa..d2b1e41139c7 100644 --- a/spellchecker/src/com/intellij/spellchecker/compress/Encoder.java +++ b/spellchecker/src/com/intellij/spellchecker/compress/Encoder.java @@ -19,13 +19,11 @@ import com.intellij.openapi.diagnostic.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.text.MessageFormat; - public final class Encoder { private final Alphabet alphabet; - private static final int offset = 2; - static final UnitBitSet WORD_OF_ENTIRELY_UNKNOWN_LETTERS = new UnitBitSet(); + private static final int offset = 0; + static final UnitBitSet WORD_OF_ENTIRELY_UNKNOWN_LETTERS = new UnitBitSet(new byte[1],new Alphabet()); private static final Logger LOG = Logger.getInstance("#com.intellij.spellchecker.compress"); public Encoder() { @@ -44,39 +42,28 @@ public final class Encoder { public UnitBitSet encode(@NotNull CharSequence letters, boolean force) { if (UnitBitSet.MAX_CHARS_IN_WORD <= letters.length() + offset) return null; int unknownLetters = 0; - UnitBitSet bs = new UnitBitSet(); + byte[] indices = new byte[letters.length()]; for (int i = 0; i < letters.length(); i++) { char letter = letters.charAt(i); int index = alphabet.getIndex(letter, force); - if (index < 0) unknownLetters++; - bs.setUnitValue(i + offset, index); - } - bs.setUnitValue(0, letters.length()); - bs.setUnitValue(1, bs.getUnitValue(2)); - if (unknownLetters == letters.length()) return WORD_OF_ENTIRELY_UNKNOWN_LETTERS; - if (unknownLetters>0) return null; - return bs; - } - - @Nullable - public String decode(@NotNull UnitBitSet bitSet) { - int wordLength = bitSet.getUnitValue(0); - char firstLetter = alphabet.getLetter(bitSet.getUnitValue(1)); - final StringBuilder result = new StringBuilder(); - for (int i = 2; i < bitSet.b.length; i++) { - int value = bitSet.getUnitValue(i); - if (value > 0 && value <= alphabet.getLastIndexUsed()) { - result.append(alphabet.getLetter(value)); + if (index < 0) { + unknownLetters++; + } + else { + indices[i] = (byte)index; } } + if (unknownLetters == letters.length()) return WORD_OF_ENTIRELY_UNKNOWN_LETTERS; + if (unknownLetters>0) return null; + return new UnitBitSet(indices, alphabet); + } - final String word = result.toString(); - final int actualLength = word.length(); - if (actualLength != wordLength || !word.startsWith(String.valueOf(firstLetter))) { - LOG.error(new MessageFormat("Error during encoding: required length - {0}, starts with {1}, but decoded: {2} ({3})") - .format(new Object[]{wordLength, firstLetter, word, actualLength})); - return null; - } - return word; + @NotNull + public String decode(@NotNull byte[] compressed) { + return UnitBitSet.decode(compressed, alphabet); + } + + public int getFirstLetterIndex(byte firstPackedByte) { + return UnitBitSet.getFirstLetterIndex(firstPackedByte, alphabet); } } \ No newline at end of file diff --git a/spellchecker/src/com/intellij/spellchecker/compress/UnitBitSet.java b/spellchecker/src/com/intellij/spellchecker/compress/UnitBitSet.java index abd2581066ba..b1c0270bc621 100644 --- a/spellchecker/src/com/intellij/spellchecker/compress/UnitBitSet.java +++ b/spellchecker/src/com/intellij/spellchecker/compress/UnitBitSet.java @@ -5,11 +5,16 @@ import org.jetbrains.annotations.NotNull; import java.util.Arrays; public class UnitBitSet { - public static final int MAX_CHARS_IN_WORD = 64; public static final int MAX_UNIT_VALUE = 255; - byte[] b = new byte[MAX_CHARS_IN_WORD]; + final byte[] b; + private final Alphabet alpha; + + public UnitBitSet(@NotNull byte[] indices, @NotNull Alphabet alphabet) { + b = indices; + alpha = alphabet; + } public int getUnitValue(int number) { final int r = b[number] & 0xFF; @@ -18,7 +23,7 @@ public class UnitBitSet { } public void setUnitValue(int number, int value) { - //assert value >= 0 : "unit value is negative" + value; + assert value >= 0 : "unit value is negative" + value; assert value <= MAX_UNIT_VALUE : "unit value is too big"; b[number] = (byte)value; } @@ -32,29 +37,75 @@ public class UnitBitSet { @Override public String toString() { final StringBuilder s = new StringBuilder(); - for (int i = 0; i < b.length; i++) { - s.append(Integer.toHexString((int)b[i] & 0xFF)); + for (byte aB : b) { + s.append(Integer.toHexString((int)aB & 0xFF)); } return s.toString(); } - public static UnitBitSet create(@NotNull UnitBitSet origin) { - UnitBitSet r = new UnitBitSet(); - System.arraycopy(origin.b, 0, r.b, 0, r.b.length); - return r; + @NotNull + public byte[] pack() { + int meaningfulBits = 32 - Integer.numberOfLeadingZeros(alpha.getLastIndexUsed()); + assert meaningfulBits <= 8; + byte[] result = new byte[(b.length * meaningfulBits + 7) / 8]; + + int byteNumber = 0; + int bitOffset = 0; + + for (byte index : b) { + int bitsToChip = Math.min(8 - bitOffset, meaningfulBits); + result[byteNumber] |= (index & ((1 << bitsToChip) - 1)) << bitOffset; + + int bitsLeft = meaningfulBits - bitsToChip; + if (bitsLeft > 0) { + byteNumber++; + result[byteNumber] |= (index >> bitsToChip) & ((1 << bitsLeft) - 1); + bitOffset = bitsLeft; + } + else { + bitOffset += bitsToChip; + } + } + return result; } - public static UnitBitSet create(byte[] value) { - final UnitBitSet r = new UnitBitSet(); - System.arraycopy(value, 0, r.b, 0, value.length); - return r; + @NotNull + public static String decode(@NotNull byte[] packed, @NotNull Alphabet alphabet) { + int meaningfulBits = 32 - Integer.numberOfLeadingZeros(alphabet.getLastIndexUsed()); + assert meaningfulBits <= 8; + + StringBuilder result = new StringBuilder(packed.length * 8 / meaningfulBits); + + int curByte = packed[0]; + int byteIndex = 0; + int bitOffset = 0; + + while (byteIndex < packed.length) { + int index = curByte & ((1 << meaningfulBits) - 1); + char letter = alphabet.getLetter(index); + if (letter == '\u0000') { + break; + } + result.append(letter); + + curByte >>>= meaningfulBits; + bitOffset += meaningfulBits; + assert bitOffset <= 8; + if (bitOffset + meaningfulBits > 8) { + if (++byteIndex == packed.length) break; + int leftOverBits = 8 - bitOffset; + curByte = packed[byteIndex] << leftOverBits | (curByte & ((1 << leftOverBits) - 1)); + bitOffset = -leftOverBits; + } + } + return result.toString(); } + public static int getFirstLetterIndex(byte firstPackedByte, @NotNull Alphabet alphabet) { + int meaningfulBits = 32 - Integer.numberOfLeadingZeros(alphabet.getLastIndexUsed()); + assert meaningfulBits <= 8; - static public byte[] getBytes(UnitBitSet origin) { - final byte[] r = new byte[origin.b[0] + 2]; - System.arraycopy(origin.b, 0, r, 0, r.length); - return r; + int index = firstPackedByte & ((1 << meaningfulBits) - 1); + return index; } - } diff --git a/spellchecker/src/com/intellij/spellchecker/engine/Suggestion.java b/spellchecker/src/com/intellij/spellchecker/engine/Suggestion.java index b5f83e7c5ee4..c88507c05990 100644 --- a/spellchecker/src/com/intellij/spellchecker/engine/Suggestion.java +++ b/spellchecker/src/com/intellij/spellchecker/engine/Suggestion.java @@ -15,6 +15,8 @@ */ package com.intellij.spellchecker.engine; +import com.intellij.openapi.util.text.StringUtil; + public class Suggestion implements Comparable{ private final String word; private final int metrics; @@ -54,10 +56,18 @@ public class Suggestion implements Comparable{ } + @Override public int compareTo(Object o) { if (!(o instanceof Suggestion)) throw new IllegalArgumentException(); Suggestion r = (Suggestion)o; - return new Integer(getMetrics()).compareTo(r.getMetrics()); + int c = new Integer(getMetrics()).compareTo(r.getMetrics()); + if (c !=0) return c; + return StringUtil.compare(word, r.word, true); + } + + @Override + public String toString() { + return word + " : " + metrics; } } diff --git a/spellchecker/testSrc/com/intellij/spellchecker/compress/DictionaryTest.java b/spellchecker/testSrc/com/intellij/spellchecker/compress/DictionaryTest.java index ad0acb20c8e5..c2ce1d39c64a 100644 --- a/spellchecker/testSrc/com/intellij/spellchecker/compress/DictionaryTest.java +++ b/spellchecker/testSrc/com/intellij/spellchecker/compress/DictionaryTest.java @@ -16,6 +16,8 @@ package com.intellij.spellchecker.compress; import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; import com.intellij.spellchecker.DefaultBundledDictionariesProvider; import com.intellij.spellchecker.StreamLoader; import com.intellij.spellchecker.dictionary.Dictionary; @@ -28,10 +30,9 @@ import gnu.trove.THashSet; import junit.framework.TestCase; import org.jetbrains.annotations.NotNull; +import java.io.File; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; +import java.util.*; @SuppressWarnings({"JUnitTestCaseWithNonTrivialConstructors"}) public class DictionaryTest extends TestCase { @@ -61,6 +62,58 @@ public class DictionaryTest extends TestCase { } } + public void testDictionaryLoadedFully() { + //cleanupDictionary(); + final Transformation transform = new Transformation(); + CompressedDictionary dictionary = CompressedDictionary.create(englishLoader(), transform); + + final Set onDisk = new THashSet(); + englishLoader().load(new Consumer() { + @Override + public void consume(String s) { + assert s != null; + String t = transform.transform(s); + if (t == null) { + return; + } + onDisk.add(t); + } + }); + + List odList = new ArrayList(onDisk); + Collections.sort(odList); + List loaded = new ArrayList(dictionary.getWords()); + Collections.sort(loaded); + + assertEquals(odList, loaded); + } + + public void cleanupDictionary() { + final Set onDisk = new THashSet(FileUtil.PATH_HASHING_STRATEGY); + englishLoader().load(new Consumer() { + @Override + public void consume(String s) { + assert s != null; + onDisk.add(s); + } + }); + + List odList = new ArrayList(onDisk); + Collections.sort(odList); + + File file = new File("C:\\Work\\Idea\\community\\spellchecker\\src\\com\\intellij\\spellchecker\\english.2"); + try { + FileUtil.writeToFile(file, StringUtil.join(odList, "\n")); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static StreamLoader englishLoader() { + return new StreamLoader(DefaultBundledDictionariesProvider.class.getResourceAsStream(ENGLISH_DIC), ENGLISH_DIC); + } + public void loadDictionaryTest(@NotNull final String name, int wordCount) throws IOException { final Transformation transform = new Transformation(); PlatformTestUtil.startPerformanceTest("load dictionary", times.get(name), new ThrowableRunnable() { @@ -130,7 +183,7 @@ public class DictionaryTest extends TestCase { } - public static void loadHalfDictionaryTest(final String name, final int maxCount) throws IOException { + public static void loadHalfDictionaryTest(final String name, final int maxCount) { final Pair, Set> sets = createWordSets(name, maxCount, 2); final Loader loader = createLoader(sets.getFirst()); CompressedDictionary dictionary = CompressedDictionary.create(loader, new Transformation()); diff --git a/spellchecker/testSrc/com/intellij/spellchecker/compress/EncodeAndCompressTest.java b/spellchecker/testSrc/com/intellij/spellchecker/compress/EncodeAndCompressTest.java index c43aa54d7432..96d70d802d70 100644 --- a/spellchecker/testSrc/com/intellij/spellchecker/compress/EncodeAndCompressTest.java +++ b/spellchecker/testSrc/com/intellij/spellchecker/compress/EncodeAndCompressTest.java @@ -9,10 +9,10 @@ public class EncodeAndCompressTest extends TestCase { Encoder encoder = new Encoder(); String word = "example"; UnitBitSet bs = encoder.encode(word, true); - byte[] compressed = UnitBitSet.getBytes(bs); - final UnitBitSet decompressed = UnitBitSet.create(compressed); - assertEquals(bs,decompressed); - String restored = encoder.decode(decompressed); + byte[] compressed = bs.pack(); + final String decompressed = UnitBitSet.decode(compressed, encoder.getAlphabet()); + assertEquals(word,decompressed); + String restored = encoder.decode(compressed); assertEquals(word,restored); } diff --git a/spellchecker/testSrc/com/intellij/spellchecker/compress/EncoderTest.java b/spellchecker/testSrc/com/intellij/spellchecker/compress/EncoderTest.java index ce7eecf366bd..1c90ddf0a6dd 100644 --- a/spellchecker/testSrc/com/intellij/spellchecker/compress/EncoderTest.java +++ b/spellchecker/testSrc/com/intellij/spellchecker/compress/EncoderTest.java @@ -18,107 +18,88 @@ package com.intellij.spellchecker.compress; import junit.framework.TestCase; public class EncoderTest extends TestCase { + public void testSimple() { + Encoder encoder = new Encoder(); + final String wordToTest = "abc"; + final UnitBitSet bitSet = encoder.encode(wordToTest, true); + assertNotNull(bitSet); + assertIndices(bitSet, 1, 2, 3); + byte[] compressed = bitSet.pack(); + assertEquals(1, compressed.length); + assertEquals(wordToTest, encoder.decode(compressed)); + } - - public void testSimple() { - Encoder encoder = new Encoder(); - final String wordToTest = "abc"; - final UnitBitSet bitSet = encoder.encode(wordToTest, true); - assertNotNull(bitSet); - assertEquals(3, encoder.getAlphabet().getLastIndexUsed()); - assertEquals(3, bitSet.getUnitValue(0)); - assertEquals(1, bitSet.getUnitValue(1)); - assertEquals(1, bitSet.getUnitValue(2)); - assertEquals(2, bitSet.getUnitValue(3)); - assertEquals(3, bitSet.getUnitValue(4)); - - assertEquals(wordToTest, encoder.decode(bitSet)); + private static void assertIndices(UnitBitSet bitSet, int... indices) { + assertEquals(indices.length, bitSet.b.length); + for (int i = 0; i < indices.length; i++) { + int index = indices[i]; + assertEquals(index, bitSet.getUnitValue(i)); } + } - public void testDouble() { - Encoder encoder = new Encoder(); - final String wordToTest = "aaa"; - final UnitBitSet bitSet = encoder.encode(wordToTest, true); - assertNotNull(bitSet); - assertEquals(1, encoder.getAlphabet().getLastIndexUsed()); - assertEquals(3, bitSet.getUnitValue(0)); - assertEquals(1, bitSet.getUnitValue(1)); - assertEquals(1, bitSet.getUnitValue(2)); - assertEquals(1, bitSet.getUnitValue(3)); - assertEquals(1, bitSet.getUnitValue(4)); + public void testDouble() { + Encoder encoder = new Encoder(); + final String wordToTest = "aaa"; + final UnitBitSet bitSet = encoder.encode(wordToTest, true); + assertNotNull(bitSet); + assertEquals(1, encoder.getAlphabet().getLastIndexUsed()); + assertIndices(bitSet, 1, 1, 1); - assertEquals(wordToTest, encoder.decode(bitSet)); - } + assertEquals(wordToTest, encoder.decode(bitSet.pack())); + } - public void testLetterRepetition() { - Encoder encoder = new Encoder(); - final String wordToTest = "aba"; - final UnitBitSet bitSet = encoder.encode(wordToTest, true); - assertNotNull(bitSet); - assertEquals(2, encoder.getAlphabet().getLastIndexUsed()); - assertEquals(3, bitSet.getUnitValue(0)); - assertEquals(1, bitSet.getUnitValue(1)); - assertEquals(1, bitSet.getUnitValue(2)); - assertEquals(2, bitSet.getUnitValue(3)); - assertEquals(1, bitSet.getUnitValue(4)); + public void testLetterRepetition() { + Encoder encoder = new Encoder(); + final String wordToTest = "aba"; + final UnitBitSet bitSet = encoder.encode(wordToTest, true); + assertNotNull(bitSet); + assertEquals(2, encoder.getAlphabet().getLastIndexUsed()); + assertIndices(bitSet, 1, 2, 1); - assertEquals(wordToTest, encoder.decode(bitSet)); - } + assertEquals(wordToTest, encoder.decode(bitSet.pack())); + } - public void testReverse() { - Encoder encoder = new Encoder(); - final String wordToTest1 = "abc"; - final UnitBitSet bitSet = encoder.encode(wordToTest1, true); - assertNotNull(bitSet); - assertEquals(3, encoder.getAlphabet().getLastIndexUsed()); - assertEquals(3, bitSet.getUnitValue(0)); - assertEquals(1, bitSet.getUnitValue(1)); - assertEquals(1, bitSet.getUnitValue(2)); - assertEquals(2, bitSet.getUnitValue(3)); - assertEquals(3, bitSet.getUnitValue(4)); + public void testReverse() { + Encoder encoder = new Encoder(); + final String wordToTest1 = "abc"; + final UnitBitSet bitSet = encoder.encode(wordToTest1, true); + assertNotNull(bitSet); + assertEquals(3, encoder.getAlphabet().getLastIndexUsed()); + assertIndices(bitSet, 1, 2, 3); - assertEquals(wordToTest1, encoder.decode(bitSet)); + byte[] pack = bitSet.pack(); + assertEquals(1, pack.length); + assertEquals(wordToTest1, encoder.decode(pack)); - final String wordToTest2 = "cba"; - final UnitBitSet bitSet2 = encoder.encode(wordToTest2, true); - assertEquals(3, encoder.getAlphabet().getLastIndexUsed()); - assertNotNull(bitSet); - assertEquals(3, bitSet2.getUnitValue(0)); - assertEquals(3, bitSet2.getUnitValue(1)); - assertEquals(3, bitSet2.getUnitValue(2)); - assertEquals(2, bitSet2.getUnitValue(3)); - assertEquals(1, bitSet2.getUnitValue(4)); + final String wordToTest2 = "cba"; + final UnitBitSet bitSet2 = encoder.encode(wordToTest2, true); + assertEquals(3, encoder.getAlphabet().getLastIndexUsed()); + assertNotNull(bitSet2); + assertIndices(bitSet2, 3, 2, 1); - assertEquals(wordToTest2, encoder.decode(bitSet2)); - } + byte[] pack2 = bitSet2.pack(); + assertEquals(1, pack2.length); + assertEquals(wordToTest2, encoder.decode(pack2)); + } - public void testWithPredefinedAlphabet() { - Encoder encoder = new Encoder(new Alphabet("abcdefghijklmnopqrst")); - final String wordToTest1 = "asia"; - //letter 'a' will be added at the end - final UnitBitSet bitSet = encoder.encode(wordToTest1, true); - assertNotNull(bitSet); - assertEquals(20, encoder.getAlphabet().getLastIndexUsed()); - assertEquals(4, bitSet.getUnitValue(0)); - assertEquals(1, bitSet.getUnitValue(1)); - assertEquals(1, bitSet.getUnitValue(2)); - assertEquals(19, bitSet.getUnitValue(3)); - assertEquals(9, bitSet.getUnitValue(4)); - assertEquals(1, bitSet.getUnitValue(5)); - - assertEquals(wordToTest1, encoder.decode(bitSet)); - - - } - - public void testUnknown() { - Encoder encoder = new Encoder(new Alphabet("abc")); - final String wordToTest1 = "def"; - final UnitBitSet bitSet = encoder.encode(wordToTest1, true); - assertEquals(bitSet, Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS); - } + public void testWithPredefinedAlphabet() { + Encoder encoder = new Encoder(new Alphabet("abcdefghijklmnopqrst")); + final String wordToTest1 = "asia"; + //letter 'a' will be added at the end + final UnitBitSet bitSet = encoder.encode(wordToTest1, true); + assertNotNull(bitSet); + assertEquals(20, encoder.getAlphabet().getLastIndexUsed()); + assertIndices(bitSet, 1, 19, 9,1); + assertEquals(wordToTest1, encoder.decode(bitSet.pack())); + } + public void testUnknown() { + Encoder encoder = new Encoder(new Alphabet("abc")); + final String wordToTest1 = "def"; + final UnitBitSet bitSet = encoder.encode(wordToTest1, true); + assertEquals(bitSet, Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS); + } } diff --git a/spellchecker/testSrc/com/intellij/spellchecker/compress/UnitBitSetTest.java b/spellchecker/testSrc/com/intellij/spellchecker/compress/UnitBitSetTest.java index dbf7169cc5e1..a4f6e2c4f310 100644 --- a/spellchecker/testSrc/com/intellij/spellchecker/compress/UnitBitSetTest.java +++ b/spellchecker/testSrc/com/intellij/spellchecker/compress/UnitBitSetTest.java @@ -17,56 +17,18 @@ package com.intellij.spellchecker.compress; import junit.framework.TestCase; -import java.util.Random; - public class UnitBitSetTest extends TestCase { public void testUnitValue() { int bitsPerUnit = 256; for (int i = 0; i < bitsPerUnit - 1; i++) { - UnitBitSet bs = new UnitBitSet(); + UnitBitSet bs = new UnitBitSet(new byte[2], new Alphabet()); bs.setUnitValue(0, i); assertEquals(i, bs.getUnitValue(0)); assertEquals(0, bs.getUnitValue(1)); } } - public void testCreateFromBitSet() { - UnitBitSet bs1 = new UnitBitSet(); - bs1.setUnitValue(5, 255); - UnitBitSet bs2 = UnitBitSet.create(bs1); - assertEquals(bs2, bs1); - } - public void testCompressorWithRandomData() { - int amount = 10; - for (int i = 0; i < 1000; i++) { - int[] values = getRandoms(UnitBitSet.MAX_UNIT_VALUE, amount); - final UnitBitSet bitSet = new UnitBitSet(); - bitSet.setUnitValue(0, values.length); - for (int i1 = 1, valuesLength = values.length; i1 < valuesLength; i1++) { - int value = values[i1]; - bitSet.setUnitValue(i1, value); - } - final byte[] compressed = UnitBitSet.getBytes(bitSet); - final UnitBitSet decompressed = UnitBitSet.create(compressed); - boolean check = (bitSet.equals(decompressed)); - if (!check) { - System.out.println("bitSet: " + bitSet); - System.out.println("decompressed: " + decompressed); - UnitBitSet.getBytes(bitSet); - UnitBitSet.create(compressed); - } - assertEquals(bitSet, decompressed); - } - } - - private static int[] getRandoms(int max, int count) { - int[] result = new int[count]; - for (int i = 0; i < count; i++) { - result[i] = new Random().nextInt(max); - } - return result; - } }