dictionary compressed by storing alphabet indices in packed bit strings

spellchecker/src/com/intellij/spellchecker/compress/CompressedDictionary.java

@@ -18,7 +18,9 @@ package com.intellij.spellchecker.compress;
 import com.intellij.spellchecker.dictionary.Dictionary;
 import com.intellij.spellchecker.dictionary.Loader;
 import com.intellij.spellchecker.engine.Transformation;
+import com.intellij.util.ArrayUtil;
 import com.intellij.util.Consumer;
+import gnu.trove.THashSet;
 import gnu.trove.TIntObjectHashMap;
 import gnu.trove.TIntObjectProcedure;
 import org.jetbrains.annotations.NonNls;
@@ -44,13 +46,13 @@ public final class CompressedDictionary implements Dictionary {
     }
   };
 
-  CompressedDictionary(@NotNull Alphabet alphabet, @NotNull Encoder encoder, @NotNull String name) {
+  private CompressedDictionary(@NotNull Alphabet alphabet, @NotNull Encoder encoder, @NotNull String name) {
     this.alphabet = alphabet;
     this.encoder = encoder;
     this.name = name;
   }
 
-  void addToDictionary(@NotNull byte[] word) {
+  private void addToDictionary(@NotNull byte[] word) {
     SortedSet<byte[]> set = rawData.get(word.length);
     if (set == null) {
       set = createSet();
@@ -60,20 +62,21 @@ public final class CompressedDictionary implements Dictionary {
     wordsCount++;
   }
 
-  void pack() {
+  private void pack() {
     lengths = new int[rawData.size()];
     words = new byte[rawData.size()][];
     rawData.forEachEntry(new TIntObjectProcedure<SortedSet<byte[]>>() {
       int row = 0;
       @Override
-      public boolean execute(int l, SortedSet<byte[]> value) {
-        lengths[row] = l;
-        words[row] = new byte[value.size() * l];
+      public boolean execute(int length, SortedSet<byte[]> value) {
+        lengths[row] = length;
+        words[row] = new byte[value.size() * length];
         int k = 0;
+        byte[] wordBytes = words[row];
         for (byte[] bytes : value) {
-          for (byte aByte : bytes) {
-            words[row][k++] = aByte;
-          }
+          assert bytes.length == length;
+          System.arraycopy(bytes, 0, wordBytes, k, bytes.length);
+          k += bytes.length;
         }
         row++;
         return true;
@@ -97,15 +100,14 @@ public final class CompressedDictionary implements Dictionary {
     int i = 0;
     for (byte[] data : words) {
       int length = lengths[i];
+      if (length < minLength || length > maxLength) continue;
       for (int x = 0; x < data.length; x += length) {
-        byte[] toTest = new byte[length];
-        System.arraycopy(data, x, toTest, 0, length);
-        if (toTest[1] != index || toTest[0] > maxLength || toTest[0] < minLength) {
-          continue;
+        if (encoder.getFirstLetterIndex(data[x]) == index) {
+          byte[] toTest = new byte[length];
+          System.arraycopy(data, x, toTest, 0, length);
+          String decoded = encoder.decode(toTest);
+          result.add(decoded);
         }
-        UnitBitSet set = UnitBitSet.create(toTest);
-        String decoded = encoder.decode(set);
-        if(decoded!=null) result.add(decoded);
       }
       i++;
     }
@@ -127,20 +129,12 @@ public final class CompressedDictionary implements Dictionary {
   @Nullable
   public Boolean contains(@NotNull String word) {
     UnitBitSet bs = encoder.encode(word, false);
-    if (bs == Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS)
-      return null;
+    if (bs == Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS) return null;
     if (bs == null) return false;
       //TODO throw new EncodingException("WORD_WITH_SOME_UNKNOWN_LETTERS");
-    byte[] compressed = UnitBitSet.getBytes(bs);
-    int index = -1;
-    for (int i = 0; i < lengths.length; i++) {
-      if (lengths[i] == compressed.length) {
-        index = i;
-        break;
-      }
-    }
+    byte[] compressed = bs.pack();
+    int index = ArrayUtil.indexOf(lengths, compressed.length);
     return index != -1 && contains(compressed, words[index]);
-
   }
 
   @Override
@@ -155,7 +149,12 @@ public final class CompressedDictionary implements Dictionary {
 
   @Override
   public Set<String> getWords() {
-    throw new UnsupportedOperationException();
+    Set<String> words = new THashSet<String>();
+    for (int i=0; i<=alphabet.getLastIndexUsed();i++) {
+      char letter = alphabet.getLetter(i);
+      words.addAll(getWords(letter));
+    }
+    return words;
   }
 
   @Override
@@ -178,6 +177,7 @@ public final class CompressedDictionary implements Dictionary {
     Alphabet alphabet = new Alphabet();
     final Encoder encoder = new Encoder(alphabet);
     final CompressedDictionary dictionary = new CompressedDictionary(alphabet, encoder, loader.getName());
+    final List<UnitBitSet> bss = new ArrayList<UnitBitSet>();
     loader.load(new Consumer<String>() {
       @Override
       public void consume(String s) {
@@ -185,11 +185,14 @@ public final class CompressedDictionary implements Dictionary {
         if (transformed != null) {
           UnitBitSet bs = encoder.encode(transformed, true);
           if (bs == null) return;
-          byte[] compressed = UnitBitSet.getBytes(bs);
-          dictionary.addToDictionary(compressed);
+          bss.add(bs);
         }
       }
     });
+    for (UnitBitSet bs : bss) {
+      byte[] compressed = bs.pack();
+      dictionary.addToDictionary(compressed);
+    }
     dictionary.pack();
     return dictionary;
   }

spellchecker/src/com/intellij/spellchecker/compress/Encoder.java

@@ -19,13 +19,11 @@ import com.intellij.openapi.diagnostic.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.text.MessageFormat;
-
 public final class Encoder {
 
   private final Alphabet alphabet;
-  private static final int offset = 2;
-  static final UnitBitSet WORD_OF_ENTIRELY_UNKNOWN_LETTERS = new UnitBitSet();
+  private static final int offset = 0;
+  static final UnitBitSet WORD_OF_ENTIRELY_UNKNOWN_LETTERS = new UnitBitSet(new byte[1],new Alphabet());
   private static final Logger LOG = Logger.getInstance("#com.intellij.spellchecker.compress");
 
   public Encoder() {
@@ -44,39 +42,28 @@ public final class Encoder {
   public UnitBitSet encode(@NotNull CharSequence letters, boolean force) {
     if (UnitBitSet.MAX_CHARS_IN_WORD <= letters.length() + offset) return null;
     int unknownLetters = 0;
-    UnitBitSet bs = new UnitBitSet();
+    byte[] indices = new byte[letters.length()];
     for (int i = 0; i < letters.length(); i++) {
       char letter = letters.charAt(i);
       int index = alphabet.getIndex(letter, force);
-      if (index < 0) unknownLetters++;
-      bs.setUnitValue(i + offset, index);
-    }
-    bs.setUnitValue(0, letters.length());
-    bs.setUnitValue(1, bs.getUnitValue(2));
-    if (unknownLetters == letters.length()) return WORD_OF_ENTIRELY_UNKNOWN_LETTERS;
-    if (unknownLetters>0) return null;
-    return bs;
-  }
-
-  @Nullable
-  public String decode(@NotNull UnitBitSet bitSet) {
-    int wordLength = bitSet.getUnitValue(0);
-    char firstLetter = alphabet.getLetter(bitSet.getUnitValue(1));
-    final StringBuilder result = new StringBuilder();
-    for (int i = 2; i < bitSet.b.length; i++) {
-      int value = bitSet.getUnitValue(i);
-      if (value > 0 && value <= alphabet.getLastIndexUsed()) {
-        result.append(alphabet.getLetter(value));
+      if (index < 0) {
+        unknownLetters++;
+      }
+      else {
+        indices[i] = (byte)index;
       }
     }
+    if (unknownLetters == letters.length()) return WORD_OF_ENTIRELY_UNKNOWN_LETTERS;
+    if (unknownLetters>0) return null;
+    return new UnitBitSet(indices, alphabet);
+  }
 
-    final String word = result.toString();
-    final int actualLength = word.length();
-    if (actualLength != wordLength || !word.startsWith(String.valueOf(firstLetter))) {
-      LOG.error(new MessageFormat("Error during encoding: required length - {0}, starts with {1}, but decoded: {2} ({3})")
-                  .format(new Object[]{wordLength, firstLetter, word, actualLength}));
-      return null;
-    }
-    return word;
+  @NotNull
+  public String decode(@NotNull byte[] compressed) {
+    return UnitBitSet.decode(compressed, alphabet);
+  }
+
+  public int getFirstLetterIndex(byte firstPackedByte) {
+    return UnitBitSet.getFirstLetterIndex(firstPackedByte, alphabet);
   }
 }

spellchecker/src/com/intellij/spellchecker/compress/UnitBitSet.java

@@ -5,11 +5,16 @@ import org.jetbrains.annotations.NotNull;
 import java.util.Arrays;
 
 public class UnitBitSet {
-
   public static final int MAX_CHARS_IN_WORD = 64;
   public static final int MAX_UNIT_VALUE = 255;
 
-  byte[] b = new byte[MAX_CHARS_IN_WORD];
+  final byte[] b;
+  private final Alphabet alpha;
+
+  public UnitBitSet(@NotNull byte[] indices, @NotNull Alphabet alphabet) {
+    b = indices;
+    alpha = alphabet;
+  }
 
   public int getUnitValue(int number) {
     final int r = b[number] & 0xFF;
@@ -18,7 +23,7 @@ public class UnitBitSet {
   }
 
   public void setUnitValue(int number, int value) {
-    //assert value >= 0 : "unit value is negative" + value;
+    assert value >= 0 : "unit value is negative" + value;
     assert value <= MAX_UNIT_VALUE : "unit value is too big";
     b[number] = (byte)value;
   }
@@ -32,29 +37,75 @@ public class UnitBitSet {
   @Override
   public String toString() {
     final StringBuilder s = new StringBuilder();
-    for (int i = 0; i < b.length; i++) {
-      s.append(Integer.toHexString((int)b[i] & 0xFF));
+    for (byte aB : b) {
+      s.append(Integer.toHexString((int)aB & 0xFF));
     }
     return s.toString();
   }
 
-  public static UnitBitSet create(@NotNull UnitBitSet origin) {
-    UnitBitSet r = new UnitBitSet();
-    System.arraycopy(origin.b, 0, r.b, 0, r.b.length);
-    return r;
+  @NotNull
+  public byte[] pack() {
+    int meaningfulBits = 32 - Integer.numberOfLeadingZeros(alpha.getLastIndexUsed());
+    assert meaningfulBits <= 8;
+    byte[] result = new byte[(b.length * meaningfulBits + 7) / 8];
+
+    int byteNumber = 0;
+    int bitOffset = 0;
+
+    for (byte index : b) {
+      int bitsToChip = Math.min(8 - bitOffset, meaningfulBits);
+      result[byteNumber] |= (index & ((1 << bitsToChip) - 1)) << bitOffset;
+
+      int bitsLeft = meaningfulBits - bitsToChip;
+      if (bitsLeft > 0) {
+        byteNumber++;
+        result[byteNumber] |= (index >> bitsToChip) & ((1 << bitsLeft) - 1);
+        bitOffset = bitsLeft;
+      }
+      else {
+        bitOffset += bitsToChip;
+      }
+    }
+    return result;
   }
 
-  public static UnitBitSet create(byte[] value) {
-    final UnitBitSet r = new UnitBitSet();
-    System.arraycopy(value, 0, r.b, 0, value.length);
-    return r;
+  @NotNull
+  public static String decode(@NotNull byte[] packed, @NotNull Alphabet alphabet) {
+    int meaningfulBits = 32 - Integer.numberOfLeadingZeros(alphabet.getLastIndexUsed());
+    assert meaningfulBits <= 8;
+
+    StringBuilder result = new StringBuilder(packed.length * 8 / meaningfulBits);
+
+    int curByte = packed[0];
+    int byteIndex = 0;
+    int bitOffset = 0;
+
+    while (byteIndex < packed.length) {
+      int index = curByte & ((1 << meaningfulBits) - 1);
+      char letter = alphabet.getLetter(index);
+      if (letter == '\u0000') {
+        break;
+      }
+      result.append(letter);
+
+      curByte >>>= meaningfulBits;
+      bitOffset += meaningfulBits;
+      assert bitOffset <= 8;
+      if (bitOffset + meaningfulBits > 8) {
+        if (++byteIndex == packed.length) break;
+        int leftOverBits = 8 - bitOffset;
+        curByte = packed[byteIndex] << leftOverBits | (curByte & ((1 << leftOverBits) - 1));
+        bitOffset = -leftOverBits;
+      }
+    }
+    return result.toString();
   }
 
+  public static int getFirstLetterIndex(byte firstPackedByte, @NotNull Alphabet alphabet) {
+    int meaningfulBits = 32 - Integer.numberOfLeadingZeros(alphabet.getLastIndexUsed());
+    assert meaningfulBits <= 8;
 
-  static public byte[] getBytes(UnitBitSet origin) {
-    final byte[] r = new byte[origin.b[0] + 2];
-    System.arraycopy(origin.b, 0, r, 0, r.length);
-    return r;
+    int index = firstPackedByte & ((1 << meaningfulBits) - 1);
+    return index;
   }
-
 }

spellchecker/src/com/intellij/spellchecker/engine/Suggestion.java

@@ -15,6 +15,8 @@
  */
 package com.intellij.spellchecker.engine;
 
+import com.intellij.openapi.util.text.StringUtil;
+
 public class Suggestion implements Comparable{
   private final String word;
   private final int metrics;
@@ -54,10 +56,18 @@ public class Suggestion implements Comparable{
   }
 
 
+  @Override
   public int compareTo(Object o) {
     if (!(o instanceof Suggestion)) throw new IllegalArgumentException();
     Suggestion r = (Suggestion)o;
-    return new Integer(getMetrics()).compareTo(r.getMetrics());
+    int c = new Integer(getMetrics()).compareTo(r.getMetrics());
+    if (c !=0) return c;
+    return StringUtil.compare(word, r.word, true);
+  }
+
+  @Override
+  public String toString() {
+    return word + " : " + metrics;
   }
 }
 

spellchecker/testSrc/com/intellij/spellchecker/compress/DictionaryTest.java

@@ -16,6 +16,8 @@
 package com.intellij.spellchecker.compress;
 
 import com.intellij.openapi.util.Pair;
+import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.spellchecker.DefaultBundledDictionariesProvider;
 import com.intellij.spellchecker.StreamLoader;
 import com.intellij.spellchecker.dictionary.Dictionary;
@@ -28,10 +30,9 @@ import gnu.trove.THashSet;
 import junit.framework.TestCase;
 import org.jetbrains.annotations.NotNull;
 
+import java.io.File;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 @SuppressWarnings({"JUnitTestCaseWithNonTrivialConstructors"})
 public class DictionaryTest extends TestCase {
@@ -61,6 +62,58 @@ public class DictionaryTest extends TestCase {
     }
   }
 
+  public void testDictionaryLoadedFully() {
+    //cleanupDictionary();
+    final Transformation transform = new Transformation();
+    CompressedDictionary dictionary = CompressedDictionary.create(englishLoader(), transform);
+
+    final Set<String> onDisk = new THashSet<String>();
+    englishLoader().load(new Consumer<String>() {
+      @Override
+      public void consume(String s) {
+        assert s != null;
+        String t = transform.transform(s);
+        if (t == null) {
+          return;
+        }
+        onDisk.add(t);
+      }
+    });
+
+    List<String> odList = new ArrayList<String>(onDisk);
+    Collections.sort(odList);
+    List<String> loaded = new ArrayList<String>(dictionary.getWords());
+    Collections.sort(loaded);
+
+    assertEquals(odList, loaded);
+  }
+
+  public void cleanupDictionary() {
+    final Set<String> onDisk = new THashSet<String>(FileUtil.PATH_HASHING_STRATEGY);
+    englishLoader().load(new Consumer<String>() {
+      @Override
+      public void consume(String s) {
+        assert s != null;
+        onDisk.add(s);
+      }
+    });
+
+    List<String> odList = new ArrayList<String>(onDisk);
+    Collections.sort(odList);
+
+    File file = new File("C:\\Work\\Idea\\community\\spellchecker\\src\\com\\intellij\\spellchecker\\english.2");
+    try {
+      FileUtil.writeToFile(file, StringUtil.join(odList, "\n"));
+    }
+    catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static StreamLoader englishLoader() {
+    return new StreamLoader(DefaultBundledDictionariesProvider.class.getResourceAsStream(ENGLISH_DIC), ENGLISH_DIC);
+  }
+
   public void loadDictionaryTest(@NotNull final String name, int wordCount) throws IOException {
     final Transformation transform = new Transformation();
     PlatformTestUtil.startPerformanceTest("load dictionary", times.get(name), new ThrowableRunnable() {
@@ -130,7 +183,7 @@ public class DictionaryTest extends TestCase {
   }
 
 
-  public static void loadHalfDictionaryTest(final String name, final int maxCount) throws IOException {
+  public static void loadHalfDictionaryTest(final String name, final int maxCount) {
     final Pair<Set<String>, Set<String>> sets = createWordSets(name, maxCount, 2);
     final Loader loader = createLoader(sets.getFirst());
     CompressedDictionary dictionary = CompressedDictionary.create(loader, new Transformation());

spellchecker/testSrc/com/intellij/spellchecker/compress/EncodeAndCompressTest.java

@@ -9,10 +9,10 @@ public class EncodeAndCompressTest extends TestCase {
         Encoder encoder = new Encoder();
         String word = "example";
         UnitBitSet bs = encoder.encode(word, true);
-        byte[] compressed = UnitBitSet.getBytes(bs);
-        final UnitBitSet decompressed = UnitBitSet.create(compressed);
-        assertEquals(bs,decompressed);
-        String restored = encoder.decode(decompressed);
+        byte[] compressed = bs.pack();
+        final String decompressed = UnitBitSet.decode(compressed, encoder.getAlphabet());
+        assertEquals(word,decompressed);
+        String restored = encoder.decode(compressed);
         assertEquals(word,restored);
     }
 

spellchecker/testSrc/com/intellij/spellchecker/compress/EncoderTest.java

@@ -18,107 +18,88 @@ package com.intellij.spellchecker.compress;
 import junit.framework.TestCase;
 
 public class EncoderTest extends TestCase {
+  public void testSimple() {
+    Encoder encoder = new Encoder();
+    final String wordToTest = "abc";
+    final UnitBitSet bitSet = encoder.encode(wordToTest, true);
+    assertNotNull(bitSet);
+    assertIndices(bitSet, 1, 2, 3);
+    byte[] compressed = bitSet.pack();
+    assertEquals(1, compressed.length);
+    assertEquals(wordToTest, encoder.decode(compressed));
+  }
 
-
-    public void testSimple() {
-        Encoder encoder = new Encoder();
-        final String wordToTest = "abc";
-        final UnitBitSet bitSet = encoder.encode(wordToTest, true);
-        assertNotNull(bitSet);
-        assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
-        assertEquals(3, bitSet.getUnitValue(0));
-        assertEquals(1, bitSet.getUnitValue(1));
-        assertEquals(1, bitSet.getUnitValue(2));
-        assertEquals(2, bitSet.getUnitValue(3));
-        assertEquals(3, bitSet.getUnitValue(4));
-
-        assertEquals(wordToTest, encoder.decode(bitSet));
+  private static void assertIndices(UnitBitSet bitSet, int... indices) {
+    assertEquals(indices.length, bitSet.b.length);
+    for (int i = 0; i < indices.length; i++) {
+      int index = indices[i];
+      assertEquals(index, bitSet.getUnitValue(i));
     }
+  }
 
 
-    public void testDouble() {
-        Encoder encoder = new Encoder();
-        final String wordToTest = "aaa";
-        final UnitBitSet bitSet = encoder.encode(wordToTest, true);
-        assertNotNull(bitSet);
-        assertEquals(1, encoder.getAlphabet().getLastIndexUsed());
-        assertEquals(3, bitSet.getUnitValue(0));
-        assertEquals(1, bitSet.getUnitValue(1));
-        assertEquals(1, bitSet.getUnitValue(2));
-        assertEquals(1, bitSet.getUnitValue(3));
-        assertEquals(1, bitSet.getUnitValue(4));
+  public void testDouble() {
+    Encoder encoder = new Encoder();
+    final String wordToTest = "aaa";
+    final UnitBitSet bitSet = encoder.encode(wordToTest, true);
+    assertNotNull(bitSet);
+    assertEquals(1, encoder.getAlphabet().getLastIndexUsed());
+    assertIndices(bitSet, 1, 1, 1);
 
-        assertEquals(wordToTest, encoder.decode(bitSet));
-    }
+    assertEquals(wordToTest, encoder.decode(bitSet.pack()));
+  }
 
-    public void testLetterRepetition() {
-        Encoder encoder = new Encoder();
-        final String wordToTest = "aba";
-        final UnitBitSet bitSet = encoder.encode(wordToTest, true);
-        assertNotNull(bitSet);
-        assertEquals(2, encoder.getAlphabet().getLastIndexUsed());
-        assertEquals(3, bitSet.getUnitValue(0));
-        assertEquals(1, bitSet.getUnitValue(1));
-        assertEquals(1, bitSet.getUnitValue(2));
-        assertEquals(2, bitSet.getUnitValue(3));
-        assertEquals(1, bitSet.getUnitValue(4));
+  public void testLetterRepetition() {
+    Encoder encoder = new Encoder();
+    final String wordToTest = "aba";
+    final UnitBitSet bitSet = encoder.encode(wordToTest, true);
+    assertNotNull(bitSet);
+    assertEquals(2, encoder.getAlphabet().getLastIndexUsed());
+    assertIndices(bitSet, 1, 2, 1);
 
-        assertEquals(wordToTest, encoder.decode(bitSet));
-    }
+    assertEquals(wordToTest, encoder.decode(bitSet.pack()));
+  }
 
-    public void testReverse() {
-        Encoder encoder = new Encoder();
-        final String wordToTest1 = "abc";
-        final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
-        assertNotNull(bitSet);
-        assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
-        assertEquals(3, bitSet.getUnitValue(0));
-        assertEquals(1, bitSet.getUnitValue(1));
-        assertEquals(1, bitSet.getUnitValue(2));
-        assertEquals(2, bitSet.getUnitValue(3));
-        assertEquals(3, bitSet.getUnitValue(4));
+  public void testReverse() {
+    Encoder encoder = new Encoder();
+    final String wordToTest1 = "abc";
+    final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
+    assertNotNull(bitSet);
+    assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
+    assertIndices(bitSet, 1, 2, 3);
 
-        assertEquals(wordToTest1, encoder.decode(bitSet));
+    byte[] pack = bitSet.pack();
+    assertEquals(1, pack.length);
+    assertEquals(wordToTest1, encoder.decode(pack));
 
-        final String wordToTest2 = "cba";
-        final UnitBitSet bitSet2 = encoder.encode(wordToTest2, true);
-        assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
-        assertNotNull(bitSet);
-        assertEquals(3, bitSet2.getUnitValue(0));
-        assertEquals(3, bitSet2.getUnitValue(1));
-        assertEquals(3, bitSet2.getUnitValue(2));
-        assertEquals(2, bitSet2.getUnitValue(3));
-        assertEquals(1, bitSet2.getUnitValue(4));
+    final String wordToTest2 = "cba";
+    final UnitBitSet bitSet2 = encoder.encode(wordToTest2, true);
+    assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
+    assertNotNull(bitSet2);
+    assertIndices(bitSet2, 3, 2, 1);
 
-        assertEquals(wordToTest2, encoder.decode(bitSet2));
-    }
+    byte[] pack2 = bitSet2.pack();
+    assertEquals(1, pack2.length);
+    assertEquals(wordToTest2, encoder.decode(pack2));
+  }
 
 
-    public void testWithPredefinedAlphabet() {
-        Encoder encoder = new Encoder(new Alphabet("abcdefghijklmnopqrst"));
-        final String wordToTest1 = "asia";
-        //letter 'a' will be added at the end
-        final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
-        assertNotNull(bitSet);
-        assertEquals(20, encoder.getAlphabet().getLastIndexUsed());
-        assertEquals(4, bitSet.getUnitValue(0));
-        assertEquals(1, bitSet.getUnitValue(1));
-        assertEquals(1, bitSet.getUnitValue(2));
-        assertEquals(19, bitSet.getUnitValue(3));
-        assertEquals(9, bitSet.getUnitValue(4));
-        assertEquals(1, bitSet.getUnitValue(5));
-
-        assertEquals(wordToTest1, encoder.decode(bitSet));
-
-
-    }
-
-    public void testUnknown() {
-        Encoder encoder = new Encoder(new Alphabet("abc"));
-        final String wordToTest1 = "def";
-        final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
-        assertEquals(bitSet, Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS);
-    }
+  public void testWithPredefinedAlphabet() {
+    Encoder encoder = new Encoder(new Alphabet("abcdefghijklmnopqrst"));
+    final String wordToTest1 = "asia";
+    //letter 'a' will be added at the end
+    final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
+    assertNotNull(bitSet);
+    assertEquals(20, encoder.getAlphabet().getLastIndexUsed());
+    assertIndices(bitSet, 1, 19, 9,1);
 
+    assertEquals(wordToTest1, encoder.decode(bitSet.pack()));
+  }
 
+  public void testUnknown() {
+    Encoder encoder = new Encoder(new Alphabet("abc"));
+    final String wordToTest1 = "def";
+    final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
+    assertEquals(bitSet, Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS);
+  }
 }

spellchecker/testSrc/com/intellij/spellchecker/compress/UnitBitSetTest.java

@@ -17,56 +17,18 @@ package com.intellij.spellchecker.compress;
 
 import junit.framework.TestCase;
 
-import java.util.Random;
-
 public class UnitBitSetTest extends TestCase {
 
   public void testUnitValue() {
     int bitsPerUnit = 256;
     for (int i = 0; i < bitsPerUnit - 1; i++) {
-      UnitBitSet bs = new UnitBitSet();
+      UnitBitSet bs = new UnitBitSet(new byte[2], new Alphabet());
       bs.setUnitValue(0, i);
       assertEquals(i, bs.getUnitValue(0));
       assertEquals(0, bs.getUnitValue(1));
     }
   }
 
-  public void testCreateFromBitSet() {
-    UnitBitSet bs1 = new UnitBitSet();
-    bs1.setUnitValue(5, 255);
-    UnitBitSet bs2 = UnitBitSet.create(bs1);
-    assertEquals(bs2, bs1);
-  }
 
-  public void testCompressorWithRandomData() {
-    int amount = 10;
-    for (int i = 0; i < 1000; i++) {
-      int[] values = getRandoms(UnitBitSet.MAX_UNIT_VALUE, amount);
-      final UnitBitSet bitSet = new UnitBitSet();
-      bitSet.setUnitValue(0, values.length);
-      for (int i1 = 1, valuesLength = values.length; i1 < valuesLength; i1++) {
-        int value = values[i1];
-        bitSet.setUnitValue(i1, value);
-      }
-      final byte[] compressed = UnitBitSet.getBytes(bitSet);
-      final UnitBitSet decompressed = UnitBitSet.create(compressed);
-      boolean check = (bitSet.equals(decompressed));
-      if (!check) {
-        System.out.println("bitSet: " + bitSet);
-        System.out.println("decompressed: " + decompressed);
-        UnitBitSet.getBytes(bitSet);
-        UnitBitSet.create(compressed);
-      }
-      assertEquals(bitSet, decompressed);
-    }
-  }
-
-  private static int[] getRandoms(int max, int count) {
-    int[] result = new int[count];
-    for (int i = 0; i < count; i++) {
-      result[i] = new Random().nextInt(max);
-    }
-    return result;
-  }
 
 }