MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Spellchecker - WI-9451 Don't check spelling of words in unknown languages

Browse Source
This commit is contained in:
Alexey Gopachenko
2012-09-06 00:40:57 +02:00
parent 4e3da0e3a9
commit 26db8d004e
5 changed files with 52 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/spellchecker/src/com/intellij/spellchecker/compress/CompressedDictionary.java
+13 -20
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2010 JetBrains s.r.o.
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -121,23 +121,20 @@ public final class CompressedDictionary implements Dictionary {
if (word == null) {
return false;
}
try {
UnitBitSet bs = encoder.encode(word, false);
if (bs == Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS) return false; // TODO: polyglot.dic contains word typppo , wtf ?! also make tests for your changes!!!
if (bs == null) return false; // fail faster w/o search
byte[] compressed = UnitBitSet.getBytes(bs);
int index = -1;
for (int i = 0; i < lengths.length; i++) {
if (lengths[i] == compressed.length) {
index = i;
break;
}
UnitBitSet bs = encoder.encode(word, false);
if (bs == Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS)
throw new EncodingException("WORD_OF_ENTIRELY_UNKNOWN_LETTERS");
if (bs == null) return false;
//TODO throw new EncodingException("WORD_WITH_SOME_UNKNOWN_LETTERS");
byte[] compressed = UnitBitSet.getBytes(bs);
int index = -1;
for (int i = 0; i < lengths.length; i++) {
if (lengths[i] == compressed.length) {
index = i;
break;
}
return index != -1 && contains(compressed, words[index]);
}
catch (EncodingException ignored) {
return false;
}
return index != -1 && contains(compressed, words[index]);
}
@@ -153,10 +150,6 @@ public final class CompressedDictionary implements Dictionary {
throw new UnsupportedOperationException();
}
public int getAlphabetLength() {
return alphabet.getLastIndexUsed();
}
public int size() {
return wordsCount;
}
plugins/spellchecker/src/com/intellij/spellchecker/dictionary/ProjectDictionary.java
+14 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
*/
package com.intellij.spellchecker.dictionary;
import com.intellij.spellchecker.compress.EncodingException;
import com.intellij.util.Consumer;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
@@ -59,11 +60,21 @@ public class ProjectDictionary implements EditableDictionary {
if (word == null || dictionaries == null) {
return false;
}
int negatives = 0;
for (Dictionary dictionary : dictionaries) {
if (dictionary.contains(word)) {
return true;
try {
if (dictionary.contains(word)) {
return true;
}
else {
negatives++;
}
}
catch (EncodingException e) {
//System.out.println("e.getMessage() = " + e.getMessage() + " " + word);
}
}
if (negatives==dictionaries.size()) throw new EncodingException("WORD_OF_ENTIRELY_UNKNOWN_LETTERS_FOR_ALL");
return false;
}
plugins/spellchecker/src/com/intellij/spellchecker/engine/BaseSpellChecker.java
+9 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2009 JetBrains s.r.o.
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@ import com.intellij.openapi.startup.StartupManager;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.spellchecker.compress.CompressedDictionary;
import com.intellij.spellchecker.compress.EncodingException;
import com.intellij.spellchecker.dictionary.Dictionary;
import com.intellij.spellchecker.dictionary.EditableDictionary;
import com.intellij.spellchecker.dictionary.EditableDictionaryLoader;
@@ -269,7 +270,13 @@ public class BaseSpellChecker implements SpellCheckerEngine {
return true;
}
return dictionary.contains(transformed);
try {
return dictionary.contains(transformed);
}
catch (EncodingException e) {
//System.out.println("e.getMessage() = " + e.getMessage() + " " + transformed);
return true;
}
}
public boolean isCorrect(@NotNull String word) {
plugins/spellchecker/testData/inspection/commentsWithMistakes/test.txt
+2 -1
View File
@@ -1 +1,2 @@
simple <TYPO descr="Typo: In word 'ttest'">ttest</TYPO> file (just plain text)
simple <TYPO descr="Typo: In word 'ttest'">ttest</TYPO> file (just plain text)
русский (например) без словаря не проверять!!!
plugins/spellchecker/testSrc/com/intellij/spellchecker/compress/EncoderTest.java
+14 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2000-2010 JetBrains s.r.o.
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@ public class EncoderTest extends TestCase {
Encoder encoder = new Encoder();
final String wordToTest = "abc";
final UnitBitSet bitSet = encoder.encode(wordToTest, true);
assertNotNull(bitSet);
assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
assertEquals(3, bitSet.getUnitValue(0));
assertEquals(1, bitSet.getUnitValue(1));
@@ -39,6 +40,7 @@ public class EncoderTest extends TestCase {
Encoder encoder = new Encoder();
final String wordToTest = "aaa";
final UnitBitSet bitSet = encoder.encode(wordToTest, true);
assertNotNull(bitSet);
assertEquals(1, encoder.getAlphabet().getLastIndexUsed());
assertEquals(3, bitSet.getUnitValue(0));
assertEquals(1, bitSet.getUnitValue(1));
@@ -53,6 +55,7 @@ public class EncoderTest extends TestCase {
Encoder encoder = new Encoder();
final String wordToTest = "aba";
final UnitBitSet bitSet = encoder.encode(wordToTest, true);
assertNotNull(bitSet);
assertEquals(2, encoder.getAlphabet().getLastIndexUsed());
assertEquals(3, bitSet.getUnitValue(0));
assertEquals(1, bitSet.getUnitValue(1));
@@ -67,6 +70,7 @@ public class EncoderTest extends TestCase {
Encoder encoder = new Encoder();
final String wordToTest1 = "abc";
final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
assertNotNull(bitSet);
assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
assertEquals(3, bitSet.getUnitValue(0));
assertEquals(1, bitSet.getUnitValue(1));
@@ -79,6 +83,7 @@ public class EncoderTest extends TestCase {
final String wordToTest2 = "cba";
final UnitBitSet bitSet2 = encoder.encode(wordToTest2, true);
assertEquals(3, encoder.getAlphabet().getLastIndexUsed());
assertNotNull(bitSet);
assertEquals(3, bitSet2.getUnitValue(0));
assertEquals(3, bitSet2.getUnitValue(1));
assertEquals(3, bitSet2.getUnitValue(2));
@@ -94,6 +99,7 @@ public class EncoderTest extends TestCase {
final String wordToTest1 = "asia";
//letter 'a' will be added at the end
final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
assertNotNull(bitSet);
assertEquals(20, encoder.getAlphabet().getLastIndexUsed());
assertEquals(4, bitSet.getUnitValue(0));
assertEquals(1, bitSet.getUnitValue(1));
@@ -107,5 +113,12 @@ public class EncoderTest extends TestCase {
}
public void testUnknown() {
Encoder encoder = new Encoder(new Alphabet("abc"));
final String wordToTest1 = "def";
final UnitBitSet bitSet = encoder.encode(wordToTest1, true);
assertEquals(bitSet, Encoder.WORD_OF_ENTIRELY_UNKNOWN_LETTERS);
}
}