diff --git a/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/language/JavaSupportTest.kt b/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/language/JavaSupportTest.kt index ca682f6d0b8c..81e391c585a9 100644 --- a/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/language/JavaSupportTest.kt +++ b/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/language/JavaSupportTest.kt @@ -59,4 +59,8 @@ class JavaSupportTest : GrazieTestBase() { fun testCommentIsNotHighlightedIfThereIsReference() { runHighlightTestForFile("ide/language/java/VectorablexxClass.java") } + + fun `test spellchecking normalization`() { + runHighlightTestForFile("ide/language/java/Normalization.java") + } } diff --git a/plugins/grazie/src/test/testData/ide/language/java/Normalization.java b/plugins/grazie/src/test/testData/ide/language/java/Normalization.java new file mode 100644 index 000000000000..dca42423a99e --- /dev/null +++ b/plugins/grazie/src/test/testData/ide/language/java/Normalization.java @@ -0,0 +1,11 @@ +class VariableJson { + int broetchen, Koenigsberg, koenigsberg, koenigsberger; + String notificacao; + + // Uebergeben> + // acucar + // Sehenswürdigkeit, açúcar, Überweisen + public static void main(String[] args) { + VariableJson variavel; + } +} \ No newline at end of file diff --git a/spellchecker/src/com/intellij/spellchecker/grazie/GrazieSpellCheckerEngine.kt b/spellchecker/src/com/intellij/spellchecker/grazie/GrazieSpellCheckerEngine.kt index ee53556b8779..c458e0ea1120 100644 --- a/spellchecker/src/com/intellij/spellchecker/grazie/GrazieSpellCheckerEngine.kt +++ b/spellchecker/src/com/intellij/spellchecker/grazie/GrazieSpellCheckerEngine.kt @@ -40,6 +40,7 @@ import com.intellij.spellchecker.engine.Transformation import com.intellij.spellchecker.grazie.async.WordListLoader import com.intellij.spellchecker.grazie.dictionary.ExtendedWordListWithFrequency import com.intellij.spellchecker.grazie.dictionary.WordListAdapter +import com.intellij.spellchecker.grazie.ranker.DiacriticSuggestionRanker import kotlinx.coroutines.* private const val MAX_WORD_LENGTH = 32 @@ -124,11 +125,13 @@ internal class GrazieSpellCheckerEngine( IgnoreRuleDictionary.standard(tooShortLength = 2), DictionaryResources.getReplacingRules("/rule/en", FromResourcesDataLoader) ), - ranker = LinearAggregatingSuggestionRanker( - JaroWinklerSuggestionRanker() to 0.43, - LevenshteinSuggestionRanker() to 0.20, - PhoneticSuggestionRanker(DoubleMetaphone()) to 0.11, - FrequencySuggestionRanker(wordList) to 0.23 + ranker = DiacriticSuggestionRanker( + LinearAggregatingSuggestionRanker( + JaroWinklerSuggestionRanker() to 0.43, + LevenshteinSuggestionRanker() to 0.20, + PhoneticSuggestionRanker(DoubleMetaphone()) to 0.11, + FrequencySuggestionRanker(wordList) to 0.23 + ) ), filter = RadiusSuggestionFilter(0.05), normalizer = StripAccentsNormalizer(), diff --git a/spellchecker/src/com/intellij/spellchecker/grazie/diacritic/Diacritics.kt b/spellchecker/src/com/intellij/spellchecker/grazie/diacritic/Diacritics.kt new file mode 100644 index 000000000000..6345412330fd --- /dev/null +++ b/spellchecker/src/com/intellij/spellchecker/grazie/diacritic/Diacritics.kt @@ -0,0 +1,36 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.spellchecker.grazie.diacritic + +import ai.grazie.nlp.utils.normalization.StripAccentsNormalizer +import com.intellij.util.io.IOUtil + +object Diacritics { + @JvmStatic + fun equalsIgnoringDiacritics(word: String, diacritic: String): Boolean { + if (!IOUtil.isAscii(diacritic)) { + if (word.equals(StripAccentsNormalizer().normalize(diacritic), ignoreCase = true)) { + return true + } + if (word.equals(replaceUmlauts(diacritic), ignoreCase = true)) { + return true + } + } + return false + } + + private fun replaceUmlauts(suggestion: String): String { + if (suggestion.any { it in "üöäßÜÖÄẞ" }) { + //ü→ue, ö→oe, ä→ae, ß→ss + return suggestion + .replace("ü", "ue") + .replace("ö", "oe") + .replace("ä", "ae") + .replace("ß", "ss") + .replace("Ü", "Ue") + .replace("Ö", "Oe") + .replace("Ä", "Ae") + .replace("ẞ", "Ss") + } + return suggestion + } +} \ No newline at end of file diff --git a/spellchecker/src/com/intellij/spellchecker/grazie/ranker/DiacriticSuggestionRanker.kt b/spellchecker/src/com/intellij/spellchecker/grazie/ranker/DiacriticSuggestionRanker.kt new file mode 100644 index 000000000000..65e2600e5e4e --- /dev/null +++ b/spellchecker/src/com/intellij/spellchecker/grazie/ranker/DiacriticSuggestionRanker.kt @@ -0,0 +1,22 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.spellchecker.grazie.ranker + +import ai.grazie.spell.suggestion.ranker.SuggestionRanker +import ai.grazie.utils.LinkedSet +import com.intellij.spellchecker.grazie.diacritic.Diacritics + +class DiacriticSuggestionRanker( + private val fallbackSuggestionRanker: SuggestionRanker, +) : SuggestionRanker { + override fun score(word: String, suggestions: LinkedSet): Map { + val weights = suggestions.associateWith { score(word, it) } + if (weights.filter { it.value > 0 }.isEmpty()) { + return fallbackSuggestionRanker.score(word, suggestions) + } + return weights + } + + private fun score(word: String, suggestion: String): Double { + return if (Diacritics.equalsIgnoringDiacritics(word, suggestion)) 1.0 else 0.0 + } +} \ No newline at end of file diff --git a/spellchecker/src/com/intellij/spellchecker/inspections/SpellCheckingInspection.java b/spellchecker/src/com/intellij/spellchecker/inspections/SpellCheckingInspection.java index b0d23a4ea071..95d2c8a91800 100644 --- a/spellchecker/src/com/intellij/spellchecker/inspections/SpellCheckingInspection.java +++ b/spellchecker/src/com/intellij/spellchecker/inspections/SpellCheckingInspection.java @@ -11,6 +11,7 @@ import com.intellij.lang.refactoring.NamesValidator; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.project.DumbAware; import com.intellij.openapi.project.DumbService; +import com.intellij.openapi.project.Project; import com.intellij.openapi.util.TextRange; import com.intellij.openapi.util.registry.Registry; import com.intellij.profile.codeInspection.InspectionProfileManager; @@ -19,7 +20,9 @@ import com.intellij.psi.PsiElementVisitor; import com.intellij.psi.PsiFile; import com.intellij.psi.PsiReference; import com.intellij.psi.util.CachedValuesManager; +import com.intellij.refactoring.rename.RenameUtil; import com.intellij.spellchecker.SpellCheckerManager; +import com.intellij.spellchecker.grazie.diacritic.Diacritics; import com.intellij.spellchecker.tokenizer.*; import com.intellij.spellchecker.util.SpellCheckerBundle; import com.intellij.util.Consumer; @@ -245,7 +248,7 @@ public final class SpellCheckingInspection extends LocalInspectionTool implement } boolean keyword = myNamesValidator.isKeyword(word, myElement.getProject()); - if (keyword || !myManager.hasProblem(word) || hasSameNamedReferenceInFile(word)) { + if (keyword || !hasProblem(word) || hasSameNamedReferenceInFile(word)) { return; } @@ -294,6 +297,23 @@ public final class SpellCheckingInspection extends LocalInspectionTool implement } return false; } + + private boolean hasProblem(String word) { + if (!myManager.hasProblem(word)) { + return false; + } + Language language = myElement.getLanguage(); + SpellcheckingStrategy strategy = getSpellcheckingStrategy(myElement, language); + if (strategy == null || !strategy.elementFitsScope(myElement, Set.of(SpellCheckingScope.Code))) { + return true; + } + + Project project = myElement.getProject(); + return SpellCheckerManager.getInstance(project).getSuggestions(word) + .stream() + .filter(suggestion -> RenameUtil.isValidName(project, myElement, suggestion)) + .noneMatch(suggestion -> Diacritics.equalsIgnoringDiacritics(word, suggestion)); + } } public enum SpellCheckingScope {