IJPL-33189 Spellcheck should ignore missing accents in object names, functions, etc.

IJPL-33193 Spellcheck should accommodate for umlauts


Co-authored-by: Peter Gromov <peter.gromov@jetbrains.com>

Merge-request: IJ-MR-165950
Merged-by: Ilia Permiashkin <ilia.permiashkin@jetbrains.com>

GitOrigin-RevId: fb9c342ff7be94a6aec07be5ce0da608e4128b57

plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/language/JavaSupportTest.kt

@@ -59,4 +59,8 @@ class JavaSupportTest : GrazieTestBase() {
   fun testCommentIsNotHighlightedIfThereIsReference() {
     runHighlightTestForFile("ide/language/java/VectorablexxClass.java")
   }
+
+  fun `test spellchecking normalization`() {
+    runHighlightTestForFile("ide/language/java/Normalization.java")
+  }
 }

plugins/grazie/src/test/testData/ide/language/java/Normalization.java

@@ -0,0 +1,11 @@
+class VariableJson {
+  int broetchen, Koenigsberg, koenigsberg, koenigsberger;
+  String notificacao;
+
+  // <TYPO descr="Typo: In word 'Uebergeben'">Uebergeben</TYPO>>
+  // <TYPO descr="Typo: In word 'acucar'">acucar</TYPO>
+  // Sehenswürdigkeit, açúcar, Überweisen
+  public static void main(String[] args) {
+    VariableJson variavel;
+  }
+}

spellchecker/src/com/intellij/spellchecker/grazie/GrazieSpellCheckerEngine.kt

@@ -40,6 +40,7 @@ import com.intellij.spellchecker.engine.Transformation
 import com.intellij.spellchecker.grazie.async.WordListLoader
 import com.intellij.spellchecker.grazie.dictionary.ExtendedWordListWithFrequency
 import com.intellij.spellchecker.grazie.dictionary.WordListAdapter
+import com.intellij.spellchecker.grazie.ranker.DiacriticSuggestionRanker
 import kotlinx.coroutines.*
 
 private const val MAX_WORD_LENGTH = 32
@@ -124,11 +125,13 @@ internal class GrazieSpellCheckerEngine(
         IgnoreRuleDictionary.standard(tooShortLength = 2),
         DictionaryResources.getReplacingRules("/rule/en", FromResourcesDataLoader)
       ),
-      ranker = LinearAggregatingSuggestionRanker(
-        JaroWinklerSuggestionRanker() to 0.43,
-        LevenshteinSuggestionRanker() to 0.20,
-        PhoneticSuggestionRanker(DoubleMetaphone()) to 0.11,
-        FrequencySuggestionRanker(wordList) to 0.23
+      ranker = DiacriticSuggestionRanker(
+        LinearAggregatingSuggestionRanker(
+          JaroWinklerSuggestionRanker() to 0.43,
+          LevenshteinSuggestionRanker() to 0.20,
+          PhoneticSuggestionRanker(DoubleMetaphone()) to 0.11,
+          FrequencySuggestionRanker(wordList) to 0.23
+        )
       ),
       filter = RadiusSuggestionFilter(0.05),
       normalizer = StripAccentsNormalizer(),

spellchecker/src/com/intellij/spellchecker/grazie/diacritic/Diacritics.kt

@@ -0,0 +1,36 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.spellchecker.grazie.diacritic
+
+import ai.grazie.nlp.utils.normalization.StripAccentsNormalizer
+import com.intellij.util.io.IOUtil
+
+object Diacritics {
+  @JvmStatic
+  fun equalsIgnoringDiacritics(word: String, diacritic: String): Boolean {
+    if (!IOUtil.isAscii(diacritic)) {
+      if (word.equals(StripAccentsNormalizer().normalize(diacritic), ignoreCase = true)) {
+        return true
+      }
+      if (word.equals(replaceUmlauts(diacritic), ignoreCase = true)) {
+        return true
+      }
+    }
+    return false
+  }
+
+  private fun replaceUmlauts(suggestion: String): String {
+    if (suggestion.any { it in "üöäßÜÖÄẞ" }) {
+      //ü→ue, ö→oe, ä→ae, ß→ss
+      return suggestion
+        .replace("ü", "ue")
+        .replace("ö", "oe")
+        .replace("ä", "ae")
+        .replace("ß", "ss")
+        .replace("Ü", "Ue")
+        .replace("Ö", "Oe")
+        .replace("Ä", "Ae")
+        .replace("ẞ", "Ss")
+    }
+    return suggestion
+  }
+}

spellchecker/src/com/intellij/spellchecker/grazie/ranker/DiacriticSuggestionRanker.kt

@@ -0,0 +1,22 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package com.intellij.spellchecker.grazie.ranker
+
+import ai.grazie.spell.suggestion.ranker.SuggestionRanker
+import ai.grazie.utils.LinkedSet
+import com.intellij.spellchecker.grazie.diacritic.Diacritics
+
+class DiacriticSuggestionRanker(
+  private val fallbackSuggestionRanker: SuggestionRanker,
+) : SuggestionRanker {
+  override fun score(word: String, suggestions: LinkedSet<String>): Map<String, Double> {
+    val weights = suggestions.associateWith { score(word, it) }
+    if (weights.filter { it.value > 0 }.isEmpty()) {
+      return fallbackSuggestionRanker.score(word, suggestions)
+    }
+    return weights
+  }
+  
+  private fun score(word: String, suggestion: String): Double {
+    return if (Diacritics.equalsIgnoringDiacritics(word, suggestion)) 1.0 else 0.0
+  }
+}

spellchecker/src/com/intellij/spellchecker/inspections/SpellCheckingInspection.java

@@ -11,6 +11,7 @@ import com.intellij.lang.refactoring.NamesValidator;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.DumbAware;
 import com.intellij.openapi.project.DumbService;
+import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.TextRange;
 import com.intellij.openapi.util.registry.Registry;
 import com.intellij.profile.codeInspection.InspectionProfileManager;
@@ -19,7 +20,9 @@ import com.intellij.psi.PsiElementVisitor;
 import com.intellij.psi.PsiFile;
 import com.intellij.psi.PsiReference;
 import com.intellij.psi.util.CachedValuesManager;
+import com.intellij.refactoring.rename.RenameUtil;
 import com.intellij.spellchecker.SpellCheckerManager;
+import com.intellij.spellchecker.grazie.diacritic.Diacritics;
 import com.intellij.spellchecker.tokenizer.*;
 import com.intellij.spellchecker.util.SpellCheckerBundle;
 import com.intellij.util.Consumer;
@@ -245,7 +248,7 @@ public final class SpellCheckingInspection extends LocalInspectionTool implement
       }
 
       boolean keyword = myNamesValidator.isKeyword(word, myElement.getProject());
-      if (keyword || !myManager.hasProblem(word) || hasSameNamedReferenceInFile(word)) {
+      if (keyword || !hasProblem(word) || hasSameNamedReferenceInFile(word)) {
         return;
       }
 
@@ -294,6 +297,23 @@ public final class SpellCheckingInspection extends LocalInspectionTool implement
       }
       return false;
     }
+
+    private boolean hasProblem(String word) {
+      if (!myManager.hasProblem(word)) {
+        return false;
+      }
+      Language language = myElement.getLanguage();
+      SpellcheckingStrategy strategy = getSpellcheckingStrategy(myElement, language);
+      if (strategy == null || !strategy.elementFitsScope(myElement, Set.of(SpellCheckingScope.Code))) {
+        return true;
+      }
+
+      Project project = myElement.getProject();
+      return SpellCheckerManager.getInstance(project).getSuggestions(word)
+        .stream()
+        .filter(suggestion -> RenameUtil.isValidName(project, myElement, suggestion))
+        .noneMatch(suggestion -> Diacritics.equalsIgnoringDiacritics(word, suggestion));
+    }
   }
 
   public enum SpellCheckingScope {