Update grazie dictionaries to .288

Merge-request: IJ-MR-170697
Merged-by: Ilia Permiashkin <ilia.permiashkin@jetbrains.com>

GitOrigin-RevId: 63e49757c320bec642f9aa822e4dbdbb60294dd7
This commit is contained in:
Ilia Permiashkin
2025-07-29 17:07:58 +00:00
committed by intellij-monorepo-bot
parent aa97410a8b
commit 91a68fd1b3
9 changed files with 55 additions and 125 deletions
+4 -8
View File
@@ -1744,22 +1744,19 @@ jvm_import(
jvm_import(
name = "grazie-tests-ai-grazie-spell-hunspell-de",
jar = "@hunspell-de-jvm-0_2_281_http//file",
source_jar = "@hunspell-de-jvm-0_2_281-sources_http//file",
jar = "@hunspell-de-0_2_288_http//file",
visibility = ["//visibility:public"]
)
jvm_import(
name = "grazie-tests-ai-grazie-spell-hunspell-ru",
jar = "@hunspell-ru-jvm-0_2_281_http//file",
source_jar = "@hunspell-ru-jvm-0_2_281-sources_http//file",
jar = "@hunspell-ru-0_2_288_http//file",
visibility = ["//visibility:public"]
)
jvm_import(
name = "grazie-tests-ai-grazie-spell-hunspell-uk",
jar = "@hunspell-uk-jvm-0_2_281_http//file",
source_jar = "@hunspell-uk-jvm-0_2_281-sources_http//file",
jar = "@hunspell-uk-0_2_288_http//file",
visibility = ["//visibility:public"]
)
@@ -4500,8 +4497,7 @@ jvm_import(
jvm_import(
name = "libraries-ai-grazie-spell-gec-engine-local-ai-grazie-spell-hunspell-en",
jar = "@hunspell-en-jvm-0_2_281_http//file",
source_jar = "@hunspell-en-jvm-0_2_281-sources_http//file",
jar = "@hunspell-en-0_2_288_http//file",
visibility = ["//visibility:public"]
)
+16 -44
View File
@@ -2205,45 +2205,24 @@ http_file(
)
http_file(
name = "hunspell-de-jvm-0_2_281_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-de-jvm/0.2.281/hunspell-de-jvm-0.2.281.jar",
sha256 = "17e8741b0bba21fd034d40a3b4190b819913f4ae9234b62e7c960bcd6cbd1c87",
downloaded_file_path = "hunspell-de-jvm-0.2.281.jar"
name = "hunspell-de-0_2_288_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-de/0.2.288/hunspell-de-0.2.288.jar",
sha256 = "8457b66279cf47c6b8f71b2b910662968aa74a7f4e4076121823b86531ff1131",
downloaded_file_path = "hunspell-de-0.2.288.jar"
)
http_file(
name = "hunspell-de-jvm-0_2_281-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-de-jvm/0.2.281/hunspell-de-jvm-0.2.281-sources.jar",
sha256 = "8b8919a2003c603c6c892930f649e00b83b49e9b754a38bb8ea70cba3a5bfb99",
downloaded_file_path = "hunspell-de-jvm-0.2.281-sources.jar"
name = "hunspell-ru-0_2_288_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-ru/0.2.288/hunspell-ru-0.2.288.jar",
sha256 = "f14ae1a3e86c5df0668127a6215bf969d419d91de4be48e8e65681314df28c96",
downloaded_file_path = "hunspell-ru-0.2.288.jar"
)
http_file(
name = "hunspell-ru-jvm-0_2_281_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-ru-jvm/0.2.281/hunspell-ru-jvm-0.2.281.jar",
sha256 = "4c6be8d96b44761d3546ba127fcfb48bd31773b40a0ff23bdb4582c7afb33123",
downloaded_file_path = "hunspell-ru-jvm-0.2.281.jar"
)
http_file(
name = "hunspell-ru-jvm-0_2_281-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-ru-jvm/0.2.281/hunspell-ru-jvm-0.2.281-sources.jar",
sha256 = "5b7a9db3233093cee59c7e93c6f086e066b319f69601a3147008599d8c4f53c3",
downloaded_file_path = "hunspell-ru-jvm-0.2.281-sources.jar"
)
http_file(
name = "hunspell-uk-jvm-0_2_281_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-uk-jvm/0.2.281/hunspell-uk-jvm-0.2.281.jar",
sha256 = "c51215918811365ae560ad4fc82e90b6b1afa3bd956162b92786857fc75e4dfe",
downloaded_file_path = "hunspell-uk-jvm-0.2.281.jar"
)
http_file(
name = "hunspell-uk-jvm-0_2_281-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-uk-jvm/0.2.281/hunspell-uk-jvm-0.2.281-sources.jar",
sha256 = "65aac85886a88e6dedb4e2e477191424d987dcf429c20770d84cd41339a8395d",
downloaded_file_path = "hunspell-uk-jvm-0.2.281-sources.jar"
name = "hunspell-uk-0_2_288_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-uk/0.2.288/hunspell-uk-0.2.288.jar",
sha256 = "610372ac711d12381dd41aff588f12d3b7a37addf3c6e0bcb74f5667e69c0c24",
downloaded_file_path = "hunspell-uk-0.2.288.jar"
)
http_file(
@@ -6209,17 +6188,10 @@ http_file(
)
http_file(
name = "hunspell-en-jvm-0_2_281_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-en-jvm/0.2.281/hunspell-en-jvm-0.2.281.jar",
sha256 = "09766747bfa2f150e8ec2149cad08265bf92694a6cced61a1d8013a995152bf1",
downloaded_file_path = "hunspell-en-jvm-0.2.281.jar"
)
http_file(
name = "hunspell-en-jvm-0_2_281-sources_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-en-jvm/0.2.281/hunspell-en-jvm-0.2.281-sources.jar",
sha256 = "3e80e17ac3d8ce626a4bbacaf349954036a129cc8ba09555e21acd287491e15b",
downloaded_file_path = "hunspell-en-jvm-0.2.281-sources.jar"
name = "hunspell-en-0_2_288_http",
url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-en/0.2.288/hunspell-en-0.2.288.jar",
sha256 = "a231a9b61136bc54c5c6f9074835d08101c4701486b830c4dcdc88ef663c6e5a",
downloaded_file_path = "hunspell-en-0.2.288.jar"
)
http_file(
@@ -88,20 +88,18 @@
</orderEntry>
<orderEntry type="module-library" exported="">
<library name="ai.grazie.spell.hunspell.en" type="repository">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-en-jvm:0.2.281">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-en:0.2.288">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-en-jvm/0.2.281/hunspell-en-jvm-0.2.281.jar">
<sha256sum>09766747bfa2f150e8ec2149cad08265bf92694a6cced61a1d8013a995152bf1</sha256sum>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-en/0.2.288/hunspell-en-0.2.288.jar">
<sha256sum>a231a9b61136bc54c5c6f9074835d08101c4701486b830c4dcdc88ef663c6e5a</sha256sum>
</artifact>
</verification>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-en-jvm/0.2.281/hunspell-en-jvm-0.2.281.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-en/0.2.288/hunspell-en-0.2.288.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-en-jvm/0.2.281/hunspell-en-jvm-0.2.281-sources.jar!/" />
</SOURCES>
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
@@ -24,7 +24,7 @@ object GraziePlugin {
}
object Hunspell : GrazieStateLifecycle {
const val version = "0.2.281"
const val version = "0.2.288"
const val url = "https://packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell"
override fun update(prevState: GrazieConfig.State, newState: GrazieConfig.State) {
@@ -3,6 +3,8 @@ package com.intellij.grazie.jlanguage
import ai.grazie.nlp.langs.LanguageISO
import ai.grazie.nlp.langs.LanguageWithVariant
import ai.grazie.spell.utils.DictionaryResources.transformSwissGermanDic
import ai.grazie.spell.utils.DictionaryResources.transformSwissGermanTrigrams
import com.intellij.grazie.GrazieDynamic
import com.intellij.grazie.GrazieDynamic.getLangDynamicFolder
import com.intellij.grazie.remote.GrazieRemote
@@ -10,14 +12,12 @@ import com.intellij.grazie.remote.HunspellDescriptor
import com.intellij.grazie.remote.LanguageToolDescriptor
import com.intellij.grazie.remote.RemoteLangDescriptor
import com.intellij.openapi.util.NlsSafe
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.spellchecker.grazie.GrazieSpellCheckerEngine
import com.intellij.spellchecker.hunspell.HunspellDictionary
import org.jetbrains.annotations.NonNls
import org.languagetool.Language
import org.languagetool.language.English
import org.languagetool.noop.NoopChunker
import java.io.File
enum class Lang(val displayName: String, val className: String, val iso: LanguageISO, @NlsSafe val nativeName: String, @NonNls val variant: String? = null) {
BRITISH_ENGLISH("English (GB)", "BritishEnglish", LanguageISO.EN, "English (Great Britain)"),
@@ -121,39 +121,12 @@ enum class Lang(val displayName: String, val className: String, val iso: Languag
}
private fun createSwissDictionary(path: String): HunspellDictionary {
val pathWithoutExtension = FileUtilRt.getNameWithoutExtension(path)
val dic = File("$pathWithoutExtension.dic").readText()
val aff = File("$pathWithoutExtension.aff").readText()
val trigrams = getTrigrams("$pathWithoutExtension.trigrams.txt")
val (dic, aff, trigrams) = HunspellDictionary.getHunspellPaths(path)
return HunspellDictionary(
transformSwissGermanDic(dic),
aff,
transformSwissGermanTrigrams(trigrams),
transformSwissGermanDic(dic.readText()),
aff.readText(),
transformSwissGermanTrigrams(trigrams.readLines()),
path,
LanguageISO.DE,
)
}
//todo use grazie-platform's DictionaryServices functions in newer versions
private fun transformSwissGermanDic(dicText: String): String {
val lines = dicText.lineSequence().drop(1).map { it.replace("ß", "ss") }.distinct().toList()
return "${lines.size}\n" + lines.joinToString("\n")
}
private fun transformSwissGermanTrigrams(trigrams: List<String>?): List<String>? {
if (trigrams == null) return null
return trigrams
.flatMap { s ->
val replaced = s.replace("ß", "ss")
if (replaced.length > 3) (0..replaced.length - 3).map { start -> replaced.substring(start, start + 3) } else listOf(replaced)
}
.distinct()
.toList()
}
private fun getTrigrams(path: String): List<String>? {
val trigrams = File(path)
return if (!trigrams.exists()) return null else trigrams.readLines()
}
@@ -17,9 +17,9 @@ enum class HunspellDescriptor(
UKRAINIAN(LanguageISO.UK, isGplLicensed = true, 2);
override val storageDescriptor: String by lazy { "$storageName.jar" }
override val storageName: String by lazy { "hunspell-$iso-jvm-${GraziePlugin.Hunspell.version}" }
override val storageName: String by lazy { "hunspell-$iso-${GraziePlugin.Hunspell.version}" }
override val file: Path by lazy { Path(storageName).resolve(DICTIONARY_DIR).resolve("$iso.dic") }
override val url: String by lazy { "${GraziePlugin.Hunspell.url}/hunspell-$iso-jvm/${GraziePlugin.Hunspell.version}/$storageDescriptor" }
override val url: String by lazy { "${GraziePlugin.Hunspell.url}/hunspell-$iso/${GraziePlugin.Hunspell.version}/$storageDescriptor" }
companion object {
private const val DICTIONARY_DIR: String = "dictionary"
@@ -19,56 +19,50 @@
<orderEntry type="module" module-name="intellij.java.testFramework" />
<orderEntry type="module-library" exported="" scope="TEST">
<library name="ai.grazie.spell.hunspell.de" type="repository">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-de-jvm:0.2.281">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-de:0.2.288">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-de-jvm/0.2.281/hunspell-de-jvm-0.2.281.jar">
<sha256sum>17e8741b0bba21fd034d40a3b4190b819913f4ae9234b62e7c960bcd6cbd1c87</sha256sum>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-de/0.2.288/hunspell-de-0.2.288.jar">
<sha256sum>8457b66279cf47c6b8f71b2b910662968aa74a7f4e4076121823b86531ff1131</sha256sum>
</artifact>
</verification>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-de-jvm/0.2.281/hunspell-de-jvm-0.2.281.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-de/0.2.288/hunspell-de-0.2.288.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-de-jvm/0.2.281/hunspell-de-jvm-0.2.281-sources.jar!/" />
</SOURCES>
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library" exported="" scope="TEST">
<library name="ai.grazie.spell.hunspell.ru" type="repository">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-ru-jvm:0.2.281">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-ru:0.2.288">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-ru-jvm/0.2.281/hunspell-ru-jvm-0.2.281.jar">
<sha256sum>4c6be8d96b44761d3546ba127fcfb48bd31773b40a0ff23bdb4582c7afb33123</sha256sum>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-ru/0.2.288/hunspell-ru-0.2.288.jar">
<sha256sum>f14ae1a3e86c5df0668127a6215bf969d419d91de4be48e8e65681314df28c96</sha256sum>
</artifact>
</verification>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-ru-jvm/0.2.281/hunspell-ru-jvm-0.2.281.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-ru/0.2.288/hunspell-ru-0.2.288.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-ru-jvm/0.2.281/hunspell-ru-jvm-0.2.281-sources.jar!/" />
</SOURCES>
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library" exported="" scope="TEST">
<library name="ai.grazie.spell.hunspell.uk" type="repository">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-uk-jvm:0.2.281">
<properties include-transitive-deps="false" maven-id="ai.grazie.spell:hunspell-uk:0.2.288">
<verification>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-uk-jvm/0.2.281/hunspell-uk-jvm-0.2.281.jar">
<sha256sum>c51215918811365ae560ad4fc82e90b6b1afa3bd956162b92786857fc75e4dfe</sha256sum>
<artifact url="file://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-uk/0.2.288/hunspell-uk-0.2.288.jar">
<sha256sum>610372ac711d12381dd41aff588f12d3b7a37addf3c6e0bcb74f5667e69c0c24</sha256sum>
</artifact>
</verification>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-uk-jvm/0.2.281/hunspell-uk-jvm-0.2.281.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-uk/0.2.288/hunspell-uk-0.2.288.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/ai/grazie/spell/hunspell-uk-jvm/0.2.281/hunspell-uk-jvm-0.2.281-sources.jar!/" />
</SOURCES>
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
@@ -30,23 +30,21 @@ class HunspellBundleInfoTest : BasePlatformTestCase() {
@ParameterizedTest
@EnumSource(LanguageISO::class, names = ["EN", "DE", "UK", "RU"])
fun `test hunspell-jvm version matches grazie plugin dictionary version`(iso: LanguageISO) {
fun `test hunspell version matches grazie plugin dictionary version`(iso: LanguageISO) {
assertEquals(GraziePlugin.Hunspell.version, getDictionaryVersion(iso))
}
private fun isUrlValid(url: String, connectTimeoutMs: Int = 10000, readTimeoutMs: Int = 10000): Boolean {
val responseCode = HttpRequests.head(url)
return HttpRequests.head(url)
.connectTimeout(connectTimeoutMs)
.readTimeout(readTimeoutMs)
.throwStatusCodeException(false)
.tryConnect()
return responseCode == HttpURLConnection.HTTP_OK
.tryConnect() == HttpURLConnection.HTTP_OK
}
private fun getDictionaryVersion(iso: LanguageISO): String {
val language = iso.name.lowercase()
val path = HunspellBundleInfoTest::class.java.getClassLoader().getResource("dictionary/$language.aff")!!.toString()
val matcher = Pattern.compile(".*/hunspell-$language-jvm-((\\d|.)+)\\.jar!/.*").matcher(path)
val path = HunspellBundleInfoTest::class.java.getClassLoader().getResource("dictionary/$iso.aff")!!.toString()
val matcher = Pattern.compile(".*/hunspell-$iso-((\\d|.)+)\\.jar!/.*").matcher(path)
if (!matcher.matches()) throw AssertionError("Unexpected Hunspell jar path $path")
return matcher.group(1)
}
@@ -23,8 +23,7 @@ class HunspellGermanTest : GrazieTestBase() {
doTestSuggestion("anschließend", "anschliessend")
}
// HunspellWordList isn't fully ready to support some cases
fun `_test standard german suggestions`() {
fun `test standard german suggestions`() {
enableProofreadingFor(setOf(Lang.GERMANY_GERMAN))
doTestSuggestion("anschliessend", "anschließend")
}