diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index 257ecb615726..72a0ceaf4878 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -1711,6 +1711,13 @@ jvm_import( visibility = ["//visibility:public"] ) +jvm_import( + name = "grazie-tests-ai-grazie-spell-hunspell-de", + jar = "@hunspell-de-jvm-0_2_281_http//file", + source_jar = "@hunspell-de-jvm-0_2_281-sources_http//file", + visibility = ["//visibility:public"] +) + java_library( name = "grazie-tests-jetbrains-intellij-deps-languagetool-language-be", exports = [ @@ -4420,8 +4427,8 @@ jvm_import( jvm_import( name = "libraries-ai-grazie-spell-gec-engine-local-ai-grazie-spell-hunspell-en", - jar = "@hunspell-en-jvm-0_2_278_http//file", - source_jar = "@hunspell-en-jvm-0_2_278-sources_http//file", + jar = "@hunspell-en-jvm-0_2_281_http//file", + source_jar = "@hunspell-en-jvm-0_2_281-sources_http//file", visibility = ["//visibility:public"] ) diff --git a/lib/MODULE.bazel b/lib/MODULE.bazel index e1a3713bdfdc..efa39697afe2 100644 --- a/lib/MODULE.bazel +++ b/lib/MODULE.bazel @@ -2203,6 +2203,20 @@ http_file( downloaded_file_path = "uom-lib-common-1.1-sources.jar" ) +http_file( + name = "hunspell-de-jvm-0_2_281_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-de-jvm/0.2.281/hunspell-de-jvm-0.2.281.jar", + sha256 = "17e8741b0bba21fd034d40a3b4190b819913f4ae9234b62e7c960bcd6cbd1c87", + downloaded_file_path = "hunspell-de-jvm-0.2.281.jar" +) + +http_file( + name = "hunspell-de-jvm-0_2_281-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-de-jvm/0.2.281/hunspell-de-jvm-0.2.281-sources.jar", + sha256 = "8b8919a2003c603c6c892930f649e00b83b49e9b754a38bb8ea70cba3a5bfb99", + downloaded_file_path = "hunspell-de-jvm-0.2.281-sources.jar" +) + http_file( name = "language-be-6_5_0_12_http", url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/org/jetbrains/intellij/deps/languagetool/language-be/6.5.0.12/language-be-6.5.0.12.jar", @@ -6166,17 +6180,17 @@ http_file( ) http_file( - name = "hunspell-en-jvm-0_2_278_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-en-jvm/0.2.278/hunspell-en-jvm-0.2.278.jar", - sha256 = "245fd3041dad1e7076ae18bed50fb76a96609d014b42e68d5bfab783fba35789", - downloaded_file_path = "hunspell-en-jvm-0.2.278.jar" + name = "hunspell-en-jvm-0_2_281_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-en-jvm/0.2.281/hunspell-en-jvm-0.2.281.jar", + sha256 = "09766747bfa2f150e8ec2149cad08265bf92694a6cced61a1d8013a995152bf1", + downloaded_file_path = "hunspell-en-jvm-0.2.281.jar" ) http_file( - name = "hunspell-en-jvm-0_2_278-sources_http", - url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-en-jvm/0.2.278/hunspell-en-jvm-0.2.278-sources.jar", - sha256 = "73121da9a28bda12e5d12d488c86408a0bedb22998036078992f0ce09d35913c", - downloaded_file_path = "hunspell-en-jvm-0.2.278-sources.jar" + name = "hunspell-en-jvm-0_2_281-sources_http", + url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell/hunspell-en-jvm/0.2.281/hunspell-en-jvm-0.2.281-sources.jar", + sha256 = "3e80e17ac3d8ce626a4bbacaf349954036a129cc8ba09555e21acd287491e15b", + downloaded_file_path = "hunspell-en-jvm-0.2.281-sources.jar" ) http_file( diff --git a/libraries/ai.grazie.spell.gec.engine.local/intellij.libraries.ai.grazie.spell.gec.engine.local.iml b/libraries/ai.grazie.spell.gec.engine.local/intellij.libraries.ai.grazie.spell.gec.engine.local.iml index 2c45591d6679..79bfc1ba420c 100644 --- a/libraries/ai.grazie.spell.gec.engine.local/intellij.libraries.ai.grazie.spell.gec.engine.local.iml +++ b/libraries/ai.grazie.spell.gec.engine.local/intellij.libraries.ai.grazie.spell.gec.engine.local.iml @@ -88,19 +88,19 @@ - + - - 245fd3041dad1e7076ae18bed50fb76a96609d014b42e68d5bfab783fba35789 + + 09766747bfa2f150e8ec2149cad08265bf92694a6cced61a1d8013a995152bf1 - + - + diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieConfig.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieConfig.kt index 5a8119e2b27a..7c50b9f6891e 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieConfig.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieConfig.kt @@ -10,6 +10,7 @@ import com.intellij.grazie.grammar.grammarRules import com.intellij.grazie.ide.msg.GrazieInitializerManager import com.intellij.grazie.jlanguage.Lang import com.intellij.grazie.jlanguage.LangTool +import com.intellij.grazie.remote.GrazieRemote.isAvailableLocally import com.intellij.grazie.text.Rule import com.intellij.openapi.components.* import com.intellij.openapi.project.ProjectManager @@ -80,16 +81,20 @@ class GrazieConfig : PersistentStateComponent, ModificationT * NOTE: By default, availableLanguages are not included into [equals]. Check for it manually. */ val availableLanguages: Set by lazy { - enabledLanguages.asSequence().filter { lang -> lang.jLanguage != null }.toCollection(CollectionFactory.createSmallMemoryFootprintLinkedSet()) + enabledLanguages.asSequence().filter { it.jLanguage != null }.toCollection(CollectionFactory.createSmallMemoryFootprintLinkedSet()) } val missedLanguages: Set - get() = enabledLanguages.asSequence().filter { it.jLanguage == null }.toCollection(CollectionFactory.createSmallMemoryFootprintLinkedSet()) + get() = enabledLanguages.asSequence().filter { isMissingLanguage(it) }.toCollection(CollectionFactory.createSmallMemoryFootprintLinkedSet()) override fun increment() = copy(version = version.next() ?: error("Attempt to increment latest version $version")) fun hasMissedLanguages(): Boolean { - return enabledLanguages.any { it.jLanguage == null } + return enabledLanguages.any { isMissingLanguage(it) } + } + + fun isMissingLanguage(lang: Lang): Boolean { + return !isAvailableLocally(lang) && lang.jLanguage == null } } @@ -152,4 +157,4 @@ class GrazieConfig : PersistentStateComponent, ModificationT stateChanged(prevState, myState) } } -} +} \ No newline at end of file diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieDynamic.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieDynamic.kt index ac9e110cdb3d..fa279790cfdb 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieDynamic.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/GrazieDynamic.kt @@ -3,7 +3,10 @@ package com.intellij.grazie import com.intellij.DynamicBundle import com.intellij.grazie.jlanguage.Lang -import com.intellij.grazie.remote.GrazieRemote +import com.intellij.grazie.remote.GrazieRemote.isAvailableLocally +import com.intellij.grazie.remote.GrazieRemote.isValidBundleForLanguage +import com.intellij.grazie.remote.HunspellDescriptor +import com.intellij.grazie.remote.LanguageToolDescriptor import com.intellij.ide.plugins.DynamicPluginListener import com.intellij.ide.plugins.IdeaPluginDescriptor import com.intellij.openapi.application.ApplicationManager @@ -20,17 +23,13 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.util.* -import kotlin.io.path.isRegularFile @ApiStatus.Internal object GrazieDynamic : DynamicPluginListener { private val myDynClassLoaders by lazy { - val oldFiles = Files.walk(dynamicFolder).filter { file -> - file.isRegularFile() && Lang.values().all { it.remote.file.toAbsolutePath() != file.toAbsolutePath() } - } - for (file in oldFiles) { - file.delete() + for (file in getOldFiles()) { + file.delete(true) } ApplicationManager.getApplication().messageBus.connect() @@ -39,16 +38,26 @@ object GrazieDynamic : DynamicPluginListener { hashSetOf( UrlClassLoader.build() .parent(GraziePlugin.classLoader) - .files(collectValidLocalBundles()).get() + .files(collectValidLocalBundles()) + .get() ) } + /** + * Function that collects outdated directories that needs to be deleted. + */ + private fun getOldFiles(): List { + return Files.list(dynamicFolder) + .filter { file -> Lang.entries.none { file.fileName.toString() == getStorageDescriptor(it) } } + .toList() + } + private fun collectValidLocalBundles(): List { - val languages = GrazieRemote.allAvailableLocally() + val languages = Lang.entries.filter { isAvailableLocally(it) } val bundles = buildSet { for (language in languages) { - val path = language.remote.file - if (language.isEnglish() || GrazieRemote.isValidBundleForLanguage(language, path)) { + val path = getLangDynamicFolder(language).resolve(language.ltRemote!!.file) + if (language.isEnglish() || isValidBundleForLanguage(language, path)) { add(path) } else { thisLogger().error(""" @@ -80,6 +89,18 @@ object GrazieDynamic : DynamicPluginListener { return Paths.get(PathManager.getSystemPath(), "grazie") } + fun getLangDynamicFolder(lang: Lang): Path = dynamicFolder.resolve(getStorageDescriptor(lang)) + + /** + * Creates a storage descriptor (directory name) for downloader. + */ + private fun getStorageDescriptor(lang: Lang): String { + if (lang.hunspellRemote != null) { + return "${lang.iso}-LT${GraziePlugin.LanguageTool.version}-HN${GraziePlugin.Hunspell.version}" + } + return "${lang.iso}-LT${GraziePlugin.LanguageTool.version}" + } + val dynamicFolder: Path get() { val result = getDynamicFolderPath() @@ -88,7 +109,8 @@ object GrazieDynamic : DynamicPluginListener { } fun loadLang(lang: Lang): Language? { - for (className in lang.remote.langsClasses) { + val remote = lang.ltRemote ?: return null + for (className in remote.langsClasses) { try { Languages.getOrAddLanguageByClassName("org.languagetool.language.$className") } diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/GraziePlugin.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/GraziePlugin.kt index 159a0f9bc61a..beada7f185f9 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/GraziePlugin.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/GraziePlugin.kt @@ -1,9 +1,17 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.grazie +import com.intellij.grazie.GrazieDynamic.getLangDynamicFolder +import com.intellij.grazie.ide.msg.GrazieStateLifecycle +import com.intellij.grazie.jlanguage.Lang +import com.intellij.grazie.remote.GrazieRemote.isAvailableLocally import com.intellij.ide.plugins.IdeaPluginDescriptor import com.intellij.ide.plugins.PluginManagerCore import com.intellij.openapi.extensions.PluginId +import com.intellij.openapi.project.ProjectManager +import com.intellij.spellchecker.SpellCheckerManager +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.jetbrains.annotations.ApiStatus import java.nio.file.Path @@ -16,6 +24,34 @@ object GraziePlugin { const val url = "https://resources.jetbrains.com/grazie/model/language-tool" } + object Hunspell : GrazieStateLifecycle { + const val version = "0.2.281" + const val url = "https://packages.jetbrains.team/maven/p/grazi/grazie-platform-public/ai/grazie/spell" + + override fun update(prevState: GrazieConfig.State, newState: GrazieConfig.State) { + GrazieScope.coroutineScope().launch(Dispatchers.IO) { + val newLanguages = newState.enabledLanguages.filterHunspell() + val prevLanguages = prevState.enabledLanguages.filterHunspell() + + ProjectManager.getInstance().openProjects.forEach { project -> + val manager = SpellCheckerManager.getInstance(project) + newLanguages.forEach { new -> + val dicPath = getLangDynamicFolder(new).resolve(new.hunspellRemote!!.file).toString() + if (!manager.isDictionaryLoad(dicPath)) { + manager.loadDictionary(dicPath) + } + } + prevLanguages.forEach { prev -> + if (prev !in newLanguages) { + val dicPath = getLangDynamicFolder(prev).resolve(prev.hunspellRemote!!.file).toString() + manager.removeDictionary(dicPath) + } + } + } + } + } + } + private val descriptor: IdeaPluginDescriptor get() = PluginManagerCore.getPlugin(PluginId.getId(id))!! @@ -34,3 +70,9 @@ object GraziePlugin { val libFolder: Path get() = descriptor.pluginPath.resolve("lib") } + +private fun Collection.filterHunspell(): Set { + return asSequence() + .filter { it.hunspellRemote != null && isAvailableLocally(it) } + .toSet() +} \ No newline at end of file diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/msg/GrazieStateLifecycle.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/msg/GrazieStateLifecycle.kt index b859e8636302..dabf9dc28f14 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/msg/GrazieStateLifecycle.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/msg/GrazieStateLifecycle.kt @@ -2,6 +2,7 @@ package com.intellij.grazie.ide.msg import com.intellij.grazie.GrazieConfig +import com.intellij.grazie.GraziePlugin import com.intellij.grazie.jlanguage.LangTool import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.components.Service @@ -24,6 +25,7 @@ class GrazieInitializerManager { val application = ApplicationManager.getApplication() val connection = application.messageBus.connect() connection.subscribe(CONFIG_STATE_TOPIC, LangTool) + connection.subscribe(CONFIG_STATE_TOPIC, GraziePlugin.Hunspell) } fun register(subscriber: GrazieStateLifecycle): MessageBusConnection { diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/ProofreadSettingsPanel.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/ProofreadSettingsPanel.kt index 1483b33d4ff4..1e653d907148 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/ProofreadSettingsPanel.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/ProofreadSettingsPanel.kt @@ -7,6 +7,7 @@ import com.intellij.grazie.ide.ui.components.dsl.panel import com.intellij.grazie.ide.ui.proofreading.component.GrazieLanguagesComponent import com.intellij.grazie.jlanguage.Lang import com.intellij.grazie.remote.GrazieRemote +import com.intellij.grazie.remote.GrazieRemote.getLanguagesBasedOnUserAgreement import com.intellij.grazie.remote.LanguageDownloader import com.intellij.ide.DataManager import com.intellij.openapi.application.EDT @@ -17,6 +18,8 @@ import com.intellij.openapi.extensions.ExtensionPointName import com.intellij.openapi.options.Configurable import com.intellij.openapi.options.ConfigurableUi import com.intellij.openapi.options.ex.Settings +import com.intellij.openapi.project.Project +import com.intellij.openapi.project.guessCurrentProject import com.intellij.profile.codeInspection.ui.ErrorsConfigurable import com.intellij.ui.HyperlinkLabel import com.intellij.ui.layout.migLayout.createLayoutConstraints @@ -36,6 +39,7 @@ private val logger = logger() class ProofreadSettingsPanel : ConfigurableUi { private val EP: ExtensionPointName = ExtensionPointName("com.intellij.grazie.proofreadSettingsExtension") private val languages = GrazieLanguagesComponent(::download) + private val project: Project = guessCurrentProject(languages.component) private val downloadingLanguages: MutableSet = ConcurrentHashMap.newKeySet() private val downloadLabel: JLabel by lazy { @@ -45,9 +49,9 @@ class ProofreadSettingsPanel : ConfigurableUi { AsyncProcessIcon("Downloading language models").apply { isVisible = false } } - private suspend fun download(lang: Lang) { - withProcessIcon(lang) { - LanguageDownloader.startDownloading(listOf(it)) + private suspend fun download(langs: Collection) { + withProcessIcon(langs) { + LanguageDownloader.startDownloading(it) } languages.updateLinkToDownloadMissingLanguages() } @@ -99,12 +103,16 @@ class ProofreadSettingsPanel : ConfigurableUi { EP.extensionList.forEach { add(it.createComponent(), CC().wrap()) } } - private suspend fun withProcessIcon(lang: Lang, download: suspend (Lang) -> Unit) { + private suspend fun withProcessIcon(langs: Collection, download: suspend (Collection) -> Unit) { var failed = false try { - if (GrazieRemote.isAvailableLocally(lang)) return + if (GrazieRemote.allAvailableLocally(langs)) return + val filteredLanguages = withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) { + getLanguagesBasedOnUserAgreement(langs, project) + } + if (filteredLanguages.isEmpty()) return if (downloadingLanguages.isEmpty()) { - downloadingLanguages.add(lang) + downloadingLanguages.addAll(langs) withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) { downloadLabel.text = msg("grazie.settings.proofreading.languages.download") asyncDownloadingIcon.resume() @@ -112,20 +120,19 @@ class ProofreadSettingsPanel : ConfigurableUi { downloadLabel.isVisible = true } } - download(lang) + download(filteredLanguages) } catch (e: Exception) { withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) { - downloadLabel.text = msg("grazie.settings.proofreading.languages.download.failed", lang.displayName) + downloadLabel.text = msg("grazie.settings.proofreading.languages.download.failed", langs.joinToString { it.displayName }) asyncDownloadingIcon.suspend() } failed = true downloadingLanguages.clear() - logger.warn("Failed to download language '${lang.displayName}'", e) throw e } finally { - downloadingLanguages.remove(lang) + downloadingLanguages.removeAll(langs) if (downloadingLanguages.isEmpty() && !failed) { withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) { asyncDownloadingIcon.isVisible = false diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/GrazieLanguagesComponent.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/GrazieLanguagesComponent.kt index c4b55211893e..2a745c203cf5 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/GrazieLanguagesComponent.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/GrazieLanguagesComponent.kt @@ -13,19 +13,18 @@ import com.intellij.grazie.jlanguage.Lang import com.intellij.icons.AllIcons import com.intellij.ui.components.labels.LinkLabel import com.intellij.util.ui.JBUI -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch import java.awt.BorderLayout import javax.swing.JPanel -class GrazieLanguagesComponent(download: suspend (Lang) -> Unit) : GrazieUIComponent { +class GrazieLanguagesComponent(download: suspend (Collection) -> Unit) : GrazieUIComponent { private val languages = GrazieLanguagesList(download) { updateLinkToDownloadMissingLanguages() } private val link: LinkLabel = LinkLabel(msg("grazie.notification.missing-languages.action"), AllIcons.General.Warning).configure { border = padding(JBUI.insetsTop(10)) - setListener({ _, _ -> GrazieConfig.get().missedLanguages.forEach { GrazieScope.coroutineScope().launch { download(it) } } }, null) + setListener({ _, _ -> GrazieScope.coroutineScope().launch { download(GrazieConfig.get().missedLanguages) } }, null) } override val component: JPanel = panel { diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesList.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesList.kt index 4623f385873f..fe501c354e5b 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesList.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesList.kt @@ -21,7 +21,7 @@ import java.awt.BorderLayout import javax.swing.JComponent import javax.swing.ListCellRenderer -class GrazieLanguagesList(private val download: suspend (Lang) -> Unit, private val onLanguageRemoved: (lang: Lang) -> Unit) : +class GrazieLanguagesList(private val download: suspend (Collection) -> Unit, private val onLanguageRemoved: (lang: Lang) -> Unit) : AddDeleteListPanel(null, emptyList()), GrazieUIComponent { private val decorator: ToolbarDecorator = MyToolbarDecorator(myList) @@ -78,8 +78,8 @@ class GrazieLanguagesList(private val download: suspend (Lang) -> Unit, private /** Returns pair of (available languages, languages to download) */ private fun getLangsForPopup(): Pair, List> { - val enabledLangs = myListModel.elements().asSequence().map { it.displayName }.toSet() - val (available, toDownload) = Lang.sortedValues().filter { it.displayName !in enabledLangs }.partition { it.isAvailable() } + val enabledLangs = myListModel.elements().asSequence().map { it.nativeName }.toSet() + val (available, toDownload) = Lang.sortedValues().filter { it.nativeName !in enabledLangs }.partition { it.isAvailable() } return available to toDownload } diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupElementRenderer.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupElementRenderer.kt index 2f9341d2af09..14fce27e218c 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupElementRenderer.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupElementRenderer.kt @@ -31,7 +31,7 @@ class GrazieLanguagesPopupElementRenderer(list: ListPopupImpl) : PopupListElemen } override fun customizeComponent(list: JList, lang: Lang, isSelected: Boolean) { - @NlsSafe val size = lang.remote.size.takeUnless { lang.isAvailable() } ?: "" + @NlsSafe val size = lang.size.toString().takeUnless { lang.isAvailable() } ?: "" mySizeLabel.configure { text = size foreground = myTextLabel.foreground.takeIf { isSelected } ?: Color.GRAY diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupStep.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupStep.kt index 35a4b5220b6f..acc8b223bdcf 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupStep.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/ui/proofreading/component/list/GrazieLanguagesPopupStep.kt @@ -6,6 +6,7 @@ import com.intellij.grazie.jlanguage.Lang import com.intellij.openapi.application.EDT import com.intellij.openapi.application.ModalityState import com.intellij.openapi.application.asContextElement +import com.intellij.openapi.diagnostic.logger import com.intellij.openapi.ui.popup.ListSeparator import com.intellij.openapi.ui.popup.PopupStep import com.intellij.openapi.ui.popup.util.BaseListPopupStep @@ -15,11 +16,12 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +private val logger = logger() + class GrazieLanguagesPopupStep( @NlsContexts.PopupTitle title: String, available: List, toDownload: List, - private val download: suspend (Lang) -> Unit, private val onResult: (Lang) -> Unit, -) - : BaseListPopupStep(title, available + toDownload) { + private val download: suspend (Collection) -> Unit, private val onResult: (Lang) -> Unit, +) : BaseListPopupStep(title, available + toDownload) { private val firstOther = toDownload.firstOrNull() override fun getSeparatorAbove(value: Lang) = if (value == firstOther) ListSeparator() else null @@ -30,9 +32,14 @@ class GrazieLanguagesPopupStep( override fun onChosen(selectedValue: Lang, finalChoice: Boolean): PopupStep<*>? { return doFinalStep { GrazieScope.coroutineScope().launch { - download(selectedValue) - withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) { - onResult(selectedValue) + try { + download(listOf(selectedValue)) + withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) { + onResult(selectedValue) + } + } + catch (e: Exception) { + logger.warn("Failed to download language '$selectedValue'", e) } } } diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/Lang.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/Lang.kt index 86de09eda27d..49b2d518e5e0 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/Lang.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/Lang.kt @@ -4,61 +4,81 @@ package com.intellij.grazie.jlanguage import ai.grazie.nlp.langs.LanguageISO import com.intellij.grazie.GrazieDynamic import com.intellij.grazie.remote.GrazieRemote +import com.intellij.grazie.remote.HunspellDescriptor +import com.intellij.grazie.remote.LanguageToolDescriptor import com.intellij.grazie.remote.RemoteLangDescriptor import com.intellij.openapi.util.NlsSafe import org.languagetool.Language import org.languagetool.language.English import org.languagetool.noop.NoopChunker -enum class Lang(val displayName: String, val className: String, val remote: RemoteLangDescriptor, @NlsSafe val nativeName: String) { - BRITISH_ENGLISH("English (GB)", "BritishEnglish", RemoteLangDescriptor.ENGLISH, "English (Great Britain)"), - AMERICAN_ENGLISH("English (US)", "AmericanEnglish", RemoteLangDescriptor.ENGLISH, "English (USA)"), - CANADIAN_ENGLISH("English (Canadian)", "CanadianEnglish", RemoteLangDescriptor.ENGLISH, "English (Canada)"), - ARABIC("Arabic", "Arabic", RemoteLangDescriptor.ARABIC, "العربيةُ"), - ASTURIAN("Asturian", "Asturian", RemoteLangDescriptor.ASTURIAN, "Asturianu"), - BELARUSIAN("Belarusian", "Belarusian", RemoteLangDescriptor.BELARUSIAN, "Беларуская"), - BRETON("Breton", "Breton", RemoteLangDescriptor.BRETON, "Brezhoneg"), - CATALAN("Catalan", "Catalan", RemoteLangDescriptor.CATALAN, "Català"), - VALENCIAN_CATALAN("Catalan (Valencian)", "ValencianCatalan", RemoteLangDescriptor.CATALAN, "Català (Valencià)"), - DANISH("Danish", "Danish", RemoteLangDescriptor.DANISH, "Dansk"), - GERMANY_GERMAN("German (Germany)", "GermanyGerman", RemoteLangDescriptor.GERMAN, "Deutsch (Deutschland)"), - AUSTRIAN_GERMAN("German (Austria)", "AustrianGerman", RemoteLangDescriptor.GERMAN, "Deutsch (Österreich)"), - SWISS_GERMAN("German (Switzerland)", "SwissGerman", RemoteLangDescriptor.GERMAN, "Deutsch (Die Schweiz)"), - GREEK("Greek", "Greek", RemoteLangDescriptor.GREEK, "Ελληνικά"), - ESPERANTO("Esperanto", "Esperanto", RemoteLangDescriptor.ESPERANTO, "Esperanto"), - SPANISH("Spanish", "Spanish", RemoteLangDescriptor.SPANISH, "Español"), - PERSIAN("Persian", "Persian", RemoteLangDescriptor.PERSIAN, "فارسی"), - FRENCH("French", "French", RemoteLangDescriptor.FRENCH, "Français"), - IRISH("Irish", "Irish", RemoteLangDescriptor.IRISH, "Gaeilge"), - GALICIAN("Galician", "Galician", RemoteLangDescriptor.GALICIAN, "Galego"), - ITALIAN("Italian", "Italian", RemoteLangDescriptor.ITALIAN, "Italiano"), - JAPANESE("Japanese", "Japanese", RemoteLangDescriptor.JAPANESE, "日本語"), - KHMER("Khmer", "Khmer", RemoteLangDescriptor.KHMER, "ភាសាខ្មែរ"), - DUTCH("Dutch", "Dutch", RemoteLangDescriptor.DUTCH, "Nederlands"), - POLISH("Polish", "Polish", RemoteLangDescriptor.POLISH, "Polski"), - PORTUGAL_PORTUGUESE("Portuguese (Portugal)", "PortugalPortuguese", RemoteLangDescriptor.PORTUGUESE, "Português (Portugal)"), - BRAZILIAN_PORTUGUESE("Portuguese (Brazil)", "BrazilianPortuguese", RemoteLangDescriptor.PORTUGUESE, "Português (Brasil)"), - ANGOLA_PORTUGUESE("Portuguese (Angola)", "AngolaPortuguese", RemoteLangDescriptor.PORTUGUESE, "Português (Angola)"), - MOZAMBIQUE_PORTUGUESE("Portuguese (Mozambique)", "MozambiquePortuguese", RemoteLangDescriptor.PORTUGUESE, "Português (Moçambique)"), - ROMANIAN("Romanian", "Romanian", RemoteLangDescriptor.ROMANIAN, "Română"), - RUSSIAN("Russian", "Russian", RemoteLangDescriptor.RUSSIAN, "Русский"), - SLOVAK("Slovak", "Slovak", RemoteLangDescriptor.SLOVAK, "Slovenčina"), - SLOVENIAN("Slovenian", "Slovenian", RemoteLangDescriptor.SLOVENIAN, "Slovenščina"), - SWEDISH("Swedish", "Swedish", RemoteLangDescriptor.SWEDISH, "Svenska"), - TAMIL("Tamil", "Tamil", RemoteLangDescriptor.TAMIL, "தமிழ்"), - TAGALOG("Tagalog", "Tagalog", RemoteLangDescriptor.TAGALOG, "Tagalog"), - UKRAINIAN("Ukrainian", "Ukrainian", RemoteLangDescriptor.UKRAINIAN, "Українська"), - CHINESE("Chinese", "Chinese", RemoteLangDescriptor.CHINESE, "中文"); +enum class Lang(val displayName: String, val className: String, val iso: LanguageISO, @NlsSafe val nativeName: String) { + BRITISH_ENGLISH("English (GB)", "BritishEnglish", LanguageISO.EN, "English (Great Britain)"), + AMERICAN_ENGLISH("English (US)", "AmericanEnglish", LanguageISO.EN, "English (USA)"), + CANADIAN_ENGLISH("English (Canadian)", "CanadianEnglish", LanguageISO.EN, "English (Canada)"), + ARABIC("Arabic", "Arabic", LanguageISO.AR, "العربيةُ"), + ASTURIAN("Asturian", "Asturian", LanguageISO.AST, "Asturianu"), + BELARUSIAN("Belarusian", "Belarusian", LanguageISO.BE, "Беларуская"), + BRETON("Breton", "Breton", LanguageISO.BR, "Brezhoneg"), + CATALAN("Catalan", "Catalan", LanguageISO.CA, "Català"), + VALENCIAN_CATALAN("Catalan (Valencian)", "ValencianCatalan", LanguageISO.CA, "Català (Valencià)"), + DANISH("Danish", "Danish", LanguageISO.DA, "Dansk"), + GERMANY_GERMAN("German (Germany)", "GermanyGerman", LanguageISO.DE, "Deutsch (Deutschland)"), + AUSTRIAN_GERMAN("German (Austria)", "AustrianGerman", LanguageISO.DE, "Deutsch (Österreich)"), + SWISS_GERMAN("German (Switzerland)", "SwissGerman", LanguageISO.DE, "Deutsch (Die Schweiz)"), + GREEK("Greek", "Greek", LanguageISO.EL, "Ελληνικά"), + ESPERANTO("Esperanto", "Esperanto", LanguageISO.EO, "Esperanto"), + SPANISH("Spanish", "Spanish", LanguageISO.ES, "Español"), + PERSIAN("Persian", "Persian", LanguageISO.FA, "فارسی"), + FRENCH("French", "French", LanguageISO.FR, "Français"), + IRISH("Irish", "Irish", LanguageISO.GA, "Gaeilge"), + GALICIAN("Galician", "Galician", LanguageISO.GL, "Galego"), + ITALIAN("Italian", "Italian", LanguageISO.IT, "Italiano"), + JAPANESE("Japanese", "Japanese", LanguageISO.JA, "日本語"), + KHMER("Khmer", "Khmer", LanguageISO.KM, "ភាសាខ្មែរ"), + DUTCH("Dutch", "Dutch", LanguageISO.NL, "Nederlands"), + POLISH("Polish", "Polish", LanguageISO.PL, "Polski"), + PORTUGAL_PORTUGUESE("Portuguese (Portugal)", "PortugalPortuguese", LanguageISO.PT, "Português (Portugal)"), + BRAZILIAN_PORTUGUESE("Portuguese (Brazil)", "BrazilianPortuguese", LanguageISO.PT, "Português (Brasil)"), + ANGOLA_PORTUGUESE("Portuguese (Angola)", "AngolaPortuguese", LanguageISO.PT, "Português (Angola)"), + MOZAMBIQUE_PORTUGUESE("Portuguese (Mozambique)", "MozambiquePortuguese", LanguageISO.PT, "Português (Moçambique)"), + ROMANIAN("Romanian", "Romanian", LanguageISO.RO, "Română"), + RUSSIAN("Russian", "Russian", LanguageISO.RU, "Русский"), + SLOVAK("Slovak", "Slovak", LanguageISO.SK, "Slovenčina"), + SLOVENIAN("Slovenian", "Slovenian", LanguageISO.SL, "Slovenščina"), + SWEDISH("Swedish", "Swedish", LanguageISO.SV, "Svenska"), + TAMIL("Tamil", "Tamil", LanguageISO.TA, "தமிழ்"), + TAGALOG("Tagalog", "Tagalog", LanguageISO.TL, "Tagalog"), + UKRAINIAN("Ukrainian", "Ukrainian", LanguageISO.UK, "Українська"), + CHINESE("Chinese", "Chinese", LanguageISO.ZH, "中文"); companion object { - fun sortedValues() = values().sortedBy(Lang::nativeName) + fun sortedValues(): List = entries.sortedBy(Lang::nativeName) // the chunker can be very memory-, disk- and CPU-expensive internal fun shouldDisableChunker(language: Language): Boolean = language is English } - val iso: LanguageISO - get() = remote.iso + val ltRemote: LanguageToolDescriptor? + get() = LanguageToolDescriptor.entries.find { it.iso == iso } + + val hunspellRemote: HunspellDescriptor? + get() = HunspellDescriptor.entries.find { it.iso == iso } + + val remoteDescriptors: List + get() = listOfNotNull(ltRemote, hunspellRemote) + + val size: Int + get() = (LanguageToolDescriptor.entries.find { it.iso == iso }?.size ?: 0) + + (HunspellDescriptor.entries.find { it.iso == iso }?.size ?: 0) + + val shortDisplayName: String + get() { + if (iso == LanguageISO.DE) return "German" + if (iso == LanguageISO.EN) return "English" + if (iso == LanguageISO.PT) return "Portuguese" + return displayName + } private var _jLanguage: Language? = null val jLanguage: Language? diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/LangTool.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/LangTool.kt index 99ce2fbd51d5..a4623082bbd6 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/LangTool.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/jlanguage/LangTool.kt @@ -38,7 +38,7 @@ object LangTool : GrazieStateLifecycle { Hunspell.setHunspellDictionaryFactory(::LuceneHunspellDictionary) } - internal fun globalIdPrefix(lang: Lang): String = "LanguageTool." + lang.remote.iso.name + "." + internal fun globalIdPrefix(lang: Lang): String = "LanguageTool." + lang.ltRemote!!.iso.name + "." fun getTool(lang: Lang): JLanguageTool { // this is equivalent to computeIfAbsent, but allows multiple threads to create tools concurrently, diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/GrazieRemote.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/GrazieRemote.kt index dc7715a3ec7d..340edd649f66 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/GrazieRemote.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/GrazieRemote.kt @@ -1,11 +1,14 @@ // Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.grazie.remote +import com.intellij.CommonBundle import com.intellij.grazie.GrazieConfig import com.intellij.grazie.GrazieDynamic -import com.intellij.grazie.GraziePlugin +import com.intellij.grazie.ide.ui.components.dsl.msg import com.intellij.grazie.jlanguage.Lang import com.intellij.openapi.project.Project +import com.intellij.openapi.ui.Messages +import com.intellij.util.concurrency.annotations.RequiresEdt import com.intellij.util.io.DigestUtil import org.jetbrains.annotations.ApiStatus import java.nio.file.Path @@ -14,15 +17,17 @@ import kotlin.io.path.exists import kotlin.io.path.inputStream object GrazieRemote { - private fun isLibExists(lib: String) = GrazieDynamic.dynamicFolder.resolve(lib).exists() || GraziePlugin.libFolder.resolve(lib).exists() - fun isAvailableLocally(lang: Lang): Boolean = lang.isEnglish() || isLibExists(lang.remote.fileName) + fun isAvailableLocally(lang: Lang): Boolean { + if (lang.isEnglish()) return true + return GrazieDynamic.getLangDynamicFolder(lang).exists() + } fun allAvailableLocally(languages: Collection): Boolean = languages.all { isAvailableLocally(it) } - fun allAvailableLocally(): List = Lang.entries.filter { isAvailableLocally(it) } - /** Downloads [lang] to local storage */ + @Deprecated("Use downloadAsync(Collection, Project) instead", replaceWith = ReplaceWith("downloadAsync(listOf(lang), project)")) + @ApiStatus.ScheduledForRemoval fun download(lang: Lang): Boolean = LanguageDownloader.download(lang) /** Downloads [languages] asynchronously to local storage */ @@ -31,9 +36,33 @@ object GrazieRemote { /** Downloads all missing languages to local storage*/ fun downloadMissing(project: Project): Unit = downloadAsync(GrazieConfig.get().missedLanguages, project) + /** + * Get user agreement before downloading licensed language bundle + * @return true if the user agrees with license, false if the user doesn't agree or agreement isn't required + */ + @RequiresEdt + fun getLanguagesBasedOnUserAgreement(languages: Collection, project: Project): Collection { + val gplLanguages = languages + .filter { it.hunspellRemote?.isGplLicensed == true } + .toList() + if (gplLanguages.isEmpty()) return languages + + val hasUserAgreement = Messages.showOkCancelDialog( + project, + msg("grazie.license.gpl.message", gplLanguages.joinToString { it.shortDisplayName }), + msg("grazie.license.gpl.title"), + CommonBundle.getOkButtonText(), + if (languages.size == gplLanguages.size) CommonBundle.getCancelButtonText() else msg("grazie.license.gpl.cancel"), + Messages.getQuestionIcon() + ) == Messages.OK + if (hasUserAgreement) return languages + return languages.filter { it.hunspellRemote?.isGplLicensed != true } + } + fun isValidBundleForLanguage(language: Lang, file: Path): Boolean { + val remote = language.ltRemote ?: return false val actualChecksum = checksum(file) - return language.remote.checksum == actualChecksum + return remote.checksum == actualChecksum } @ApiStatus.Internal diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/HunspellDescriptor.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/HunspellDescriptor.kt new file mode 100644 index 000000000000..068103ba4d6a --- /dev/null +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/HunspellDescriptor.kt @@ -0,0 +1,39 @@ +package com.intellij.grazie.remote + +import ai.grazie.nlp.langs.LanguageISO +import com.intellij.grazie.GraziePlugin +import java.io.FilenameFilter +import java.nio.file.Path +import kotlin.io.path.Path + + +enum class HunspellDescriptor( + override val iso: LanguageISO, + val isGplLicensed: Boolean, + override val size: Int, +) : RemoteLangDescriptor { + RUSSIAN(LanguageISO.RU, isGplLicensed = false, 2), + GERMAN(LanguageISO.DE, isGplLicensed = true, 2), + UKRAINIAN(LanguageISO.UK, isGplLicensed = true, 2); + + override val storageDescriptor: String by lazy { "$storageName.jar" } + override val storageName: String by lazy { "hunspell-$iso-jvm-${GraziePlugin.Hunspell.version}" } + override val file: Path by lazy { Path(storageName).resolve(DICTIONARY_DIR).resolve("$iso.dic") } + override val url: String by lazy { "${GraziePlugin.Hunspell.url}/hunspell-$iso-jvm/${GraziePlugin.Hunspell.version}/$storageDescriptor" } + + companion object { + private const val DICTIONARY_DIR: String = "dictionary" + + /** + * Filter that is used to unpack hunspell jar dictionary. + * It only retains the content of the "dictionary" directory, licenses and notice files + */ + fun filenameFilter(): FilenameFilter { + return FilenameFilter { dir, name -> + dir.name == HunspellDescriptor.DICTIONARY_DIR || + dir.parent == HunspellDescriptor.DICTIONARY_DIR || + name.startsWith("GPL") || name.equals("license") || name.equals("notice") + } + } + } +} \ No newline at end of file diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/LanguageDownloader.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/LanguageDownloader.kt index 55e32158bc92..167207de9147 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/LanguageDownloader.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/LanguageDownloader.kt @@ -8,7 +8,11 @@ import com.intellij.grazie.GrazieScope import com.intellij.grazie.ide.ui.components.dsl.msg import com.intellij.grazie.jlanguage.Lang import com.intellij.grazie.remote.GrazieRemote.allAvailableLocally +import com.intellij.grazie.remote.GrazieRemote.getLanguagesBasedOnUserAgreement import com.intellij.grazie.remote.GrazieRemote.isAvailableLocally +import com.intellij.openapi.application.EDT +import com.intellij.openapi.application.ModalityState +import com.intellij.openapi.application.asContextElement import com.intellij.openapi.diagnostic.thisLogger import com.intellij.openapi.project.Project import com.intellij.openapi.util.io.NioFiles @@ -16,28 +20,37 @@ import com.intellij.platform.ide.progress.ModalTaskOwner import com.intellij.platform.ide.progress.runWithModalProgressBlocking import com.intellij.platform.ide.progress.withBackgroundProgress import com.intellij.util.download.DownloadableFileService +import com.intellij.util.io.ZipUtil import com.intellij.util.lang.UrlClassLoader import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.jetbrains.annotations.ApiStatus +import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.copyTo @Suppress("DialogTitleCapitalization") internal object LanguageDownloader { + @Deprecated("Use downloadAsync(Collection, Project) instead", replaceWith = ReplaceWith("downloadAsync(listOf(lang), project)")) + @ApiStatus.ScheduledForRemoval fun download(lang: Lang): Boolean { if (isAvailableLocally(lang)) return true val path = runDownload(lang) ?: return false - performGrazieUpdate(listOf(lang to path)) + performGrazieUpdate(LanguageBundles(lang to path)) return true } fun downloadAsync(languages: Collection, project: Project) { if (allAvailableLocally(languages)) return GrazieScope.coroutineScope().launch { + val filteredLanguages = withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) { + getLanguagesBasedOnUserAgreement(languages, project) + } + if (languages.isEmpty()) return@launch withBackgroundProgress(project, msg("grazie.settings.proofreading.languages.download"), true) { - startDownloading(languages) + startDownloading(filteredLanguages) } } } @@ -56,7 +69,7 @@ internal object LanguageDownloader { msg("grazie.settings.proofreading.languages.download") ) { performDownload(listOf(language)) - }.single().second + }.languages.entries.single().value } catch (exception: Throwable) { thisLogger().warn(exception) @@ -64,15 +77,30 @@ internal object LanguageDownloader { } } - private fun performGrazieUpdate(bundles: List>) { - bundles.forEach { (lang, path) -> - check(GrazieRemote.isValidBundleForLanguage(lang, path)) { "Language bundle checksum became invalid right before loading it: $lang" } + private fun performGrazieUpdate(bundles: LanguageBundles) { + if (bundles.languages.isNotEmpty()) { + bundles.languages.forEach { (lang, path) -> + // Each language has its own LT jar file, at least for now + val jarPath = path.resolve(lang.ltRemote!!.storageName) + check(GrazieRemote.isValidBundleForLanguage(lang, jarPath)) { "Language bundle checksum became invalid right before loading it: $lang" } + } + val classLoader = UrlClassLoader.build() + .parent(GraziePlugin.classLoader) + .files(bundles.languages.map { it.value.resolve(it.key.ltRemote!!.storageName) }) + .get() + GrazieDynamic.addDynClassLoader(classLoader) } - val classLoader = UrlClassLoader.build() - .parent(GraziePlugin.classLoader) - .files(bundles.map { it.second }) - .get() - GrazieDynamic.addDynClassLoader(classLoader) + bundles.hunspellLangs.forEach { (lang, path) -> + val zip = path.resolve(lang.hunspellRemote!!.storageDescriptor) + val outputDir = path.resolve(lang.hunspellRemote!!.storageName) + Files.createDirectories(outputDir) + ZipUtil.extract(zip, outputDir, HunspellDescriptor.filenameFilter()) + NioFiles.deleteRecursively(zip) + } + reloadGrazie() + } + + private fun reloadGrazie() { // force reloading available language classes GrazieConfig.update { it.copy() } // drop caches, restart highlighting @@ -80,33 +108,58 @@ internal object LanguageDownloader { } @Throws(IllegalStateException::class) - private fun performDownload(languages: Collection): List> { - val bundles = doDownload(languages) - val invalidBundles = bundles + private fun performDownload(languages: Collection): LanguageBundles { + val bundles = downloadLanguages(languages) + val invalidBundles = bundles.jLangs + .map { (lang, path) -> lang to path.resolve(lang.ltRemote!!.storageName) } .filter { !GrazieRemote.isValidBundleForLanguage(it.first, it.second) } .map { it.second } if (invalidBundles.isNotEmpty()) { - bundles.forEach { NioFiles.deleteRecursively(it.second) } + bundles.languages.forEach { NioFiles.deleteRecursively(it.value) } throw IllegalStateException("Failed to verify integrity of downloaded language bundle for languages ${invalidBundles}.") } return bundles } private fun promptToSelectLanguageBundleManually(language: Lang): Path? { + language.ltRemote ?: return null val selectedFile = OfflineLanguageBundleSelectionDialog.show(null, language) ?: return null - val targetPath = GrazieDynamic.dynamicFolder.resolve(language.remote.fileName) + val targetPath = GrazieDynamic.getLangDynamicFolder(language).resolve(language.ltRemote!!.storageName) selectedFile.copyTo(targetPath, overwrite = true) return targetPath } - private fun doDownload(languages: Collection): List> { + private fun downloadLanguages(languages: Collection): LanguageBundles { val downloaderService = DownloadableFileService.getInstance() - val descriptors = languages - .map { downloaderService.createFileDescription(it.remote.url, it.remote.fileName) } - val paths = downloaderService - .createDownloader(descriptors, msg("grazie.settings.proofreading.languages.download")) - .download(GrazieDynamic.dynamicFolder.toFile()) - .map { it.first.toPath() } - return languages.zip(paths) + val paths = mutableMapOf() + try { + languages.forEach { lang -> + val folder = GrazieDynamic.getLangDynamicFolder(lang) + val descriptors = lang.remoteDescriptors + .map { it.url to it.storageDescriptor } + .map { downloaderService.createFileDescription(it.first, it.second) } + downloaderService + .createDownloader(descriptors, msg("grazie.settings.proofreading.languages.download")) + .download(folder.toFile()) + paths.put(lang, folder) + } + } + catch (e: Exception) { + paths.forEach { NioFiles.deleteRecursively(it.value) } + throw e + } + return LanguageBundles(paths) + } + + /** + * [Path] in this map corresponds to the absolute path of [RemoteLangDescriptor.storageDescriptor] + */ + private data class LanguageBundles(val languages: Map) { + val hunspellLangs: Map + get() = languages.filterKeys { it.hunspellRemote != null } + val jLangs: Map + get() = languages.filterKeys { it.ltRemote != null } + + constructor(lang: Pair) : this(mapOf(lang)) } } \ No newline at end of file diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/LanguageToolDescriptor.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/LanguageToolDescriptor.kt new file mode 100644 index 000000000000..3dbbfc444d96 --- /dev/null +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/LanguageToolDescriptor.kt @@ -0,0 +1,91 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.grazie.remote + +import ai.grazie.nlp.langs.LanguageISO +import com.intellij.grazie.GraziePlugin +import java.nio.file.Path +import kotlin.io.path.Path + +// These checksums may be obtained by running the [GrazieLanguageBundleInfoTest] +private const val EN_CHECKSUM = "e8d6689b88d9d58810e6edc1eb4a7105" +private const val AR_CHECKSUM = "224c6b9caed912036cbd3a966c57e5ae" +private const val AST_CHECKSUM = "c43afc5eabefdb3009e995f011196a50" +private const val BE_CHECKSUM = "cc938a3c469b254c124bd2b9a7f37530" +private const val BR_CHECKSUM = "5e71090b9b50cee612a4a972ad231210" +private const val CA_CHECKSUM = "546f2345599e58da15228db7eb2e7629" +private const val DA_CHECKSUM = "fbbc3df19ee5a9bfd87802e87975c588" +private const val DE_CHECKSUM = "3ac0c372a665fd6e93cf7974e19e1410" +private const val EL_CHECKSUM = "a6a57ea4ef35bcea71bc9e601a16ff50" +private const val EO_CHECKSUM = "8e1a11ab0c2a669f98e01c28698a735e" +private const val ES_CHECKSUM = "9d6ff14e4f32dffb36182724e00b328c" +private const val FA_CHECKSUM = "f8307ae6ac5931835f18b4dcb1f0fdf2" +private const val FR_CHECKSUM = "4a94f481437b58018682bc6a3afb80b8" +private const val GA_CHECKSUM = "7cec917d8f3a2ff5209ab2e7f45114a1" +private const val GL_CHECKSUM = "77dc2c28f49f6020cc55c67847ac3dd6" +private const val IT_CHECKSUM = "3384246394ef168eb2b1b33133dd37a3" +private const val JA_CHECKSUM = "1301561f934693e5d6af720fb40e7da8" +private const val KM_CHECKSUM = "1fbd2e2839f0f55a1f3aadeea5c30a80" +private const val NL_CHECKSUM = "67335aa741e28eeb9240ed58601f2c06" +private const val PL_CHECKSUM = "80b420ee734d17d49cd44161900686d6" +private const val PT_CHECKSUM = "f074a6dff5ca136bb58a2459af7047ed" +private const val RO_CHECKSUM = "00364d568765bccfeeb323985e503bac" +private const val RU_CHECKSUM = "66a9eae4c6bdf3b905abd4c63f981699" +private const val SK_CHECKSUM = "f55829f1a0787bb5a0c2616ef6f626dc" +private const val SL_CHECKSUM = "0fb8907a330092f96a4e7c8e8c3fdb99" +private const val SV_CHECKSUM = "2a7640fb469ebaf0e00286010ba198a6" +private const val TA_CHECKSUM = "cb8a371558e03428ea25c8d65f6041bc" +private const val TL_CHECKSUM = "da95f9ec497cc5f451474652c2789781" +private const val UK_CHECKSUM = "428451f61de4c1873919067d0eac4b67" +private const val ZH_CHECKSUM = "33590632112f2bf7a25653f99ea18ad2" + +enum class LanguageToolDescriptor( + val langsClasses: List, + override val size: Int, + override val iso: LanguageISO, + val checksum: String, +) : RemoteLangDescriptor { + ARABIC(listOf("Arabic"), 13, LanguageISO.AR, AR_CHECKSUM), + ASTURIAN(listOf("Asturian"), 1, LanguageISO.AST, AST_CHECKSUM), + BELARUSIAN(listOf("Belarusian"), 1, LanguageISO.BE, BE_CHECKSUM), + BRETON(listOf("Breton"), 2, LanguageISO.BR, BR_CHECKSUM), + CATALAN(listOf("Catalan", "ValencianCatalan", "BalearicCatalan"), 4, LanguageISO.CA, CA_CHECKSUM), + DANISH(listOf("Danish"), 1, LanguageISO.DA, DA_CHECKSUM), + GERMAN(listOf("GermanyGerman", "AustrianGerman", "SwissGerman"), 20, LanguageISO.DE, DE_CHECKSUM), + GREEK(listOf("Greek"), 1, LanguageISO.EL, EL_CHECKSUM), + ENGLISH( + listOf("BritishEnglish", "AmericanEnglish", "CanadianEnglish"), + 16, + LanguageISO.EN, + EN_CHECKSUM + ), + ESPERANTO(listOf("Esperanto"), 1, LanguageISO.EO, EO_CHECKSUM), + SPANISH(listOf("Spanish"), 3, LanguageISO.ES, ES_CHECKSUM), + PERSIAN(listOf("Persian"), 1, LanguageISO.FA, FA_CHECKSUM), + FRENCH(listOf("French"), 2, LanguageISO.FR, FR_CHECKSUM), + IRISH(listOf("Irish"), 13, LanguageISO.GA, GA_CHECKSUM), + GALICIAN(listOf("Galician"), 5, LanguageISO.GL, GL_CHECKSUM), + ITALIAN(listOf("Italian"), 1, LanguageISO.IT, IT_CHECKSUM), + JAPANESE(listOf("Japanese"), 21, LanguageISO.JA, JA_CHECKSUM), + KHMER(listOf("Khmer"), 1, LanguageISO.KM, KM_CHECKSUM), + DUTCH(listOf("Dutch"), 37, LanguageISO.NL, NL_CHECKSUM), + POLISH(listOf("Polish"), 5, LanguageISO.PL, PL_CHECKSUM), + PORTUGUESE( + listOf("PortugalPortuguese", "BrazilianPortuguese", "AngolaPortuguese", "MozambiquePortuguese"), + 5, + LanguageISO.PT, + PT_CHECKSUM + ), + ROMANIAN(listOf("Romanian"), 2, LanguageISO.RO, RO_CHECKSUM), + RUSSIAN(listOf("Russian"), 5, LanguageISO.RU, RU_CHECKSUM), + SLOVAK(listOf("Slovak"), 3, LanguageISO.SK, SK_CHECKSUM), + SLOVENIAN(listOf("Slovenian"), 1, LanguageISO.SL, SL_CHECKSUM), + SWEDISH(listOf("Swedish"), 1, LanguageISO.SV, SV_CHECKSUM), + TAMIL(listOf("Tamil"), 1, LanguageISO.TA, TA_CHECKSUM), + TAGALOG(listOf("Tagalog"), 1, LanguageISO.TL, TL_CHECKSUM), + UKRAINIAN(listOf("Ukrainian"), 7, LanguageISO.UK, UK_CHECKSUM), + CHINESE(listOf("Chinese"), 8, LanguageISO.ZH, ZH_CHECKSUM); + + override val storageName: String by lazy { "$iso-${GraziePlugin.LanguageTool.version}.jar" } + override val file: Path by lazy { Path(storageName) } + override val url: String by lazy { "${GraziePlugin.LanguageTool.url}/${GraziePlugin.LanguageTool.version}/$storageName" } +} diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/OfflineLanguageBundleSelectionDialog.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/OfflineLanguageBundleSelectionDialog.kt index 359e8752d235..b644576bf623 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/OfflineLanguageBundleSelectionDialog.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/OfflineLanguageBundleSelectionDialog.kt @@ -28,7 +28,7 @@ internal class OfflineLanguageBundleSelectionDialog private constructor( override fun createCenterPanel(): JComponent { return panel { row { - text(msg("grazie.offline.language.bundle.dialog.text", language.nativeName, language.remote.url)) + text(msg("grazie.offline.language.bundle.dialog.text", language.nativeName, language.ltRemote!!.url)) } } } diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/RemoteLangDescriptor.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/RemoteLangDescriptor.kt index cf2e2f089ff9..e7bcf9877cb8 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/RemoteLangDescriptor.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/remote/RemoteLangDescriptor.kt @@ -1,91 +1,45 @@ -// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.grazie.remote import ai.grazie.nlp.langs.LanguageISO -import com.intellij.grazie.GrazieDynamic -import com.intellij.grazie.GraziePlugin import java.nio.file.Path -// These checksums may be obtained by running the [GrazieLanguageBundleInfoTest] -private const val EN_CHECKSUM = "e8d6689b88d9d58810e6edc1eb4a7105" -private const val AR_CHECKSUM = "224c6b9caed912036cbd3a966c57e5ae" -private const val AST_CHECKSUM = "c43afc5eabefdb3009e995f011196a50" -private const val BE_CHECKSUM = "cc938a3c469b254c124bd2b9a7f37530" -private const val BR_CHECKSUM = "5e71090b9b50cee612a4a972ad231210" -private const val CA_CHECKSUM = "546f2345599e58da15228db7eb2e7629" -private const val DA_CHECKSUM = "fbbc3df19ee5a9bfd87802e87975c588" -private const val DE_CHECKSUM = "3ac0c372a665fd6e93cf7974e19e1410" -private const val EL_CHECKSUM = "a6a57ea4ef35bcea71bc9e601a16ff50" -private const val EO_CHECKSUM = "8e1a11ab0c2a669f98e01c28698a735e" -private const val ES_CHECKSUM = "9d6ff14e4f32dffb36182724e00b328c" -private const val FA_CHECKSUM = "f8307ae6ac5931835f18b4dcb1f0fdf2" -private const val FR_CHECKSUM = "4a94f481437b58018682bc6a3afb80b8" -private const val GA_CHECKSUM = "7cec917d8f3a2ff5209ab2e7f45114a1" -private const val GL_CHECKSUM = "77dc2c28f49f6020cc55c67847ac3dd6" -private const val IT_CHECKSUM = "3384246394ef168eb2b1b33133dd37a3" -private const val JA_CHECKSUM = "1301561f934693e5d6af720fb40e7da8" -private const val KM_CHECKSUM = "1fbd2e2839f0f55a1f3aadeea5c30a80" -private const val NL_CHECKSUM = "67335aa741e28eeb9240ed58601f2c06" -private const val PL_CHECKSUM = "80b420ee734d17d49cd44161900686d6" -private const val PT_CHECKSUM = "f074a6dff5ca136bb58a2459af7047ed" -private const val RO_CHECKSUM = "00364d568765bccfeeb323985e503bac" -private const val RU_CHECKSUM = "66a9eae4c6bdf3b905abd4c63f981699" -private const val SK_CHECKSUM = "f55829f1a0787bb5a0c2616ef6f626dc" -private const val SL_CHECKSUM = "0fb8907a330092f96a4e7c8e8c3fdb99" -private const val SV_CHECKSUM = "2a7640fb469ebaf0e00286010ba198a6" -private const val TA_CHECKSUM = "cb8a371558e03428ea25c8d65f6041bc" -private const val TL_CHECKSUM = "da95f9ec497cc5f451474652c2789781" -private const val UK_CHECKSUM = "428451f61de4c1873919067d0eac4b67" -private const val ZH_CHECKSUM = "33590632112f2bf7a25653f99ea18ad2" -enum class RemoteLangDescriptor( - val langsClasses: List, - val size: String, - val iso: LanguageISO, - val checksum: String -) { - ARABIC(listOf("Arabic"), "13 MB", LanguageISO.AR, AR_CHECKSUM), - ASTURIAN(listOf("Asturian"), "1 MB", LanguageISO.AST, AST_CHECKSUM), - BELARUSIAN(listOf("Belarusian"), "1 MB", LanguageISO.BE, BE_CHECKSUM), - BRETON(listOf("Breton"), "2 MB", LanguageISO.BR, BR_CHECKSUM), - CATALAN(listOf("Catalan", "ValencianCatalan", "BalearicCatalan"), "4 MB", LanguageISO.CA, CA_CHECKSUM), - DANISH(listOf("Danish"), "1 MB", LanguageISO.DA, DA_CHECKSUM), - GERMAN(listOf("GermanyGerman", "AustrianGerman", "SwissGerman"), "20 MB", LanguageISO.DE, DE_CHECKSUM), - GREEK(listOf("Greek"), "1 MB", LanguageISO.EL, EL_CHECKSUM), - ENGLISH( - listOf("BritishEnglish", "AmericanEnglish", "CanadianEnglish"), - "16 MB", - LanguageISO.EN, - EN_CHECKSUM - ), - ESPERANTO(listOf("Esperanto"), "1 MB", LanguageISO.EO, EO_CHECKSUM), - SPANISH(listOf("Spanish"), "3 MB", LanguageISO.ES, ES_CHECKSUM), - PERSIAN(listOf("Persian"), "1 MB", LanguageISO.FA, FA_CHECKSUM), - FRENCH(listOf("French"), "2 MB", LanguageISO.FR, FR_CHECKSUM), - IRISH(listOf("Irish"), "13 MB", LanguageISO.GA, GA_CHECKSUM), - GALICIAN(listOf("Galician"), "5 MB", LanguageISO.GL, GL_CHECKSUM), - ITALIAN(listOf("Italian"), "1 MB", LanguageISO.IT, IT_CHECKSUM), - JAPANESE(listOf("Japanese"), "21 MB", LanguageISO.JA, JA_CHECKSUM), - KHMER(listOf("Khmer"), "1 MB", LanguageISO.KM, KM_CHECKSUM), - DUTCH(listOf("Dutch"), "37 MB", LanguageISO.NL, NL_CHECKSUM), - POLISH(listOf("Polish"), "5 MB", LanguageISO.PL, PL_CHECKSUM), - PORTUGUESE( - listOf("PortugalPortuguese", "BrazilianPortuguese", "AngolaPortuguese", "MozambiquePortuguese"), - "5 MB", - LanguageISO.PT, - PT_CHECKSUM - ), - ROMANIAN(listOf("Romanian"), "2 MB", LanguageISO.RO, RO_CHECKSUM), - RUSSIAN(listOf("Russian"), "5 MB", LanguageISO.RU, RU_CHECKSUM), - SLOVAK(listOf("Slovak"), "3 MB", LanguageISO.SK, SK_CHECKSUM), - SLOVENIAN(listOf("Slovenian"), "1 MB", LanguageISO.SL, SL_CHECKSUM), - SWEDISH(listOf("Swedish"), "1 MB", LanguageISO.SV, SV_CHECKSUM), - TAMIL(listOf("Tamil"), "1 MB", LanguageISO.TA, TA_CHECKSUM), - TAGALOG(listOf("Tagalog"), "1 MB", LanguageISO.TL, TL_CHECKSUM), - UKRAINIAN(listOf("Ukrainian"), "7 MB", LanguageISO.UK, UK_CHECKSUM), - CHINESE(listOf("Chinese"), "8 MB", LanguageISO.ZH, ZH_CHECKSUM); +interface RemoteLangDescriptor { + /** + * Returns the file name or directory name where dictionaries are stored locally. + * In the case of simple files (for example, .jar), it simply returns the file name. + */ + val storageName: String - val fileName: String by lazy { "$iso-${GraziePlugin.LanguageTool.version}.jar" } - val file: Path by lazy { GrazieDynamic.dynamicFolder.resolve(fileName) } - val url: String by lazy { "${GraziePlugin.LanguageTool.url}/${GraziePlugin.LanguageTool.version}/$fileName" } -} + /** + * An object that may be used to locate a file in a local file system. + * Implementation is free to decide what the file exactly is. + * For example, it is a jar file in case of [LanguageToolDescriptor] and a .dic file in case of [HunspellDescriptor]. + * + * Must return a relative path to [com.intellij.grazie.GrazieDynamic.getLangDynamicFolder] + */ + val file: Path + + /** + * The URL from which the file can be downloaded. + */ + val url: String + + /** + * The size of the file in megabytes. + */ + val size: Int + + /** + * The ISO code of the language. + */ + val iso: LanguageISO + + /** + * Used to create a storage descriptor for downloader. + * + * In the case of simple files (for example, .jar), it simply returns the storage name. + */ + val storageDescriptor: String + get() = storageName +} \ No newline at end of file diff --git a/plugins/grazie/src/main/resources/messages/GrazieBundle.properties b/plugins/grazie/src/main/resources/messages/GrazieBundle.properties index 6b4bb244942a..02ff4436d9a3 100644 --- a/plugins/grazie/src/main/resources/messages/GrazieBundle.properties +++ b/plugins/grazie/src/main/resources/messages/GrazieBundle.properties @@ -20,6 +20,11 @@ grazie.offline.language.bundle.dialog.select.action.text=Select language bundle grazie.offline.language.bundle.dialog.descriptor.title=Choose Language Bundle for {0} Language grazie.offline.language.bundle.dialog.descriptor.error={0} is not a valid language bundle for {1} language. +# License warning +grazie.license.gpl.title=GPL License +grazie.license.gpl.message={0} dictionaries are licensed as free software under General Public License. \Do you want to proceed with the download? +grazie.license.gpl.cancel=Download other dictionaries + # Grammar Inspection Variables grazie.grammar.inspection.grammar.text=Grammar @@ -82,7 +87,6 @@ grazie.settings.grammar.scope.places-to-check.comments=Comments grazie.settings.grammar.scope.places-to-check.documentation=Documentation grazie.settings.grammar.scope.places-to-check.commits=Commit messages - grazie.settings.grammar.exceptions.empty.text=No exceptions added. grazie.settings.grammar.exceptions.empty.text.explanation=To add an exception, press {0} on a grammar mistake. grazie.settings.grammar.exceptions.add.title=Add New Exception diff --git a/plugins/grazie/src/test/BUILD.bazel b/plugins/grazie/src/test/BUILD.bazel index 26dd09f2c81f..d7cac4422388 100644 --- a/plugins/grazie/src/test/BUILD.bazel +++ b/plugins/grazie/src/test/BUILD.bazel @@ -29,6 +29,7 @@ jvm_library( "//plugins/grazie", "@lib//:kotlin-test", "//java/testFramework", + "@lib//:grazie-tests-ai-grazie-spell-hunspell-de", "@lib//:grazie-tests-org-jetbrains-intellij-deps-languagetool-languagetool-core", "@lib//:grazie-tests-org-jetbrains-intellij-deps-languagetool-language-ru", "@lib//:grazie-tests-org-jetbrains-intellij-deps-languagetool-language-de", diff --git a/plugins/grazie/src/test/intellij.grazie.tests.iml b/plugins/grazie/src/test/intellij.grazie.tests.iml index f8f36e006b11..f4c41066cb6b 100644 --- a/plugins/grazie/src/test/intellij.grazie.tests.iml +++ b/plugins/grazie/src/test/intellij.grazie.tests.iml @@ -17,6 +17,24 @@ + + + + + + 17e8741b0bba21fd034d40a3b4190b819913f4ae9234b62e7c960bcd6cbd1c87 + + + + + + + + + + + + diff --git a/plugins/grazie/src/test/kotlin/com/intellij/grazie/hunspell/HunspellBundleInfoTest.kt b/plugins/grazie/src/test/kotlin/com/intellij/grazie/hunspell/HunspellBundleInfoTest.kt new file mode 100644 index 000000000000..e0e388f42e6b --- /dev/null +++ b/plugins/grazie/src/test/kotlin/com/intellij/grazie/hunspell/HunspellBundleInfoTest.kt @@ -0,0 +1,55 @@ +package com.intellij.grazie.hunspell + +import com.intellij.grazie.GraziePlugin +import com.intellij.grazie.remote.HunspellDescriptor +import com.intellij.openapi.diagnostic.Logger +import com.intellij.testFramework.fixtures.BasePlatformTestCase +import com.intellij.util.io.HttpRequests +import org.junit.Assume +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.net.HttpURLConnection +import java.util.regex.Pattern + +@RunWith(JUnit4::class) +class HunspellBundleInfoTest: BasePlatformTestCase() { + + val logger: Logger = Logger.getInstance(HunspellBundleInfoTest::class.java) + + @Test + fun `check that grazie dictionary exists`() { + Assume.assumeTrue("Must not be run under TeamCity", !IS_UNDER_TEAMCITY) + HunspellDescriptor.entries.forEach { + if (!isUrlValid(it.url)) { + fail("Failed to verify that hunspell dictionary ${it.url} exists") + } + } + } + + @Test + fun `test hunspell-en-jvm version matches grazie plugin dictionary version`() { + assertEquals(GraziePlugin.Hunspell.version, getDictionaryVersion()) + } + + private fun isUrlValid(url: String, connectTimeoutMs: Int = 10000, readTimeoutMs: Int = 10000): Boolean { + return try { + val responseCode = HttpRequests.head(url) + .connectTimeout(connectTimeoutMs) + .readTimeout(readTimeoutMs) + .throwStatusCodeException(false) // Don't throw exceptions for non-200 status codes + .tryConnect() + responseCode == HttpURLConnection.HTTP_OK + } catch (e: Exception) { + logger.error("Request to $url failed", e) + false + } + } + + private fun getDictionaryVersion(): String { + val path = HunspellBundleInfoTest::class.java.getClassLoader().getResource("dictionary/en.aff")!!.toString() + val matcher = Pattern.compile(".*/hunspell-en-jvm-((\\d|.)+)\\.jar!/.*").matcher(path) + if (!matcher.matches()) throw AssertionError("Unexpected Hunspell jar path $path") + return matcher.group(1) + } +} \ No newline at end of file diff --git a/plugins/grazie/src/test/kotlin/com/intellij/grazie/hunspell/HunspellGermanTest.kt b/plugins/grazie/src/test/kotlin/com/intellij/grazie/hunspell/HunspellGermanTest.kt new file mode 100644 index 000000000000..bbf731aaecc8 --- /dev/null +++ b/plugins/grazie/src/test/kotlin/com/intellij/grazie/hunspell/HunspellGermanTest.kt @@ -0,0 +1,54 @@ +package com.intellij.grazie.hunspell + +import ai.grazie.gec.spell.en.dict.HunspellDeutschLibraryDescriptor +import com.intellij.grazie.GrazieDynamic +import com.intellij.grazie.GraziePlugin +import com.intellij.grazie.GrazieTestBase +import com.intellij.grazie.jlanguage.Lang +import com.intellij.grazie.remote.HunspellDescriptor +import com.intellij.openapi.application.PathManager +import com.intellij.openapi.util.io.NioFiles +import com.intellij.spellchecker.SpellCheckerManager.Companion.getInstance +import com.intellij.util.io.ZipUtil +import java.nio.file.Files +import kotlin.io.path.Path + + +class HunspellGermanTest : GrazieTestBase() { + + override fun setUp() { + super.setUp() + + val hunspellLib = "hunspell-de-jvm-${GraziePlugin.Hunspell.version}.jar" + val zipPath = PathManager.getJarPathForClass(HunspellDeutschLibraryDescriptor::class.java) + if (zipPath == null) { + fail("$hunspellLib not found in classpath") + } + val zip = Path(zipPath!!) + if (!Files.exists(zip)) { + fail("$hunspellLib not found in classpath") + } + val deDir = GrazieDynamic.getLangDynamicFolder(Lang.GERMANY_GERMAN) + val outputDir = deDir.resolve(HunspellDescriptor.GERMAN.storageName) + Files.createDirectories(outputDir) + ZipUtil.extract(zip, outputDir, HunspellDescriptor.filenameFilter()) + getInstance(project).loadDictionary(deDir.resolve(HunspellDescriptor.GERMAN.file).toString()) + } + + override fun tearDown() { + try { + getInstance(project).removeDictionary(HunspellDescriptor.GERMAN.file.toString()) + NioFiles.deleteRecursively(GrazieDynamic.getLangDynamicFolder(Lang.GERMANY_GERMAN)) + } + catch (e: Throwable) { + addSuppressedException(e) + } + finally { + super.tearDown() + } + } + + fun `test hunspell de`() { + runHighlightTestForFile("hunspell/Hunspell.java") + } +} \ No newline at end of file diff --git a/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/GrazieStaticSearchableOptionsTest.kt b/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/GrazieStaticSearchableOptionsTest.kt index 011a1043f307..6320d95f859a 100644 --- a/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/GrazieStaticSearchableOptionsTest.kt +++ b/plugins/grazie/src/test/kotlin/com/intellij/grazie/ide/GrazieStaticSearchableOptionsTest.kt @@ -46,8 +46,7 @@ class GrazieStaticSearchableOptionsTest { } private fun collectLanguageProofreadingOptions(): List { - val languages = Lang.values() - return languages.map { "${it.displayName} ${it.nativeName}".trim() } + return Lang.entries.map { "${it.displayName} ${it.nativeName}".trim() } } private fun collectRuleOptions(): List { diff --git a/plugins/grazie/src/test/kotlin/com/intellij/grazie/remote/GrazieLanguageBundleInfoTest.kt b/plugins/grazie/src/test/kotlin/com/intellij/grazie/remote/LanguageToolBundleInfoTest.kt similarity index 77% rename from plugins/grazie/src/test/kotlin/com/intellij/grazie/remote/GrazieLanguageBundleInfoTest.kt rename to plugins/grazie/src/test/kotlin/com/intellij/grazie/remote/LanguageToolBundleInfoTest.kt index 1b2bd85b0e32..f4695f003abe 100644 --- a/plugins/grazie/src/test/kotlin/com/intellij/grazie/remote/GrazieLanguageBundleInfoTest.kt +++ b/plugins/grazie/src/test/kotlin/com/intellij/grazie/remote/LanguageToolBundleInfoTest.kt @@ -15,21 +15,21 @@ import org.junit.runners.JUnit4 import java.nio.file.Path @RunWith(JUnit4::class) -class GrazieLanguageBundleInfoTest: BasePlatformTestCase() { +class LanguageToolBundleInfoTest: BasePlatformTestCase() { @get:Rule val temporaryDirectory = TemporaryDirectory() /** * Test basically doing the same stuff as [com.jetbrains.resharper.external.services.grazie.resources.GrazieResourceBuilder] * If it fails with "some-LANG-id' is not a language code known to LanguageTool.", you should add the new language class to - * the [com.intellij.grazie.remote.RemoteLangDescriptor] + * the [com.intellij.grazie.remote.LanguageToolDescriptor] */ @Test fun `check that all languages are loaded correctly`() { - Lang.entries.map { lang -> + Lang.entries.filter { it.ltRemote != null }.map { lang -> var jLanguage = lang.jLanguage if (jLanguage == null) { - val logger = Logger.getInstance(GrazieLanguageBundleInfoTest::class.java) + val logger = Logger.getInstance(LanguageToolBundleInfoTest::class.java) logger.info("Language pack for ${lang.displayName} not found, downloading...") if (!GrazieRemote.download(lang)) { logger.info("Failed to download language pack for ${lang.displayName}") @@ -46,28 +46,28 @@ class GrazieLanguageBundleInfoTest: BasePlatformTestCase() { fun `verify hardcoded checksums are valid`() { // Do not run this test on build server, since artifact downloading will produce flaky failures Assume.assumeTrue("Must not be run under TeamCity", !IS_UNDER_TEAMCITY) - val languages = Lang.values() + val remotes = LanguageToolDescriptor.entries val expected = linkedMapOf() val actual = linkedMapOf() - for (language in languages) { - println("Checking $language") - val key = language.remote.iso.toString().uppercase() + for (remote in remotes) { + println("Checking $remote") + val key = remote.iso.toString().uppercase() if (expected.contains(key)) { println("Already checked for $key") continue } - val path = downloadBundle(language) + val path = downloadBundle(remote) expected[key] = "private const val ${key}_CHECKSUM = \"${GrazieRemote.checksum(path)}\"" - actual[key] = "private const val ${key}_CHECKSUM = \"${language.remote.checksum}\"" + actual[key] = "private const val ${key}_CHECKSUM = \"${remote.checksum}\"" } assertEquals("In case language tool was updated, please update checksums in RemoteLangDescriptor.kt", expected.values.joinToString("\n"), actual.values.joinToString("\n")) } - private fun downloadBundle(language: Lang): Path { + private fun downloadBundle(remote: LanguageToolDescriptor): Path { val downloaderService = DownloadableFileService.getInstance() val downloader = downloaderService.createDownloader( - listOf(downloaderService.createFileDescription(language.remote.url, language.remote.fileName)), - "Downloading $language" + listOf(downloaderService.createFileDescription(remote.url, remote.storageName)), + "Downloading ${remote.iso}" ) val result = downloader.download(temporaryDirectory.createDir().toFile()).single() return result.first.toPath() diff --git a/plugins/grazie/src/test/testData/hunspell/Hunspell.java b/plugins/grazie/src/test/testData/hunspell/Hunspell.java new file mode 100644 index 000000000000..a2e49c38ed30 --- /dev/null +++ b/plugins/grazie/src/test/testData/hunspell/Hunspell.java @@ -0,0 +1,3 @@ +class Hunspell { + // Das ist sehr libraryspezifische Code. +} \ No newline at end of file diff --git a/spellchecker/src/com/intellij/spellchecker/SpellCheckerManager.kt b/spellchecker/src/com/intellij/spellchecker/SpellCheckerManager.kt index 8524f8da63a3..27243f8ad186 100644 --- a/spellchecker/src/com/intellij/spellchecker/SpellCheckerManager.kt +++ b/spellchecker/src/com/intellij/spellchecker/SpellCheckerManager.kt @@ -219,19 +219,22 @@ class SpellCheckerManager @Internal constructor(@Internal val project: Project, spellChecker.addModifiableDictionary(projectDictionary!!) } - internal fun loadDictionary(path: String) { + fun loadDictionary(path: String) { val dictionaryProvider = findApplicable(path) if (dictionaryProvider == null) { spellChecker!!.loadDictionary(FileLoader(path)) + return } - else { - val dictionary = dictionaryProvider.get(path) - if (dictionary != null) { - spellChecker!!.addDictionary(dictionary) - } + val dictionary = dictionaryProvider.get(path) + if (dictionary != null) { + spellChecker!!.addDictionary(dictionary) } } + fun removeDictionary(path: String): Unit = spellChecker!!.removeDictionary(path) + + fun isDictionaryLoad(path: String): Boolean = spellChecker!!.isDictionaryLoad(path) + fun hasProblem(word: String): Boolean { return !spellChecker!!.isCorrect(word) && !isCorrectExtensionWord(word) } diff --git a/spellchecker/src/com/intellij/spellchecker/hunspell/HunspellDictionary.kt b/spellchecker/src/com/intellij/spellchecker/hunspell/HunspellDictionary.kt index d393e73542a6..b9b889b35f90 100644 --- a/spellchecker/src/com/intellij/spellchecker/hunspell/HunspellDictionary.kt +++ b/spellchecker/src/com/intellij/spellchecker/hunspell/HunspellDictionary.kt @@ -4,14 +4,13 @@ package com.intellij.spellchecker.hunspell import ai.grazie.spell.lists.hunspell.HunspellWordList import com.intellij.openapi.progress.ProgressManager import com.intellij.openapi.util.io.FileUtilRt -import com.intellij.openapi.vfs.VfsUtil.findFileByIoFile import com.intellij.spellchecker.dictionary.Dictionary import com.intellij.util.Consumer import java.io.File import java.io.FileNotFoundException import java.io.InputStreamReader -internal data class HunspellBundle(val dic: File, val aff: File) +internal data class HunspellBundle(val dic: File, val aff: File, val trigrams: File) class HunspellDictionary(path: String, name: String? = null) : Dictionary { companion object { @@ -21,8 +20,9 @@ class HunspellDictionary(path: String, name: String? = null) : Dictionary { val pathWithoutExtension = FileUtilRt.getNameWithoutExtension(path) val dic = File("$pathWithoutExtension.dic") val aff = File("$pathWithoutExtension.aff") + val trigrams = File("$pathWithoutExtension.trigrams.txt") - return if (dic.exists() && aff.exists()) HunspellBundle(dic, aff) else null + return if (dic.exists() && aff.exists()) HunspellBundle(dic, aff, trigrams) else null } fun isHunspell(path: String): Boolean { @@ -35,21 +35,20 @@ class HunspellDictionary(path: String, name: String? = null) : Dictionary { private val alphabet: HashSet = HashSet() init { - val bundle = loadHunspellBundle(path) if (bundle !== null) { - this.dict = bundle.aff.inputStream().use { affix -> - bundle.dic.inputStream().use { dictionary -> - HunspellWordList( - affix, - dictionary, - checkCanceled = { ProgressManager.checkCanceled() } - ) - } + var trigrams: List? = null + if (bundle.trigrams.exists()) { + trigrams = InputStreamReader(bundle.trigrams.inputStream(), Charsets.UTF_8).use { it.readLines() } } - val file = findFileByIoFile(bundle.dic, true)!! - InputStreamReader(file.inputStream, file.charset).use { reader -> + this.dict = HunspellWordList.create( + bundle.aff.readText(), + bundle.dic.readText(), + trigrams + ) { ProgressManager.checkCanceled() } + + InputStreamReader(bundle.dic.inputStream()).use { reader -> reader.forEachLine { line -> line.takeWhile { it != ' ' && it != '/' }.lowercase().chars().forEach { this.alphabet.add(it) } } diff --git a/spellchecker/testSrc/com/intellij/spellchecker/dictionary/AppDictionaryTest.java b/spellchecker/testSrc/com/intellij/spellchecker/dictionary/AppDictionaryTest.java index 6463dfb0ef86..9e46768f2bee 100644 --- a/spellchecker/testSrc/com/intellij/spellchecker/dictionary/AppDictionaryTest.java +++ b/spellchecker/testSrc/com/intellij/spellchecker/dictionary/AppDictionaryTest.java @@ -4,6 +4,7 @@ package com.intellij.spellchecker.dictionary; import com.intellij.spellchecker.inspection.SpellcheckerInspectionTestCase; import com.intellij.util.containers.ContainerUtil; +import org.jetbrains.annotations.NotNull; import java.util.*; @@ -27,7 +28,7 @@ public class AppDictionaryTest extends SpellcheckerInspectionTestCase { doContainTest(wordToCheck, Present); } - private static void doContainTest(String wordToCheck, Dictionary.LookupStatus lookupStatus) { + private static void doContainTest(String wordToCheck, @NotNull Dictionary.LookupStatus lookupStatus) { assertEquals(lookupStatus, APP_DICTIONARY.lookup(wordToCheck)); } @@ -36,7 +37,7 @@ public class AppDictionaryTest extends SpellcheckerInspectionTestCase { } public void testContainsNeg() { - doContainTest("eeeee", null); + doContainTest("eeeee", Alien); } public void testWords() { diff --git a/spellchecker/testSrc/com/intellij/spellchecker/dictionary/ProjectDictionaryTest.java b/spellchecker/testSrc/com/intellij/spellchecker/dictionary/ProjectDictionaryTest.java index 1bf53b4d9999..a70482b530d9 100644 --- a/spellchecker/testSrc/com/intellij/spellchecker/dictionary/ProjectDictionaryTest.java +++ b/spellchecker/testSrc/com/intellij/spellchecker/dictionary/ProjectDictionaryTest.java @@ -3,6 +3,7 @@ package com.intellij.spellchecker.dictionary; import com.intellij.spellchecker.inspection.SpellcheckerInspectionTestCase; import com.intellij.util.containers.ContainerUtil; +import org.jetbrains.annotations.NotNull; import java.util.*; @@ -28,7 +29,7 @@ public class ProjectDictionaryTest extends SpellcheckerInspectionTestCase { doContainTest(wordToCheck, Present); } - private static void doContainTest(String wordToCheck, Dictionary.LookupStatus status) { + private static void doContainTest(String wordToCheck, @NotNull Dictionary.LookupStatus status) { assertEquals(status, myProjectDictionary.lookup(wordToCheck)); } @@ -37,7 +38,7 @@ public class ProjectDictionaryTest extends SpellcheckerInspectionTestCase { } public void testContainsNeg() { - doContainTest("eeeee", null); + doContainTest("eeeee", Alien); } public void testWords() {