From 18e028cf9035796de9defe815aea159d2f6bc2cf Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Mon, 29 Apr 2024 14:16:19 +0200 Subject: [PATCH] IJPL-148294 simplify grazie dependencies in spellchecker, report module level library GitOrigin-RevId: 2701139177d6f97d71c1ca9242a7953997937132 --- .../build/CommunityLibraryLicenses.kt | 3 - .../intellij/build/impl/JarPackager.kt | 101 ++---- .../ProjectStructureMapping.kt | 38 +-- spellchecker/intellij.spellchecker.iml | 313 +++++++----------- 4 files changed, 173 insertions(+), 282 deletions(-) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt index 4af103192d3b..05de43c2f86f 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt @@ -383,19 +383,16 @@ object CommunityLibraryLicenses { LibraryLicense(name = "Grazie AI", libraryName = "ai.grazie.spell.gec.engine.local", url = "https://packages.jetbrains.team/maven/p/grazi/grazie-platform-public/", additionalLibraryNames = listOf("ai.grazie.nlp.patterns", - "ai.grazie.nlp.phonetics", "ai.grazie.nlp.common", "ai.grazie.nlp.langs", "ai.grazie.nlp.similarity", "ai.grazie.nlp.detect", - "ai.grazie.nlp.stemmer", "ai.grazie.nlp.tokenizer", "ai.grazie.utils.common", "ai.grazie.utils.json", "ai.grazie.utils.lucene.lt.compatibility", "ai.grazie.model.bdg.jvm", "ai.grazie.model.common", - "ai.grazie.model.gec", "ai.grazie.model.text", "ai.grazie.spell.hunspell.en", "ai.grazie.emb", diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt index e37dab59887e..d04fe21463f0 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt @@ -124,7 +124,7 @@ class JarPackager private constructor( private val assets = LinkedHashMap() private val libToMetadata = HashMap() - private val copiedFiles = HashMap() + private val copiedFiles = HashMap() private val helper = (context as BuildContextImpl).jarPackagerDependencyHelper @@ -152,36 +152,21 @@ class JarPackager private constructor( packager.computeModuleCustomLibrarySources(layout) val clientModuleFilter = context.jetBrainsClientModuleFilter - val libraryToMerge = packager.computeProjectLibrariesSources( - outDir = outputDir, - layout = layout, - copiedFiles = packager.copiedFiles, - clientModuleFilter = clientModuleFilter, - ) + val libraryToMerge = packager.computeProjectLibrariesSources(outDir = outputDir, layout = layout, copiedFiles = packager.copiedFiles, clientModuleFilter = clientModuleFilter) if (isRootDir) { for ((key, value) in predefinedMergeRules) { - packager.mergeLibsByPredicate( - jarName = key, - libraryToMerge = libraryToMerge, - outputDir = outputDir, - predicate = value, - clientModuleFilter = clientModuleFilter, - ) + packager.mergeLibsByPredicate(jarName = key, libraryToMerge = libraryToMerge, outputDir = outputDir, predicate = value, clientModuleFilter = clientModuleFilter) } + if (!libraryToMerge.isEmpty()) { val clientLibraries = libraryToMerge.filterKeys { clientModuleFilter.isProjectLibraryIncluded(it.name) } if (clientLibraries.isNotEmpty()) { - packager.filesToSourceWithMappings( - uberJarFile = outputDir.resolve(PlatformJarNames.LIB_CLIENT_JAR), - libraryToMerge = clientLibraries, - ) + packager.filesToSourceWithMappings(uberJarFile = outputDir.resolve(PlatformJarNames.LIB_CLIENT_JAR), libraryToMerge = clientLibraries) } + val nonClientLibraries = libraryToMerge.filterKeys { !clientModuleFilter.isProjectLibraryIncluded(it.name) } if (nonClientLibraries.isNotEmpty()) { - packager.filesToSourceWithMappings( - uberJarFile = outputDir.resolve(PlatformJarNames.LIB_JAR), - libraryToMerge = nonClientLibraries, - ) + packager.filesToSourceWithMappings(uberJarFile = outputDir.resolve(PlatformJarNames.LIB_JAR), libraryToMerge = nonClientLibraries) } } } @@ -302,7 +287,7 @@ class JarPackager private constructor( computeSourcesForModule( item = ModuleItem( moduleName = moduleName, - // relative path with `/` is always packed by dev-mode, so, we don't need to fix resolving for now and can imporove it later + // relative path with `/` is always packed by dev-mode, so, we don't need to fix resolving for now and can improve it later relativeOutputFile = if (descriptor.getAttributeValue("package") == null) "modules/$moduleName.jar" else layout.getMainJarName(), reason = "<- ${layout.mainModule} (plugin content)", ), @@ -426,7 +411,7 @@ class JarPackager private constructor( item: ModuleItem, layout: BaseLayout, module: JpsModule, - copiedFiles: MutableMap, + copiedFiles: MutableMap, asset: AssetDescriptor, ) { val moduleName = module.name @@ -441,7 +426,7 @@ class JarPackager private constructor( if (includeProjectLib) { if (platformLayout!!.hasLibrary(libName) || layout.hasLibrary(libName)) { //if (item.reason == ModuleIncludeReasons.PRODUCT_MODULES) { - // Span.current().addEvent("$libName is not included into module $moduleName as explicitly included into platform layout") + // Span.current().addEvent("$libName is not included into module $moduleName as explicitly included in platform layout") //} continue } @@ -485,7 +470,7 @@ class JarPackager private constructor( } val targetFile = outDir.resolve(item.relativeOutputFile) - val files = getLibraryFiles(library = library, copiedFiles = copiedFiles, isModuleLevel = true, targetFile = targetFile) + val files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile) for (i in (files.size - 1) downTo 0) { val file = files.get(i) val fileName = file.fileName.toString() @@ -552,7 +537,7 @@ class JarPackager private constructor( library = library, targetFile = targetFile, relativeOutputFile = relativePath, - files = getLibraryFiles(library = library, copiedFiles = copiedFiles, isModuleLevel = true, targetFile = targetFile) + files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile) ) } else { @@ -569,7 +554,7 @@ class JarPackager private constructor( library = library, targetFile = targetFile, relativeOutputFile = relativePath, - files = getLibraryFiles(library = library, copiedFiles = copiedFiles, isModuleLevel = true, targetFile = targetFile) + files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile) ) } } @@ -611,7 +596,7 @@ class JarPackager private constructor( private fun computeProjectLibrariesSources( outDir: Path, layout: BaseLayout, - copiedFiles: MutableMap, + copiedFiles: MutableMap, clientModuleFilter: JetBrainsClientModuleFilter ): MutableMap> { if (layout.includedProjectLibraries.isEmpty()) { @@ -626,24 +611,22 @@ class JarPackager private constructor( libToMetadata.put(library, libraryData) val libName = library.name var packMode = libraryData.packMode - if (packMode == LibraryPackMode.MERGED && - !predefinedMergeRules.values.any { it(libName, clientModuleFilter) } && - !isLibraryMergeable(libName)) { + if (packMode == LibraryPackMode.MERGED && !predefinedMergeRules.values.any { it(libName, clientModuleFilter) } && !isLibraryMergeable(libName)) { packMode = LibraryPackMode.STANDALONE_MERGED } val outPath = libraryData.outPath - val files = getLibraryFiles(library = library, copiedFiles = copiedFiles, isModuleLevel = false, targetFile = null) if (packMode == LibraryPackMode.MERGED && outPath == null) { - toMerge.put(library, files) + toMerge.put(library, getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = null)) } else { var libOutputDir = outDir if (outPath != null) { if (outPath.endsWith(".jar")) { + val targetFile = outDir.resolve(outPath) filesToSourceWithMapping( - asset = getJarAsset(targetFile = outDir.resolve(outPath), relativeOutputFile = outPath, metaInfDir = null), - files = files, + asset = getJarAsset(targetFile = targetFile, relativeOutputFile = outPath, metaInfDir = null), + files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile), library = library, relativeOutputFile = outPath, ) @@ -659,15 +642,16 @@ class JarPackager private constructor( library = library, targetFile = targetFile, relativeOutputFile = if (outDir == libOutputDir) "" else outDir.relativize(targetFile).invariantSeparatorsPathString, - files = files, + files = getLibraryFiles(library = library, copiedFiles = copiedFiles, targetFile = targetFile), ) } else { - for (file in files) { + for (file in library.getPaths(JpsOrderRootType.COMPILED)) { var fileName = file.fileName.toString() if (packMode == LibraryPackMode.STANDALONE_SEPARATE_WITHOUT_VERSION_NAME) { fileName = removeVersionFromJar(fileName) } + val targetFile = libOutputDir.resolve(fileName) addLibrary( library = library, @@ -794,43 +778,20 @@ private fun removeVersionFromJar(fileName: String): String { return if (matcher.matches()) "${matcher.group(1)}.jar" else fileName } -private fun getLibraryFiles( - library: JpsLibrary, - copiedFiles: MutableMap, - isModuleLevel: Boolean, - targetFile: Path? -): MutableList { +private fun getLibraryFiles(library: JpsLibrary, copiedFiles: MutableMap, targetFile: Path?): MutableList { val files = library.getPaths(JpsOrderRootType.COMPILED) val libName = library.name if (libName == "ktor-client-jvm") { return files } - // allow duplication if packed into the same target file and have the same common prefix - files.removeIf { - val alreadyCopiedFor = copiedFiles.get(it) ?: return@removeIf false - val alreadyCopiedLibraryName = alreadyCopiedFor.library.name - - if (alreadyCopiedFor.library.name.startsWith("ktor-") && libName.startsWith("ktor-")) { - return@removeIf true - } - - alreadyCopiedFor.targetFile == targetFile && - (alreadyCopiedLibraryName.startsWith("ktor-") || - alreadyCopiedLibraryName.startsWith("commons-") || - alreadyCopiedLibraryName.startsWith("ai.grazie.") || - (isModuleLevel && alreadyCopiedLibraryName == libName)) - } - - for (file in files) { - val alreadyCopiedFor = copiedFiles.putIfAbsent(file, CopiedFor(library, targetFile)) - if (alreadyCopiedFor != null) { - // check name - we allow having the same named module level library name - if (isModuleLevel && alreadyCopiedFor.library.name == libName) { - continue - } - - throw IllegalStateException("File $file from $libName is already provided by ${alreadyCopiedFor.library.name} library") + val iterator = files.iterator() + while (iterator.hasNext()) { + val file = iterator.next() + // allow duplication if packed into the same target file + val alreadyCopiedFor = copiedFiles.putIfAbsent(CopiedForKey(file, targetFile), CopiedFor(library, targetFile)) ?: continue + if (alreadyCopiedFor.targetFile == targetFile) { + iterator.remove() } } return files @@ -871,6 +832,8 @@ internal val commonModuleExcludes: List = FileSystems.getDefault(). ) } +// null targetFile means main jar +private data class CopiedForKey(@JvmField val file: Path, @JvmField val targetFile: Path?) private data class CopiedFor(@JvmField val library: JpsLibrary, @JvmField val targetFile: Path?) private suspend fun buildJars( diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.kt index ec7e6e348259..815f1ae9cef8 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/projectStructureMapping/ProjectStructureMapping.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("LiftReturnOrAssignment") package org.jetbrains.intellij.build.impl.projectStructureMapping @@ -160,28 +160,28 @@ private fun writeModuleItem(writer: JsonGenerator, entry: ModuleOutputEntry) { } } -private fun writeModuleLibraries(fileEntries: List, - moduleName: String, - writer: JsonGenerator, - buildPaths: BuildPaths) { - var opened = false - for (entry in fileEntries) { - if (entry !is ModuleLibraryFileEntry || entry.moduleName != moduleName) { - continue - } +private fun writeModuleLibraries(fileEntries: List, moduleName: String, writer: JsonGenerator, buildPaths: BuildPaths) { + val entriesGroupedByLibraryName = fileEntries.asSequence() + .filterIsInstance() + .filter { it.moduleName == moduleName } + .groupBy { it.libraryName } - if (!opened) { - writer.writeArrayFieldStart("libraries") - opened = true - } - writer.writeStartObject() - writer.writeStringField("name", shortenAndNormalizePath(entry.libraryFile!!, buildPaths)) - writer.writeNumberField("size", entry.size) - writer.writeEndObject() + if (entriesGroupedByLibraryName.isEmpty()) { + return } - if (opened) { + + writer.writeObjectFieldStart("libraries") + for ((libName, entries) in entriesGroupedByLibraryName) { + writer.writeArrayFieldStart(libName) + for (entry in entries) { + writer.writeStartObject() + writer.writeStringField("name", shortenAndNormalizePath(entry.libraryFile!!, buildPaths)) + writer.writeNumberField("size", entry.size) + writer.writeEndObject() + } writer.writeEndArray() } + writer.writeEndObject() } private fun writeProjectLibs(entries: List, writer: JsonGenerator, buildPaths: BuildPaths) { diff --git a/spellchecker/intellij.spellchecker.iml b/spellchecker/intellij.spellchecker.iml index 1a849c6ac9dd..9602b417acf2 100644 --- a/spellchecker/intellij.spellchecker.iml +++ b/spellchecker/intellij.spellchecker.iml @@ -23,163 +23,166 @@ - + bee241c073eaf57297542a6334923772893f4ab45fdb68bfbaf136c5fd788478 - - - - - - - - - - - - - - - 322b44a98febb80df044420f5f16c99740a681def786992a9d4bccc3e51ecb3a - - - - - - - - - - - - - - - + + f778502ea8b24033e3894b4a94640529fad266406eb649ed700d874e80125470 + + + aa11dfbff5186775158ce2323b7da6744fd2fa586bf35c8b995819a44dfb629b + + + 4aff241f00efbe7c17960a3a3efb1ba046586a562b53a25a68d2964fafff48d7 + + + c9cc99691881cd236ee357dc55182831a93b29e642e216c056ca174252c17905 + + + 0a47e60164535dcb2d6d0c4806a2174b6032e37f42925694c82fbfe6674b1ae2 + 4ebe7800b5262105b16ee626dfd33186eb9fc29d73eeb23aaf3b0cc22dc5687c - - - - - - - - - - - - - - - + + 88ba4475ec92a7f1a40a44dc6f631e4426952f5f943174edcb7be10b9e773166 + + + 8c07f392ec82b0af5b8a65f641e97c8517d1fc813cf2b8cac168742bd6da8392 + + + 7d21779a19dc3ccb037a1a5b85543162e358dbc656bea4926703315fdc125315 + + + 4cbf17f5c7049077cc1b8f885cb332b8d2271c7d3cd71d1501cda3e9328936d6 + 7ed984ae133ad476a3b632bb02f6a4724a2aa0106f8cfeff592775ecaffd215c + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - 0a47e60164535dcb2d6d0c4806a2174b6032e37f42925694c82fbfe6674b1ae2 - - - - - - - - - - - - - - - - - - 7d21779a19dc3ccb037a1a5b85543162e358dbc656bea4926703315fdc125315 - - - - - - - - - - - - - - - - - - 8c07f392ec82b0af5b8a65f641e97c8517d1fc813cf2b8cac168742bd6da8392 - - - - - - - - - - - - - - - - - - 4cbf17f5c7049077cc1b8f885cb332b8d2271c7d3cd71d1501cda3e9328936d6 - - - - - - - - - - - - - + 55d4a76b741ff08d67e8cfe3dcabf84203c357e6d03b0c47c6bab894a45403a4 + + 322b44a98febb80df044420f5f16c99740a681def786992a9d4bccc3e51ecb3a + + + c9cc99691881cd236ee357dc55182831a93b29e642e216c056ca174252c17905 + + + 4ebe7800b5262105b16ee626dfd33186eb9fc29d73eeb23aaf3b0cc22dc5687c + + + aa11dfbff5186775158ce2323b7da6744fd2fa586bf35c8b995819a44dfb629b + + + 4aff241f00efbe7c17960a3a3efb1ba046586a562b53a25a68d2964fafff48d7 + + + 8c07f392ec82b0af5b8a65f641e97c8517d1fc813cf2b8cac168742bd6da8392 + + + 7ed984ae133ad476a3b632bb02f6a4724a2aa0106f8cfeff592775ecaffd215c + + + 7d21779a19dc3ccb037a1a5b85543162e358dbc656bea4926703315fdc125315 + + + 0a47e60164535dcb2d6d0c4806a2174b6032e37f42925694c82fbfe6674b1ae2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -201,78 +204,6 @@ - - - - - - 88ba4475ec92a7f1a40a44dc6f631e4426952f5f943174edcb7be10b9e773166 - - - - - - - - - - - - - - - - - - f778502ea8b24033e3894b4a94640529fad266406eb649ed700d874e80125470 - - - - - - - - - - - - - - - - - - aa11dfbff5186775158ce2323b7da6744fd2fa586bf35c8b995819a44dfb629b - - - - - - - - - - - - - - - - - - 4aff241f00efbe7c17960a3a3efb1ba046586a562b53a25a68d2964fafff48d7 - - - - - - - - - - - -