From e609b67cfd12931580536dc8764ac5ebf8b397c7 Mon Sep 17 00:00:00 2001 From: Titouan Bion Date: Wed, 8 Oct 2025 18:52:09 +0200 Subject: [PATCH] [fleet] Move `fleet.bundles` to Community (cherry picked from commit c96573a42577eb7c277816db07337ca0183f369b) FLEET-MR-6894 GitOrigin-RevId: 2361ae5b2e6bd1b4c16c6f9a94fffef0f5d1f687 --- .idea/modules.xml | 1 + build/bazel-generated-file-list.txt | 1 + fleet/bundles/BUILD.bazel | 33 ++ fleet/bundles/fleet.bundles.iml | 42 ++ fleet/bundles/gradlebuild/build.gradle.kts | 73 ++++ .../.pseudoCommonKotlinSourceSet | 0 .../fleet/bundles/CoordinatesPlatform.kt | 38 ++ .../fleet/bundles/CoordinatesResolution.kt | 46 +++ .../fleet/bundles/LayerDependencies.kt | 140 +++++++ .../fleet/bundles/PluginRepository.kt | 105 +++++ .../fleet/bundles/PluginResolution.kt | 354 ++++++++++++++++ .../fleet/bundles/PluginSerialization.kt | 252 ++++++++++++ .../fleet/bundles/PluginUtils.kt | 53 +++ .../srcCommonMain/fleet/bundles/Plugins.kt | 389 ++++++++++++++++++ .../bundles/CoordinatesResolution.jvm.kt | 55 +++ 15 files changed, 1582 insertions(+) create mode 100644 fleet/bundles/BUILD.bazel create mode 100644 fleet/bundles/fleet.bundles.iml create mode 100644 fleet/bundles/gradlebuild/build.gradle.kts create mode 100644 fleet/bundles/srcCommonMain/.pseudoCommonKotlinSourceSet create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesPlatform.kt create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesResolution.kt create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/LayerDependencies.kt create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/PluginRepository.kt create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/PluginResolution.kt create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/PluginSerialization.kt create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/PluginUtils.kt create mode 100644 fleet/bundles/srcCommonMain/fleet/bundles/Plugins.kt create mode 100644 fleet/bundles/srcJvmMain/fleet/bundles/CoordinatesResolution.jvm.kt diff --git a/.idea/modules.xml b/.idea/modules.xml index c65b3e097cd2..bdb749bc865e 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -4,6 +4,7 @@ + diff --git a/build/bazel-generated-file-list.txt b/build/bazel-generated-file-list.txt index 687ec420ad92..32b9ef8830d5 100644 --- a/build/bazel-generated-file-list.txt +++ b/build/bazel-generated-file-list.txt @@ -221,6 +221,7 @@ commandInterface community-resources fleet/andel fleet/bifurcan +fleet/bundles fleet/compiler-plugins fleet/fastutil fleet/junit4 diff --git a/fleet/bundles/BUILD.bazel b/fleet/bundles/BUILD.bazel new file mode 100644 index 000000000000..4446abf41383 --- /dev/null +++ b/fleet/bundles/BUILD.bazel @@ -0,0 +1,33 @@ +### auto-generated section `build fleet.bundles` start +load("//build:compiler-options.bzl", "create_kotlinc_options") +load("@rules_jvm//:jvm.bzl", "jvm_library") + +create_kotlinc_options( + name = "custom_bundles", + opt_in = [ + "kotlinx.coroutines.ExperimentalCoroutinesApi", + "kotlin.ExperimentalStdlibApi", + ], + x_consistent_data_class_copy_visibility = True, + x_context_parameters = True, + x_jvm_default = "all-compatibility", + x_lambdas = "class" +) + +jvm_library( + name = "bundles", + module_name = "fleet.bundles", + visibility = ["//visibility:public"], + srcs = glob(["srcCommonMain/**/*.kt", "srcCommonMain/**/*.java", "srcCommonMain/**/*.form", "srcJvmMain/**/*.kt", "srcJvmMain/**/*.java", "srcJvmMain/**/*.form"], allow_empty = True, exclude = ["**/module-info.java"]), + kotlinc_opts = ":custom_bundles", + deps = [ + "@lib//:kotlin-stdlib", + "@lib//:kotlinx-serialization-core", + "@lib//:kotlinx-serialization-json", + "@lib//:kotlinx-coroutines-core", + "//fleet/util/core", + "//fleet/modules/api", + "//fleet/bifurcan", + ] +) +### auto-generated section `build fleet.bundles` end \ No newline at end of file diff --git a/fleet/bundles/fleet.bundles.iml b/fleet/bundles/fleet.bundles.iml new file mode 100644 index 000000000000..c79d06fa1679 --- /dev/null +++ b/fleet/bundles/fleet.bundles.iml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + $KOTLIN_BUNDLED$/lib/kotlinx-serialization-compiler-plugin.jar + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/fleet/bundles/gradlebuild/build.gradle.kts b/fleet/bundles/gradlebuild/build.gradle.kts new file mode 100644 index 000000000000..75698d939bcf --- /dev/null +++ b/fleet/bundles/gradlebuild/build.gradle.kts @@ -0,0 +1,73 @@ +// IMPORT__MARKER_START +import fleet.buildtool.conventions.configureAtMostOneJvmTargetOrThrow +import fleet.buildtool.conventions.withJavaSourceSet +// IMPORT__MARKER_END +plugins { + alias(libs.plugins.kotlin.multiplatform) + id("fleet.project-module-conventions") + id("fleet.toolchain-conventions") + id("fleet.module-publishing-conventions") + id("fleet.sdk-repositories-publishing-conventions") + id("fleet.open-source-module-conventions") + alias(libs.plugins.dokka) + // GRADLE_PLUGINS__MARKER_START + id("fleet-module") + alias(jps.plugins.kotlin.serialization) + // GRADLE_PLUGINS__MARKER_END +} + +fleetModule { + module { + name = "fleet.bundles" + importedFromJps {} + } +} + +@OptIn(org.jetbrains.kotlin.gradle.ExperimentalWasmDsl::class) +kotlin { + // KOTLIN__MARKER_START + compilerOptions.freeCompilerArgs = listOf( + "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi", + "-opt-in=kotlin.ExperimentalStdlibApi", + "-Xlambdas=class", + "-Xconsistent-data-class-copy-visibility", + "-Xcontext-parameters", + "-XXLanguage:+AllowEagerSupertypeAccessibilityChecks", + ) + jvm {} + wasmJs { + browser {} + } + sourceSets.commonMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcCommonMain")) } + sourceSets.commonMain.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesCommonMain")) } + sourceSets.commonTest.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcCommonTest")) } + sourceSets.commonTest.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesCommonTest")) } + sourceSets.jvmMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcJvmMain")) } + configureAtMostOneJvmTargetOrThrow { compilations.named("main") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../srcJvmMain")) } } } + sourceSets.jvmMain.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesJvmMain")) } + sourceSets.jvmTest.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcJvmTest")) } + configureAtMostOneJvmTargetOrThrow { compilations.named("test") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../srcJvmTest")) } } } + sourceSets.jvmTest.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesJvmTest")) } + sourceSets.wasmJsMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcWasmJsMain")) } + sourceSets.wasmJsMain.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesWasmJsMain")) } + sourceSets.wasmJsTest.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcWasmJsTest")) } + sourceSets.wasmJsTest.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesWasmJsTest")) } + sourceSets.commonMain.dependencies { + implementation(jps.org.jetbrains.kotlin.kotlin.stdlib1993400674.get().let { "${it.group}:${it.name}:${it.version}" }) { + exclude(group = "org.jetbrains", module = "annotations") + } + implementation(jps.org.jetbrains.kotlinx.kotlinx.serialization.core.jvm1739247612.get().let { "${it.group}:kotlinx-serialization-core:${it.version}" }) { + isTransitive = false + } + implementation(jps.org.jetbrains.kotlinx.kotlinx.serialization.json.jvm231489733.get().let { "${it.group}:kotlinx-serialization-json:${it.version}" }) { + isTransitive = false + } + implementation(jps.com.intellij.platform.kotlinx.coroutines.core.jvm134738847.get().let { "${it.group}:kotlinx-coroutines-core:${it.version}" }) { + isTransitive = false + } + implementation(project(":fleet.util.core")) + implementation(project(":fleet.modules.api")) + implementation(project(":fleet.bifurcan")) + } + // KOTLIN__MARKER_END +} \ No newline at end of file diff --git a/fleet/bundles/srcCommonMain/.pseudoCommonKotlinSourceSet b/fleet/bundles/srcCommonMain/.pseudoCommonKotlinSourceSet new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesPlatform.kt b/fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesPlatform.kt new file mode 100644 index 000000000000..594d0ad11d77 --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesPlatform.kt @@ -0,0 +1,38 @@ +package fleet.bundles + +import fleet.util.Os +import kotlinx.serialization.Serializable +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.json.Json + +@Serializable +enum class CoordinatesPlatform { + WindowsX64, + WindowsAarch64, + LinuxX64, + LinuxAarch64, + MacOsX64, + MacOsAarch64, + Wasm +} + +val Coordinates.platforms: List? + get() = meta[KnownCoordinatesMeta.Platforms]?.let { + runCatching { + Json.decodeFromString(ListSerializer(CoordinatesPlatform.serializer()), it) + }.getOrDefault(emptyList()) + } + +private val currentPlatform by lazy { + if (Os.INSTANCE.isWasm) CoordinatesPlatform.Wasm + else when (Os.INSTANCE.type) { + Os.Type.Windows -> if (Os.INSTANCE.isAarch64) CoordinatesPlatform.WindowsAarch64 else CoordinatesPlatform.WindowsX64 + Os.Type.Linux -> if (Os.INSTANCE.isAarch64) CoordinatesPlatform.LinuxAarch64 else CoordinatesPlatform.LinuxX64 + Os.Type.MacOS -> if (Os.INSTANCE.isAarch64) CoordinatesPlatform.MacOsAarch64 else CoordinatesPlatform.MacOsX64 + Os.Type.Unknown -> null + } +} + +fun Iterable.filterCoordinatesByPlatform(): List = filter { coordinates -> + coordinates.platforms?.contains(currentPlatform) != false +} diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesResolution.kt b/fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesResolution.kt new file mode 100644 index 000000000000..7688dd93561c --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/CoordinatesResolution.kt @@ -0,0 +1,46 @@ +package fleet.bundles + +import kotlin.jvm.JvmInline + +class ResolutionException(coordinates: Coordinates, cause: Throwable? = null) : Exception("Can't resolve $coordinates", cause) + +/** + * Responsible for downloading jars, checking their hashes, and caching + * @throws ResolutionException + */ +interface CoordinatesResolution { + suspend fun resolveFile(coordinates: Coordinates): ResolvedFile + suspend fun resolvePluginPartsCoordinates(coordinates: Coordinates): PluginParts + suspend fun resolveModule(coordinates: ModuleCoordinates): ModuleOnDisc + suspend fun resolveResource(coordinates: Coordinates): ResourceBundle +} + +@JvmInline +value class ResolvedFile(val path: String) + +suspend fun CoordinatesResolution.resolve(layer: PluginLayer): ResolvedPluginLayer = + ResolvedPluginLayer(modules = layer.modules, + modulePath = layer.modulePath.map { moduleCoords -> + resolveModule(moduleCoords) + }.toSet(), + resources = layer.resources.filterCoordinatesByPlatform().map { + resolveResource(it) + }.toSet()) + +suspend fun CoordinatesResolution.resolveParts(descriptor: PluginDescriptor): PluginParts { + val coordinates = requireNotNull(descriptor.partsCoordinates) { + "missing parts coordinates in descriptor: $descriptor" + } + return resolvePluginPartsCoordinates(coordinates) +} + +data class ResolvedPluginLayer(val modulePath: Set, + val modules: Set, + val resources: Set) + +data class ModuleOnDisc(val path: String, + val serializedModuleDescriptor: String?) + +fun interface ResourceBundle { + operator fun get(key: String): ByteArray? +} \ No newline at end of file diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/LayerDependencies.kt b/fleet/bundles/srcCommonMain/fleet/bundles/LayerDependencies.kt new file mode 100644 index 000000000000..f6fe2bfb2959 --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/LayerDependencies.kt @@ -0,0 +1,140 @@ +package fleet.bundles + +private fun deps(vararg pairs: Pair): Map> = + pairs.groupBy(keySelector = { (dependant, _) -> dependant }, valueTransform = { (_, dependency) -> dependency }) + +val dockLayer = LayerSelector("dock") +val frontendApiLayer = LayerSelector("frontendApi") +val frontendImplLayer = LayerSelector("frontendImpl") +@Deprecated(message = "use `frontendImpl` instead", replaceWith = ReplaceWith("frontendImplLayer")) +val frontendLayer = LayerSelector("frontend") +val workspaceApiLayer = LayerSelector("workspaceApi") +val workspaceImplLayer = LayerSelector("workspaceImpl") +@Deprecated(message = "use `workspaceImpl` instead", replaceWith = ReplaceWith("workspaceImpl")) +val workspaceLayer = LayerSelector("workspace") +val commonApiLayer = LayerSelector("commonApi") +val commonImplLayer = LayerSelector("commonImpl") +@Deprecated(message = "use `commonImpl` instead", replaceWith = ReplaceWith("commonImpl")) +val commonLayer = LayerSelector("common") +val testLayer = LayerSelector("test") + +@Suppress("DEPRECATION") +val deprecatedLayers = listOf( + frontendLayer, + commonLayer, + workspaceLayer, +) + +@Suppress("DEPRECATION") +//this describes dependencies between layers inside a plugin +val internalReadability: Map> = deps( + frontendImplLayer to frontendApiLayer, + frontendImplLayer to commonImplLayer, + frontendImplLayer to commonApiLayer, + frontendImplLayer to dockLayer, + frontendApiLayer to commonApiLayer, + workspaceImplLayer to workspaceApiLayer, + workspaceImplLayer to commonApiLayer, + workspaceImplLayer to commonImplLayer, + workspaceApiLayer to commonApiLayer, + frontendLayer to commonLayer, + frontendLayer to dockLayer, + workspaceLayer to commonLayer +) + +@Suppress("DEPRECATION") +//this describes dependencies between plugins +val externalReadability: Map> = deps( + frontendImplLayer to frontendApiLayer, + frontendImplLayer to commonApiLayer, + frontendApiLayer to frontendApiLayer, + frontendApiLayer to commonApiLayer, + + workspaceImplLayer to workspaceApiLayer, + workspaceImplLayer to commonApiLayer, + workspaceApiLayer to workspaceApiLayer, + workspaceApiLayer to commonApiLayer, + + commonApiLayer to commonApiLayer, + commonImplLayer to commonApiLayer, + + //working with old plugins (to be removed) + workspaceApiLayer to workspaceLayer, + workspaceImplLayer to workspaceLayer, + workspaceLayer to workspaceApiLayer, + + workspaceLayer to workspaceLayer, + frontendApiLayer to frontendLayer, + frontendImplLayer to frontendLayer, + frontendLayer to frontendApiLayer, + frontendLayer to frontendLayer, + frontendLayer to commonLayer, + frontendLayer to dockLayer, + commonLayer to dockLayer, + workspaceLayer to commonLayer, + commonLayer to commonLayer, + frontendLayer to commonApiLayer, + workspaceLayer to commonApiLayer, + commonLayer to commonApiLayer, + commonApiLayer to commonLayer, + commonImplLayer to commonLayer, + dockLayer to dockLayer, +) + +@Suppress("DEPRECATION") +val frontendLayers: Set = setOf( + dockLayer, + frontendApiLayer, + frontendImplLayer, + commonApiLayer, + commonImplLayer, + frontendLayer, + commonLayer, +) + +@Suppress("DEPRECATION") +val workspaceLayers: Set = setOf( + dockLayer, + workspaceApiLayer, + workspaceImplLayer, + commonApiLayer, + commonImplLayer, + workspaceLayer, + commonLayer, +) + +fun Collection.sortByInternalDependencies(): List { + val currentStack = mutableListOf() + + fun logCycle(): Nothing { + currentStack.reverse() + val endCycleIndex = currentStack.lastIndexOf(currentStack.first()) + throw IllegalArgumentException( + "Cyclic dependency: [" + + currentStack.subList(0, endCycleIndex + 1).joinToString(separator = " <- ") { it.selector } + + "] <- " + + currentStack.subList(endCycleIndex + 1, currentStack.size).joinToString(separator = " <- ") { it.selector }) + } + + val times = HashMap() + var timestamp = 0 + fun dfs(selector: LayerSelector) { + when (times[selector]) { + null -> { + currentStack.add(selector) + times[selector] = -1 + internalReadability[selector]?.forEach { dependency -> + dfs(dependency) + } + times[selector] = timestamp++ + currentStack.removeLast() + } + -1 -> { + currentStack.add(selector) + logCycle() + } + } + } + forEach { dfs(it) } + return times.entries.sortedBy { it.value }.map { it.key } +} diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/PluginRepository.kt b/fleet/bundles/srcCommonMain/fleet/bundles/PluginRepository.kt new file mode 100644 index 000000000000..1c911cba866b --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/PluginRepository.kt @@ -0,0 +1,105 @@ +package fleet.bundles + +/** + * Could be backed by Marketplace, or Maven repository + * Subject for trust?? + **/ +interface PluginRepository { + companion object { + val Empty = object : PluginRepository { + override suspend fun getLatestVersions(names: Set, shipVersion: PluginVersion): Map = emptyMap() + override suspend fun getPlugin(pluginName: PluginName, pluginVersion: PluginVersion): PluginDescriptor? = null + override fun presentableName(): String = "Empty PluginRepository" + override fun cacheKey(): String = "empty" // we should not attempt to re-resolve against the Empty repository if we already have a resolution result against it, it will lead to the same result + } + } + + /** + * Should check signature of [fleet.bundles.PluginDescriptor] before returning it + **/ + suspend fun getLatestVersions(names: Set, shipVersion: PluginVersion): Map + suspend fun getPlugin(pluginName: PluginName, pluginVersion: PluginVersion): PluginDescriptor? + + fun presentableName() : String = toString() + + /** + * Cache invalidation looks at this identifier to know whether a resolution against that repository is stale or not. + */ + fun cacheKey() : String +} + +fun PluginSet.asPluginRepository(): PluginRepository { + return object : PluginRepository { + override suspend fun getLatestVersions(names: Set, shipVersion: PluginVersion): Map = + plugins.groupBy(PluginDescriptor::name).mapNotNull { (name, bs) -> + bs.filter { + it.compatibleShipVersionRange?.let { range -> + shipVersion in range.from..range.to + } ?: false + } + .maxByOrNull { it.version } + ?.let { it.name to it.version } + }.toMap() + + override suspend fun getPlugin(pluginName: PluginName, pluginVersion: PluginVersion): PluginDescriptor? = + plugins.find { spec -> spec.name == pluginName && spec.version == pluginVersion } + + override fun presentableName(): String = "Plugin Repository from plugins set with ${this@asPluginRepository.plugins.size} plugins" + override fun cacheKey(): String = buildString { + shipVersions.sorted().forEach { shipVersion -> + append(shipVersion) + } + plugins.sortedBy { descriptor -> descriptor.name.name }.forEach { descriptor -> + append(descriptor.name.name) + append(descriptor.version.versionString) + descriptor.deps.entries.sortedBy { (name, _) -> name.name }.forEach { (name, version) -> + append(name.name) + append(version.version.versionString) + } + } + } + } +} + +fun PluginRepository.compose(other: PluginRepository): PluginRepository { + val self = this + return object : PluginRepository { + override suspend fun getLatestVersions(names: Set, shipVersion: PluginVersion): Map = + self.getLatestVersions(names, shipVersion).merge(other.getLatestVersions(names, shipVersion)) { _, v1, v2 -> + max(v1, v2) + } + + override suspend fun getPlugin(pluginName: PluginName, pluginVersion: PluginVersion): PluginDescriptor? { + //TODO[jetzajac]: ambigs? + return other.getPlugin(pluginName, pluginVersion) ?: self.getPlugin(pluginName, pluginVersion) + } + + override fun presentableName(): String { + return "Composed PluginRepository: ${self.presentableName()} -> ${other.presentableName()}" + } + + override fun cacheKey(): String = "${self.cacheKey()}-${other.cacheKey()}" + } +} + +private fun Map.merge(other: Map, f: (K, V, V) -> V = {_, _, v -> v}): Map { + val x = toMutableMap() + other.forEach { (k, v) -> + when { + x.containsKey(k) -> { + x[k] = f(k, x[k]!! as V, v) + } + else -> x[k] = v + } + } + return x +} + +private fun > max(c1: T, c2: T): T { + val x = c1.compareTo(c2) + return when { + x == 0 -> c1 + x < 0 -> c2 + else -> c1 + } +} diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/PluginResolution.kt b/fleet/bundles/srcCommonMain/fleet/bundles/PluginResolution.kt new file mode 100644 index 000000000000..1e2228dcb028 --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/PluginResolution.kt @@ -0,0 +1,354 @@ +package fleet.bundles + +import fleet.util.async.catching +import fleet.util.letIf +import fleet.util.logging.logger +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +@Serializable +sealed class PluginCommand { + @Serializable + @SerialName("add") + data class Add(val pluginName: PluginName, val exactVersion: PluginVersion? = null) : PluginCommand() + + @SerialName("remove") + @Serializable + data class Remove(val pluginName: PluginName) : PluginCommand() +} + +data class RequestedPluginConfiguration( + val requestedVersions: Map, + val forbidInstallation: Set +) { + companion object { + val EMPTY = RequestedPluginConfiguration(emptyMap(), emptySet()) + } +} + +private val log by lazy { logger() } + +/** + * collected from settings + * */ +data class PluginsConfig(val commands: List) { + companion object +} + +data class PluginResolutionResult(val config: ResolvedPluginsConfig, + val problems: List) { + sealed class Problem { + data class PluginNotFound(val name: PluginName) : Problem() + data class Conflict(val pluginName: PluginName, + val dependencyName: PluginName, + val takenVersion: PluginVersion, + val requiredVersion: VersionRequirement) : Problem() + data class FetchIssue(val pluginName: PluginName, + val throwable: Throwable) : Problem() + } +} + +private data class FrontResolutionArguments( + val alreadyResolvedPlugins: Map, + val front: Map +) + +private data class FrontResolutionResult( + val resolvedPlugins: Map, + val nextFront: Map, + val problemsFound: List +) + +private typealias FrontResolution = suspend (FrontResolutionArguments) -> FrontResolutionResult + +/** + * Resolution algorithm. Resolves target plugins layer by layer using provided `frontResolution` lambda. + * @param frontResolution lambda to resolve the current layer and define the next one. + * @param concretedPlugins additional fixed plugin set used as a base layer to satisfy dependencies. + * @return `PluginResolutionResult` with all the plugins which dependencies were satisfied + */ +private suspend fun resolvePluginsConfig( + pluginsConfig: PluginsConfig, + shipVersion: PluginVersion, + repo: PluginRepository, + concretedPlugins: Map = emptyMap(), + frontResolution: FrontResolution +): PluginResolutionResult { + val problems = ArrayList() + val requestedPluginConfiguration = calculateRequestedPlugins(pluginsConfig) + val requestedPlugins = requestedPluginConfiguration.requestedVersions + val forbidInstallation = requestedPluginConfiguration.forbidInstallation + val pluginsWithVersionRequirement = requestedPlugins.mapNotNullTo(HashSet()) { (name, version) -> + if (version != null) name else null + } + val pluginsWithoutVersionRequirement = requestedPlugins.mapNotNullTo(HashSet()) { (name, version) -> + if (version == null) name else null + } + val latestVersions = repo.getLatestVersions(pluginsWithoutVersionRequirement, shipVersion) + + problems.addAll(pluginsWithoutVersionRequirement + .filter { name -> !latestVersions.containsKey(name) } + .map(PluginResolutionResult.Problem::PluginNotFound)) + + val requestedPluginsWithVersions = + (pluginsWithVersionRequirement.map { name -> + name to requestedPlugins[name]!! + } + + pluginsWithoutVersionRequirement.mapNotNull { name -> + latestVersions[name]?.let { version -> name to version } + }).toMap() + + val pluginsToLoad = run { + val resolvedPlugins = hashMapOf() + var front = requestedPluginsWithVersions + while (front.isNotEmpty()) { + front = front.filterNot { forbidInstallation.contains(it.key) } // the whole subtree will be dropped, no reason to resolve it + val arguments = FrontResolutionArguments(resolvedPlugins, front) + val result = frontResolution(arguments) + resolvedPlugins.putAll(result.resolvedPlugins) + problems.addAll(result.problemsFound) + front = result.nextFront + } + filterMissingPluginsOut(resolvedPlugins, concretedPlugins, requestedPluginsWithVersions.keys) + } + + return PluginResolutionResult(ResolvedPluginsConfig(bundlesToLoad = pluginsToLoad.values.toSet(), + shipVersion = shipVersion), + problems = problems) +} + +/** + * Resolves plugins from provided `PluginsConfig` using the provided `PluginRepository`. + * + * @param ignoreFrontendOnly - to filter out frontend-only plugins, such as themes, keymaps or icon packs during resolution + * @return PluginResolutionResult with all the plugins which dependencies were satisfied + */ +suspend fun resolveWorkspacePlugins( + pluginsConfig: PluginsConfig, + shipVersion: PluginVersion, + repo: PluginRepository, + ignoreFrontendOnly: Boolean = true, +): PluginResolutionResult { + return resolvePluginsConfig(pluginsConfig, shipVersion, repo) { arguments -> + val problems = ArrayList() + val pluginsByNamePrime = repo.getPlugins(arguments.front).let { fetchingResult -> + problems.addAll(fetchingResult.problems) + fetchingResult.descriptors.letIf(ignoreFrontendOnly) { ds -> + ds.filterNot { it.value.meta[KnownMeta.FrontendOnly] == "true" } + } + } + val conflicts: Map> = pluginsByNamePrime + .flatMap { (pluginName, plugin) -> + plugin.deps.mapNotNull { (dependencyName, versionRequirement) -> + val satisfied = arguments.alreadyResolvedPlugins[dependencyName] + when { + satisfied == null -> dependencyName to versionRequirement.version + satisfied.version.satisfies(versionRequirement) -> null + else -> { + //TODO: try to satisfy existing dependencies with the required version??? + problems.add(PluginResolutionResult.Problem.Conflict(pluginName = pluginName, + dependencyName = dependencyName, + takenVersion = satisfied.version, + requiredVersion = versionRequirement)) + null + } + } + } + } + .groupBy(keySelector = { (name, _) -> name }, + valueTransform = { (_, requirement) -> requirement }) + + val nextDeps = conflicts.map { (name, requirements) -> + //TODO: handle conflicts better + name to requirements.max() + }.toMap() + + FrontResolutionResult(resolvedPlugins = pluginsByNamePrime, + nextFront = nextDeps, + problemsFound = problems) + } +} + + +/** + * Resolves frontend only plugins from provided config, using workspaceResolvedConfig as a base layer. + * + * @return only frontend-only plugins, which dependencies were satisfied by the workspaceResolvedConfig + */ +suspend fun resolveFrontendOnlyPlugins( + pluginsConfig: PluginsConfig, + shipVersion: PluginVersion, + repo: PluginRepository, + workspaceResolvedConfig: ResolvedPluginsConfig +): PluginResolutionResult { + val concretedBaseLayer = workspaceResolvedConfig.bundlesToLoad.associateBy { it.name } + return resolvePluginsConfig(pluginsConfig, shipVersion, repo, concretedBaseLayer) { arguments -> + val problems = ArrayList() + val pluginsByNamePrime = repo.getPlugins(arguments.front) + .let { fetchingResult -> + problems.addAll(fetchingResult.problems) + fetchingResult.descriptors + } + .filterValues { it.meta[KnownMeta.FrontendOnly] == "true" } + val conflicts: Map> = pluginsByNamePrime + .flatMap { (pluginName, plugin) -> + plugin.deps.mapNotNull { (dependencyName, versionRequirement) -> + val satisfiedByWorkspace = concretedBaseLayer[dependencyName] + val satisfied = arguments.alreadyResolvedPlugins[dependencyName] + val frontendOnly = plugin.meta[KnownMeta.FrontendOnly] == "true" + if (frontendOnly) when { + satisfied == null -> dependencyName to versionRequirement.version + satisfied.version.satisfies(versionRequirement) -> null + else -> { + // TODO: try to satisfy existing dependencies with the required version??? + problems.add(PluginResolutionResult.Problem.Conflict(pluginName = pluginName, + dependencyName = dependencyName, + takenVersion = satisfied.version, + requiredVersion = versionRequirement)) + null + } + } + else { + when { + satisfiedByWorkspace == null -> { + problems.add(PluginResolutionResult.Problem.PluginNotFound(pluginName)) + } + !satisfiedByWorkspace.version.satisfies(versionRequirement) -> { + problems.add(PluginResolutionResult.Problem.Conflict(pluginName = pluginName, + dependencyName = dependencyName, + takenVersion = satisfiedByWorkspace.version, + requiredVersion = versionRequirement)) + } + } + null + } + } + } + .groupBy(keySelector = { (name, _) -> name }, + valueTransform = { (_, requirement) -> requirement }) + + val nextDeps = conflicts.map { (name, requirements) -> + //TODO: handle conflicts better + name to requirements.max() + }.toMap() + + FrontResolutionResult(resolvedPlugins = pluginsByNamePrime, + nextFront = nextDeps, + problemsFound = problems) + } +} + +fun calculateRequestedPlugins(pluginsConfig: PluginsConfig): RequestedPluginConfiguration { + val versionsToInstall = mutableMapOf() + val forbidInstallation = mutableSetOf() + + pluginsConfig.commands.forEach { command -> + when (command) { + is PluginCommand.Add -> versionsToInstall[command.pluginName] = command.exactVersion + is PluginCommand.Remove -> { + versionsToInstall.remove(command.pluginName) + forbidInstallation.add(command.pluginName) + } + } + } + + return RequestedPluginConfiguration(versionsToInstall, forbidInstallation) +} + +private fun filterMissingPluginsOut(resolvedOnes: Map, + concretedPlugins: Map, + requestedPlugins: Set): Map { + fun walkSubtree(resolvedOnes: Map, + pluginName: PluginName): Map? = + if (concretedPlugins[pluginName] != null) { + emptyMap() + } + else { + resolvedOnes[pluginName]?.let { plugin -> + HashMap().let { res -> + res[pluginName] = plugin + plugin.deps.keys.forEach { dep -> + val subtree = walkSubtree(resolvedOnes, dep) + if (subtree == null) { + log.warn { "Filtering out plugin ${pluginName.name}, dependency ${dep.name} not found in resolved plugins" } + return null + } + else { + res.putAll(subtree) + } + } + res + } + } + } + return hashMapOf().apply { + requestedPlugins.mapNotNull { walkSubtree(resolvedOnes, it) }.forEach { m -> + putAll(m) + } + } +} + +@Serializable +data class ResolvedPluginsConfig(val bundlesToLoad: Set, + val shipVersion: PluginVersion) { + companion object { + val json = Json { prettyPrint = true } + } + + override fun toString(): String = json.encodeToString(serializer(), this) +} + +data class PluginFetchingResult( + val descriptors: Map, + val problems: List +) + +private suspend fun PluginRepository.getPlugins(plugins: Map): PluginFetchingResult = + coroutineScope { + val problems = mutableListOf() + val descriptors = plugins.map { (name, version) -> + name to async { + catching { getPlugin(name, version) } + } + }.mapNotNull { (name, deferred) -> + deferred.await().let { result -> + result.onFailure { e -> + problems.add(PluginResolutionResult.Problem.FetchIssue(name, e)) + }.getOrNull()?.let { + plugin -> name to plugin + } + } + }.toMap() + PluginFetchingResult(descriptors, problems) + } + +private fun PluginVersion.satisfies(requirement: VersionRequirement): Boolean { + return when (requirement) { + is VersionRequirement.Above -> this >= requirement.version + is VersionRequirement.CompatibleWith -> major == requirement.version.major && this >= requirement.version + } +} + +private fun PluginResolutionResult.Problem.presentable() = when (this) { + is PluginResolutionResult.Problem.Conflict -> "$pluginName - CONFLICT: ${dependencyName.name} requires ${requiredVersion.version}, but ${takenVersion.presentableText} was taken" + is PluginResolutionResult.Problem.FetchIssue -> "$pluginName - FETCH ISSUE" + is PluginResolutionResult.Problem.PluginNotFound -> "${name.name} - PLUGIN NOT FOUND" +} + +fun PluginResolutionResult.simplifiedPresentation(): String = buildString { + appendLine() + append("PROBLEMS:") + append(problems.joinToString(separator = "\n") { it.presentable() }) + appendLine() + append("SHIP version: ${config.shipVersion.presentableText}") + appendLine() + append("DESCRIPTORS:") + appendLine() + append(config.bundlesToLoad.joinToString(separator = "\n") { "${it.name.name} - ${it.version.presentableText}" }) + appendLine() + append("_".repeat(40)) + appendLine() +} \ No newline at end of file diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/PluginSerialization.kt b/fleet/bundles/srcCommonMain/fleet/bundles/PluginSerialization.kt new file mode 100644 index 000000000000..79830add2d91 --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/PluginSerialization.kt @@ -0,0 +1,252 @@ +package fleet.bundles + +import fleet.util.Base64WithOptionalPadding +import fleet.bifurcan.toSortedMap +import fleet.bifurcan.toSortedSet +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.json.Json +import kotlin.io.encoding.ExperimentalEncodingApi + +internal class VersionSerializer : StringSerializer( + toString = { + when (it) { + is VersionRequirement.CompatibleWith -> it.version.versionString + is VersionRequirement.Above -> it.version.versionString + "+" + } + }, + fromString = { + when { + it.endsWith("+") -> VersionRequirement.Above(PluginVersion.fromString(it.dropLast(1))) + else -> VersionRequirement.CompatibleWith(PluginVersion.fromString(it)) + } + } +) + +internal class PluginNameSerializer : StringSerializer(PluginName::name, ::PluginName) + +internal class PluginVersionSerializer : StringSerializer({ it.versionString }, { PluginVersion.fromString(it) }) + +/** + * Serialize a [PluginVersion] using the string representation compatible with Marketplace compatibility range fields of the plugin descriptor's JSON + */ +// TODO: remove this and its usages once Marketplace supports Nightly two digit versions in Fleet's compatibility ranges as well +internal class PluginVersionForCompatibilityRangeSerializer : StringSerializer({ it.marketplaceCompatibilityRangeVersionString }, { PluginVersion.fromString(it) }) + +internal class LayerSelectorSerializer : StringSerializer(LayerSelector::selector, ::LayerSelector) + +internal object PluginLayerSerializer : DataSerializer( + serializer = PluginLayerSurrogate.serializer(), + toData = PluginLayerSurrogate::fromPluginLayer, + fromData = PluginLayerSurrogate::toPluginLayer, +) + +/** + * Surrogate of [PluginLayer] ensuring sorted [Set] at construction to allow stability at serialization + */ +@Serializable +@SerialName("PluginLayer") +data class PluginLayerSurrogate( + private val modulePath: Set, + private val modules: Set, + private val resources: Set = emptySet(), +) { + fun toPluginLayer(): PluginLayer = PluginLayer( + modulePath = modulePath, + modules = modules, + resources = resources, + ) + + companion object { + fun fromPluginLayer(pl: PluginLayer): PluginLayerSurrogate = PluginLayerSurrogate( + modulePath = pl.modulePath.toSortedSet(compareBy(coordinatesComparator) { it.coordinates }), + modules = pl.modules.toSortedSet(), + resources = pl.resources.toSortedSet(coordinatesComparator) + ) + } +} + +private val coordinatesComparator = compareBy { + when (it) { + is Coordinates.Local -> it.path + is Coordinates.Remote -> it.url + } +} + +internal object PluginPartsSerializer : DataSerializer( + serializer = PluginPartsSurrogate.serializer(), + toData = PluginPartsSurrogate::fromPluginParts, + fromData = PluginPartsSurrogate::toPluginParts, +) + +/** + * Surrogate of [PluginParts] ensuring sorted [Map] at construction to allow stability at serialization + */ +@Serializable +@SerialName("PluginDescriptor") +internal data class PluginPartsSurrogate( + val layers: Map, +) { + fun toPluginParts(): PluginParts = PluginParts( + layers = layers + ) + + companion object { + fun fromPluginParts(pp: PluginParts): PluginPartsSurrogate = PluginPartsSurrogate( + layers = pp.layers.toSortedMap(compareBy { selector -> selector.selector }) + ) + } +} + +internal object PluginDescriptorSerializer : DataSerializer( + serializer = PluginDescriptorSurrogate.serializer(), + toData = PluginDescriptorSurrogate::fromPluginDescriptor, + fromData = PluginDescriptorSurrogate::toPluginDescriptor, +) + +/** + * Surrogate of [PluginDescriptor] ensuring sorted [Map] and [Set] at construction to allow stability at serialization + */ +@Serializable +@SerialName("PluginDescriptor") +internal data class PluginDescriptorSurrogate( + private val formatVersion: Int = 0, + @SerialName("id") + private val name: PluginName, + private val version: PluginVersion, + @SerialName("dependencies") + private val deps: Map = emptyMap(), + private val compatibleShipVersionRange: ShipVersionRange? = null, + private val signature: PluginSignature? = null, + private val meta: Map = emptyMap(), +) { + fun toPluginDescriptor(): PluginDescriptor = PluginDescriptor( + formatVersion = formatVersion, + name = name, + version = version, + deps = deps, + compatibleShipVersionRange = compatibleShipVersionRange, + signature = signature, + meta = meta, + ) + + companion object { + fun fromPluginDescriptor(pd: PluginDescriptor): PluginDescriptorSurrogate = PluginDescriptorSurrogate( + formatVersion = pd.formatVersion, + name = pd.name, + version = pd.version, + deps = pd.deps.toSortedMap(compareBy { pluginName -> pluginName.name }), + compatibleShipVersionRange = pd.compatibleShipVersionRange, + signature = pd.signature, + meta = pd.meta.toSortedMap(), + ) + } +} + +internal object PluginSetSerializer : DataSerializer( + serializer = PluginSetSurrogate.serializer(), + toData = PluginSetSurrogate::fromPluginSet, + fromData = PluginSetSurrogate::toPluginSet, +) + +/** + * Suroggate of [PluginSet] ensuring sorted [Set] at construction to allow stability at serialization + */ +@Serializable +@SerialName("PluginSet") +internal class PluginSetSurrogate( + private val shipVersions: Set, + private val plugins: Set, +) { + fun toPluginSet(): PluginSet = PluginSet( + shipVersions = shipVersions, + plugins = plugins, + ) + + companion object { + fun fromPluginSet(ps: PluginSet): PluginSetSurrogate = PluginSetSurrogate( + shipVersions = ps.shipVersions.toSortedSet(), + plugins = ps.plugins.toSortedSet(compareBy(PluginDescriptor::encodeToString)) + ) + } +} + +@OptIn(ExperimentalEncodingApi::class) +internal object PluginSignatureSerializer : DataSerializer( + serializer = String.serializer(), + toData = { value -> Base64WithOptionalPadding.encode(value.bytes) }, + fromData = { data -> PluginSignature(Base64WithOptionalPadding.decode(data)) } +) + +open class StringSerializer( + toString: (T) -> String, + fromString: (String) -> T, +) : DataSerializer(String.serializer(), toString, fromString) + +open class DataSerializer( + val serializer: KSerializer, + val toData: (T) -> D, + val fromData: (D) -> T, +) : KSerializer { + + override val descriptor: SerialDescriptor + get() = serializer.descriptor + + override fun deserialize(decoder: Decoder): T { + return fromData(serializer.deserialize(decoder)) + } + + override fun serialize(encoder: Encoder, value: T) { + serializer.serialize(encoder, toData(value)) + } +} + +class KnownMeta { + companion object { + const val ReadableName: String = "readableName" + const val Description: String = "description" + const val DocumentationUrl: String = "documentation-url" + + /** + * Equivalent to vendor `publicName` in Marketplace + */ + const val VendorPublicName: String = "vendor" + + /** + * Equivalent to vendor `name` in Marketplace + */ + const val VendorId: String = "vendorName" + const val Visible: String = "visible" + + const val FrontendOnly: String = "frontend-only" + + const val PartsCoordinates: String = "partsCoordinates" + + const val DefaultIconCoordinates: String = "defaultIconCoordinates" + const val DarkIconCoordinates: String = "darkIconCoordinates" + + /** + * A set of Fleet product codes with which this plugin is compatible, represented as a comma-separated string + * + * Used by Marketplace as a discriminant for version parsing. + * Format between FL and other Fleet products is no the same, FL uses SemVer, other Fleet products uses IntelliJ versioning. + */ + const val SupportedProducts: String = "supportedProducts" + } +} + +internal class JsonFormat { + companion object { + val json = Json { ignoreUnknownKeys = true } + } +} + +fun PluginDescriptor.Companion.decodeFromString(s: String): PluginDescriptor = JsonFormat.json.decodeFromString(serializer(), s) + +fun PluginDescriptor.encodeToString(): String = JsonFormat.json.encodeToString(PluginDescriptor.serializer(), this) +fun PluginDescriptor.encodeToSignableString(): String = copy(signature = null, compatibleShipVersionRange = null).encodeToString() diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/PluginUtils.kt b/fleet/bundles/srcCommonMain/fleet/bundles/PluginUtils.kt new file mode 100644 index 000000000000..ba41116a8730 --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/PluginUtils.kt @@ -0,0 +1,53 @@ +package fleet.bundles + +private fun PluginLayer.intersection(other: PluginLayer?): PluginLayer? = + when { + other == null -> null + else -> PluginLayer(modulePath = modulePath.intersect(other.modulePath), + modules = modules.intersect(other.modules), + resources = resources.intersect(other.resources)) + } + +private fun PluginLayer.merge(other: PluginLayer?): PluginLayer = + other?.let { + PluginLayer(modulePath = modulePath.union(other.modulePath), + modules = modules.union(other.modules), + resources = resources.union(other.resources)) + } ?: this + + +private fun PluginLayer.subtract(other: PluginLayer?): PluginLayer = + when { + other == null -> this + else -> PluginLayer(modulePath = modulePath.subtract(other.modulePath), + modules = modules.subtract(other.modules), + resources = resources.subtract(other.resources)) + } + +fun PluginParts.eliminateIntersections(): PluginParts = + eliminateIntersectionsImpl(frontendLayer, + workspaceLayer, + commonLayer) + .eliminateIntersectionsImpl(frontendImplLayer, + workspaceImplLayer, + commonImplLayer) + .eliminateIntersectionsImpl(frontendApiLayer, + workspaceApiLayer, + commonApiLayer) + +private fun PluginParts.eliminateIntersectionsImpl(frontendS: LayerSelector, + workspaceS: LayerSelector, + commonS: LayerSelector): PluginParts { + val common = layers[frontendS]?.intersection(layers[workspaceS]) + val pairs = listOf(commonS to (layers[commonS]?.merge(common) ?: common), + frontendS to layers[frontendS]?.subtract(common), + workspaceS to layers[workspaceS]?.subtract(common)) + return copy(layers = layers.toMutableMap().apply { + pairs.forEach { (selector, layer) -> + when { + layer != null -> put(selector, layer) + else -> remove(selector) + } + } + }) +} diff --git a/fleet/bundles/srcCommonMain/fleet/bundles/Plugins.kt b/fleet/bundles/srcCommonMain/fleet/bundles/Plugins.kt new file mode 100644 index 000000000000..1e853363d212 --- /dev/null +++ b/fleet/bundles/srcCommonMain/fleet/bundles/Plugins.kt @@ -0,0 +1,389 @@ +package fleet.bundles + +import fleet.util.logging.KLoggers +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +@Serializable +enum class PluginVendor(val isSafeToReport: Boolean) { + Platform(true), JetBrains(true), ThirdParty(false) +} + +@Serializable(with = PluginNameSerializer::class) +data class PluginName(val name: String) + +/** + * VersionRequirement is what can present in extension's dependencies + */ +@Serializable(with = VersionSerializer::class) +sealed class VersionRequirement { + abstract val version: PluginVersion + + data class CompatibleWith(override val version: PluginVersion) : VersionRequirement() + data class Above(override val version: PluginVersion) : VersionRequirement() +} + +/** + * Represents Semantic Version. + */ +@Serializable(with = PluginVersionSerializer::class) +data class PluginVersion( + val major: Int, + val minor: Int, + val patch: Int, + val preRelease: String?, +) : Comparable { + + companion object { + fun fromString(s: String): PluginVersion { + val major = s.substringBefore('.').toIntOrNull() + return when { + major == null || major < 0 -> throw IllegalArgumentException("Cannot parse `$s` as PluginVersion, must start by a strictly positive integer") + major <= 1 -> fromSemverString(s) + else -> fromIntelliJUnifiedVersioningString(s) + } + } + + /** + * Legacy build numbering of FL was using SemVer + */ + private fun fromSemverString(s: String): PluginVersion { + val majorEndIdx = s.indexOf('.') + if (majorEndIdx >= 0) { + val minorEndIdx = s.indexOf('.', majorEndIdx + 1) + if (minorEndIdx >= 0) { + val preReleaseIdx = s.indexOf('-', minorEndIdx + 1) + val patchEndIdx = if (preReleaseIdx >= 0) preReleaseIdx else s.length + val major = s.substring(0, majorEndIdx).toIntOrNull() + val minor = s.substring(majorEndIdx + 1, minorEndIdx).toIntOrNull() + val patch = s.substring(minorEndIdx + 1, patchEndIdx).toIntOrNull() + val preRelease = if (preReleaseIdx >= 0) s.substring(preReleaseIdx + 1) else null + if (major != null && minor != null && patch != null) { + return PluginVersion(major = major, minor = minor, patch = patch, preRelease = preRelease) + } + } + } + throw IllegalArgumentException("Cannot parse `$s` as PluginVersion.") + } + + /** + * Build numbering of AIR and next products of the Fleet platform, using https://youtrack.jetbrains.com/articles/IJPL-A-109 + */ + private fun fromIntelliJUnifiedVersioningString(s: String): PluginVersion { + val intComponents = s.split(".", limit = 3).map { + it.toIntOrSnapshotOrNull() ?: throw IllegalArgumentException("Cannot parse `$s` as PluginVersion, all components must be integers") + } + return when (intComponents.size) { + 3 -> { + val (major, minor, patch) = intComponents + when (major) { + UnifiedVersionComponent.Snapshot -> throw IllegalArgumentException("Cannot parse `$s` as PluginVersion, major cannot be '$SNAPSHOT'") + is UnifiedVersionComponent.IntComponent -> when (minor) { + UnifiedVersionComponent.Snapshot -> throw IllegalArgumentException("Cannot parse `$s` as PluginVersion, minor cannot be '$SNAPSHOT' if there is a patch number") + is UnifiedVersionComponent.IntComponent -> when (patch) { + is UnifiedVersionComponent.IntComponent -> PluginVersion(major = major.value, minor = minor.value, patch = patch.value, preRelease = null) + UnifiedVersionComponent.Snapshot -> PluginVersion(major = major.value, minor = minor.value, patch = 0, preRelease = SNAPSHOT) // hackily set to `0`, I don't see what else we could do here + } + } + } + } + 2 -> { // nightly build + val (major, minor) = intComponents + when (major) { + UnifiedVersionComponent.Snapshot -> throw IllegalArgumentException("Cannot parse `$s` as PluginVersion, major cannot be '$SNAPSHOT'") + is UnifiedVersionComponent.IntComponent -> when (minor) { + is UnifiedVersionComponent.IntComponent -> PluginVersion(major = major.value, minor = minor.value, patch = 0, preRelease = null) // hackily set to `0`, I don't see what else we could do here + UnifiedVersionComponent.Snapshot -> PluginVersion(major = major.value, minor = 0, patch = 0, preRelease = SNAPSHOT) // hackily set to `0`, I don't see what else we could do here + } + } + } + else -> throw IllegalArgumentException("Cannot parse `$s` as PluginVersion, must be either XXX.YYY or XXX.YYY.ZZZ") + } + } + + private const val SNAPSHOT = "SNAPSHOT" + + sealed class UnifiedVersionComponent { + data class IntComponent(val value: Int) : UnifiedVersionComponent() + data object Snapshot : UnifiedVersionComponent() + } + + fun String.toIntOrSnapshotOrNull(): UnifiedVersionComponent? = when (this) { + SNAPSHOT -> UnifiedVersionComponent.Snapshot + else -> toIntOrNull()?.let { UnifiedVersionComponent.IntComponent(it) } + } + } + + val presentableText: String get() = versionString + + /** + * String representation of this [PluginVersion] for Marketplace compatibility range fields of the plugin descriptor's JSON + */ + val marketplaceCompatibilityRangeVersionString: String + get() = buildVersionString(withSemverPatchEvenWhenZero = true) + + val versionString: String + get() = buildVersionString(withSemverPatchEvenWhenZero = false) + + /** + * Build the [String] representation of that [PluginVersion] used in business logic. + * + * Handling of the case where patch=0 depends on [withSemverPatchEvenWhenZero] parameter. + * + * Note: patch=0 usually happens when the original plugin version was a Nightly version. + * Indeed, Nightly versions do not contain a patch component. + * For backward compatibility, in such a case, we had to set it to 0 until we change [PluginVersion] altogether not to represent a SemVer. + * + * @param withSemverPatchEvenWhenZero whether to include a SemVer compatible `.0` patch when patch=0 + */ + private fun buildVersionString(withSemverPatchEvenWhenZero: Boolean): String = buildString { + append(major) + append(".") + append(minor) + if (withSemverPatchEvenWhenZero || patch != 0) { + append(".") + append(patch) + } + if (preRelease != null) { + append("-$preRelease") + } + } + + override fun compareTo(other: PluginVersion): Int { + val result = compareValuesBy(this, other, { it.major }, { it.minor }, { it.patch }) + return if (result != 0) { + result + } + else { + comparePreRelease(preRelease, other.preRelease) + } + } + + private fun comparePreRelease(pre1: String?, pre2: String?): Int { + return when { + pre1 == pre2 -> 0 + pre1 == null -> 1 + pre2 == null -> -1 + else -> { + val iterator1 = pre1.splitToSequence('.').iterator() + val iterator2 = pre2.splitToSequence('.').iterator() + + while (iterator1.hasNext() && iterator2.hasNext()) { + val segment1 = iterator1.next() + val intSegment1 = segment1.toIntOrNull() + val segment2 = iterator2.next() + val intSegment2 = segment2.toIntOrNull() + val result = when { + intSegment1 != null && intSegment2 != null -> { + intSegment1.compareTo(intSegment2) + } + intSegment1 == null && intSegment2 == null -> { + segment1.compareTo(segment2) + } + intSegment1 == null -> { + // According to SemVer specification numeric segments has lower precedence + // than non-numeric segments + 1 + } + else -> -1 + } + if (result != 0) { + return result + } + } + return if (iterator1.hasNext()) { + 1 + } + else { + -1 + } + } + } + } + + // should be in sync with https://github.com/JetBrains/intellij-plugin-verifier/blob/6a04cd7c94eb806877e26a093378eaf2b85e0d73/intellij-plugin-structure/structure-fleet/src/main/kotlin/com/jetbrains/plugin/structure/fleet/FleetPluginDescriptor.kt#L146 + fun toLong(): Long { + val components = buildList { + if (major != 0) { + add(major) + } + if (minor != 0) { + add(minor) + } + if (patch != 0) { + add(patch) + } + if (preRelease != null) { + add(SNAPSHOT_VALUE) + } + } + return CompatibilityUtils.versionAsLong(components.toIntArray()) + } +} + +/** + * Represents a [PluginDescriptor]'s value + */ +@Serializable(with = PluginDescriptorSerializer::class) +data class PluginDescriptor( + val formatVersion: Int = 0, + val name: PluginName, + val version: PluginVersion, + val deps: Map = emptyMap(), + val compatibleShipVersionRange: ShipVersionRange? = null, + val signature: PluginSignature? = null, + val meta: Map = emptyMap(), +) { + override fun toString(): String = prettyJson.encodeToString(serializer(), this) +} + +private val prettyJson = Json { + prettyPrint = true +} + +@OptIn(ExperimentalSerializationApi::class) +private val defaultJson = Json { // we cannot depend on `fleet.util.serialization.DefaultJson` from `fleet.bundles` + ignoreUnknownKeys = true + encodeDefaults = true + explicitNulls = false +} + +val PluginDescriptor.partsCoordinates: Coordinates? get() = metaAsCoordinates(KnownMeta.PartsCoordinates) +val PluginDescriptor.defaultIcon: Coordinates? get() = metaAsCoordinates(KnownMeta.DefaultIconCoordinates) +val PluginDescriptor.darkIcon: Coordinates? get() = metaAsCoordinates(KnownMeta.DarkIconCoordinates) + +fun PluginDescriptor.metaAsCoordinates(metaKey: String): Coordinates? = meta[metaKey]?.let { serializedCoordinates -> + defaultJson.decodeFromString(Coordinates.serializer(), serializedCoordinates) +} + +private const val JETBRAINS_VENDOR = "JetBrains" + +fun PluginDescriptor?.getVendorType(): PluginVendor { + val vendorId = this?.meta?.get(KnownMeta.VendorId) + + return when { + vendorId == null -> PluginVendor.Platform + vendorId == JETBRAINS_VENDOR -> PluginVendor.JetBrains + else -> { + PluginVendor.ThirdParty + } + } +} + +@Serializable(with = PluginSignatureSerializer::class) +data class PluginSignature(val bytes: ByteArray) { + override fun equals(other: Any?): Boolean = + other is PluginSignature && other.bytes.contentEquals(bytes) + + override fun hashCode(): Int = + bytes.contentHashCode() + + override fun toString(): String = + "PluginSignature(size=${bytes.size}, hash=${hashCode().toString(16)}" +} + +@Serializable +data class ShipVersionRange( + @Serializable(with = PluginVersionForCompatibilityRangeSerializer::class) + val from: PluginVersion, + @Serializable(with = PluginVersionForCompatibilityRangeSerializer::class) + val to: PluginVersion, +) + +@Serializable(with = LayerSelectorSerializer::class) +data class LayerSelector(val selector: String) + +@Serializable +data class ModuleCoordinates( + val coordinates: Coordinates, + val serializedModuleDescriptor: String?, +) + +@Serializable(with = PluginLayerSerializer::class) +data class PluginLayer( + val modulePath: Set, + val modules: Set, + val resources: Set, +) + +@Serializable(with = PluginPartsSerializer::class) +data class PluginParts(val layers: Map) + +@Serializable +sealed interface Coordinates { + val meta: Map + + // to reference e.g. a plugin file in marketplace (which might also be in code-cache already) + @Serializable + @SerialName("Remote") + data class Remote(val url: String, val hash: String, override val meta: Map = emptyMap()) : Coordinates { + companion object { + const val HASH_ALGORITHM: String = "SHA3-256" + } + } + + // to reference a folder with classes, should be used when running from sources only + @Serializable + @SerialName("Local") + data class Local(val path: String, override val meta: Map = emptyMap()) : Coordinates +} + +class KnownCoordinatesMeta { + companion object { + const val Platforms: String = "platforms" + } +} + +@Serializable(with = PluginSetSerializer::class) +data class PluginSet( + val shipVersions: Set, + val plugins: Set, +) + +private const val SNAPSHOT_VALUE = Int.MAX_VALUE + +private object CompatibilityUtils { + private const val MAX_BUILD_VALUE = 100000 + private const val MAX_COMPONENT_VALUE = 10000 + private val NUMBERS_OF_NINES by lazy { initNumberOfNines() } + + private fun initNumberOfNines(): IntArray { + val numbersOfNines = ArrayList() + var i = 99999 + val maxIntDiv10 = Int.MAX_VALUE / 10 + while (i < maxIntDiv10) { + i = i * 10 + 9 + numbersOfNines.add(i) + } + + return numbersOfNines.toIntArray() + } + + fun versionAsLong(components: IntArray): Long { + val baselineVersion = components.getOrElse(0) { 0 } + val build = components.getOrElse(1) { 0 } + var longVersion = branchBuildAsLong(baselineVersion, build) + + if (components.size >= 3) { + val component = components[2] + longVersion += if (component == Int.MAX_VALUE) MAX_COMPONENT_VALUE - 1 else component + } + + return longVersion + } + + private fun isNumberOfNines(p: Int) = NUMBERS_OF_NINES.any { it == p } + + private fun branchBuildAsLong(branch: Int, build: Int): Long { + val result = if (build == Int.MAX_VALUE || isNumberOfNines(build)) { + MAX_BUILD_VALUE - 1 + } + else { + build + } + + return branch.toLong() * MAX_COMPONENT_VALUE * MAX_BUILD_VALUE + result.toLong() * MAX_COMPONENT_VALUE + } +} diff --git a/fleet/bundles/srcJvmMain/fleet/bundles/CoordinatesResolution.jvm.kt b/fleet/bundles/srcJvmMain/fleet/bundles/CoordinatesResolution.jvm.kt new file mode 100644 index 000000000000..d3a7c173ddbb --- /dev/null +++ b/fleet/bundles/srcJvmMain/fleet/bundles/CoordinatesResolution.jvm.kt @@ -0,0 +1,55 @@ +package fleet.bundles + +import fleet.multiplatform.shims.multiplatformIO +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import java.nio.file.Path +import java.util.zip.ZipInputStream +import kotlin.io.path.pathString +import kotlin.io.path.readBytes +import kotlin.io.path.readText + +class CoordinatesResolutionImpl(private val fileResolver: suspend (Coordinates) -> Path) : CoordinatesResolution { + companion object { + private val json = Json { + ignoreUnknownKeys = true + encodeDefaults = true + } + } + + override suspend fun resolveFile(coordinates: Coordinates): ResolvedFile { + return withContext(Dispatchers.multiplatformIO) { + ResolvedFile(fileResolver(coordinates).pathString) + } + } + + override suspend fun resolvePluginPartsCoordinates(coordinates: Coordinates): PluginParts { + val parts = withContext(Dispatchers.multiplatformIO) { + fileResolver(coordinates).readText() + } + return json.decodeFromString(PluginParts.serializer(), parts).eliminateIntersections() + } + + override suspend fun resolveModule(coordinates: ModuleCoordinates): ModuleOnDisc { + return withContext(Dispatchers.multiplatformIO) { + ModuleOnDisc(path = fileResolver(coordinates.coordinates).pathString, + serializedModuleDescriptor = coordinates.serializedModuleDescriptor) + } + } + + override suspend fun resolveResource(coordinates: Coordinates): ResourceBundle { + val pathString = resolveFile(coordinates).path + val resources = Path.of(pathString) + .readBytes() + .let(::unzip) + return ResourceBundle { resources[it.removePrefix("/")] } + } +} + +fun unzip(zipBytes: ByteArray): Map { + return ZipInputStream(zipBytes.inputStream()).use { zip -> + generateSequence { zip.nextEntry }.filter { !it.isDirectory }.associate { it.name.removePrefix("/") to zip.readBytes() } + } +} +