diff --git a/.idea/modules.xml b/.idea/modules.xml index 63ba8a827487..a1b86574a338 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -11,6 +11,7 @@ + diff --git a/build/bazel-generated-file-list.txt b/build/bazel-generated-file-list.txt index 8aa52101a965..2092a8e3ec83 100644 --- a/build/bazel-generated-file-list.txt +++ b/build/bazel-generated-file-list.txt @@ -228,6 +228,7 @@ fleet/kernel fleet/ktor/network/tls fleet/lsp.protocol fleet/modules/api +fleet/modules/jvm fleet/multiplatform.shims fleet/preferences fleet/reporting/api diff --git a/fleet/modules/jvm/BUILD.bazel b/fleet/modules/jvm/BUILD.bazel new file mode 100644 index 000000000000..e5b53a9270f9 --- /dev/null +++ b/fleet/modules/jvm/BUILD.bazel @@ -0,0 +1,27 @@ +### auto-generated section `build fleet.modules.jvm` start +load("//build:compiler-options.bzl", "create_kotlinc_options") +load("@rules_jvm//:jvm.bzl", "jvm_library") + +create_kotlinc_options( + name = "custom_jvm", + x_consistent_data_class_copy_visibility = True, + x_context_parameters = True, + x_jvm_default = "all-compatibility", + x_lambdas = "class" +) + +jvm_library( + name = "jvm", + module_name = "fleet.modules.jvm", + visibility = ["//visibility:public"], + srcs = glob(["src/**/*.kt", "src/**/*.java", "src/**/*.form"], allow_empty = True, exclude = ["**/module-info.java"]), + kotlinc_opts = ":custom_jvm", + deps = [ + "@lib//:kotlin-stdlib", + "//fleet/modules/api", + "@lib//:jetbrains-annotations", + "//fleet/util/modules", + "//fleet/util/logging/api", + ] +) +### auto-generated section `build fleet.modules.jvm` end \ No newline at end of file diff --git a/fleet/modules/jvm/fleet.modules.jvm.iml b/fleet/modules/jvm/fleet.modules.jvm.iml new file mode 100644 index 000000000000..39cd8a9a8f23 --- /dev/null +++ b/fleet/modules/jvm/fleet.modules.jvm.iml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/fleet/modules/jvm/gradlebuild/build.gradle.kts b/fleet/modules/jvm/gradlebuild/build.gradle.kts new file mode 100644 index 000000000000..3426ca6dd9b8 --- /dev/null +++ b/fleet/modules/jvm/gradlebuild/build.gradle.kts @@ -0,0 +1,59 @@ +// IMPORT__MARKER_START +import fleet.buildtool.conventions.configureAtMostOneJvmTargetOrThrow +import fleet.buildtool.conventions.withJavaSourceSet +// IMPORT__MARKER_END +plugins { + alias(libs.plugins.kotlin.multiplatform) + id("fleet.project-module-conventions") + id("fleet.toolchain-conventions") + id("fleet.module-publishing-conventions") + id("fleet.sdk-repositories-publishing-conventions") + id("fleet.open-source-module-conventions") + alias(libs.plugins.dokka) + // GRADLE_PLUGINS__MARKER_START + id("fleet-module") + // GRADLE_PLUGINS__MARKER_END +} + +fleetModule { + module { + name = "fleet.modules.jvm" + importedFromJps {} + } +} + +@OptIn(org.jetbrains.kotlin.gradle.ExperimentalWasmDsl::class) +kotlin { + // KOTLIN__MARKER_START + compilerOptions.freeCompilerArgs = listOf( + "-Xlambdas=class", + "-Xconsistent-data-class-copy-visibility", + "-Xcontext-parameters", + "-XXLanguage:+AllowEagerSupertypeAccessibilityChecks", + ) + jvm {} + sourceSets.jvmMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../src")) } + configureAtMostOneJvmTargetOrThrow { compilations.named("main") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../src")) } } } + sourceSets.commonMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcCommonMain")) } + sourceSets.commonMain.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesCommonMain")) } + sourceSets.commonTest.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcCommonTest")) } + sourceSets.commonTest.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesCommonTest")) } + sourceSets.jvmMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcJvmMain")) } + configureAtMostOneJvmTargetOrThrow { compilations.named("main") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../srcJvmMain")) } } } + sourceSets.jvmMain.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesJvmMain")) } + sourceSets.jvmTest.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcJvmTest")) } + configureAtMostOneJvmTargetOrThrow { compilations.named("test") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../srcJvmTest")) } } } + sourceSets.jvmTest.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesJvmTest")) } + sourceSets.commonMain.dependencies { + implementation(jps.org.jetbrains.kotlin.kotlin.stdlib1993400674.get().let { "${it.group}:${it.name}:${it.version}" }) { + exclude(group = "org.jetbrains", module = "annotations") + } + implementation(jps.org.jetbrains.annotations1504825916.get()) + implementation(project(":fleet.modules.api")) + implementation(project(":fleet.util.logging.api")) + } + sourceSets.jvmMain.dependencies { + implementation(project(":fleet.util.modules")) + } + // KOTLIN__MARKER_END +} \ No newline at end of file diff --git a/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModule.kt b/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModule.kt new file mode 100644 index 000000000000..7a2400fc6fd8 --- /dev/null +++ b/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModule.kt @@ -0,0 +1,33 @@ +package fleet.modules.jvm + +import fleet.modules.api.FleetModule +import fleet.modules.api.FleetModuleLayer +import java.util.ServiceLoader +import kotlin.reflect.KClass +import kotlin.streams.asSequence + +data class JvmFleetModule(val module: Module) : FleetModule { + override val name: String + get() = module.name + + override val layer: FleetModuleLayer + get() = JvmFleetModuleLayer(module.getLayer()) + + override fun getEntityTypeProvider(providerName: String): Any? { + val providerClass = module.classLoader.loadClass(providerName) + return providerClass.getField("INSTANCE").get(null) + } + + override fun getResource(path: String): ByteArray? { + return module.getResourceAsStream(path)?.readBytes() + } + + override fun findServices(service: KClass, requestor: KClass<*>): Iterable { + val moduleLayer = module.layer + return JvmFleetModuleLayer.findServices(moduleLayer, service, requestor).stream().asSequence().takeWhile { + it.type().module.layer == moduleLayer + }.filter { + it.type().module == module + }.map(ServiceLoader.Provider::get).asIterable() + } +} diff --git a/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayer.kt b/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayer.kt new file mode 100644 index 000000000000..a8f7c0178beb --- /dev/null +++ b/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayer.kt @@ -0,0 +1,37 @@ +package fleet.modules.jvm + +import fleet.modules.api.FleetModule +import fleet.modules.api.FleetModuleLayer +import java.util.* +import kotlin.reflect.KClass + +data class JvmFleetModuleLayer(val layer: ModuleLayer) : FleetModuleLayer { + companion object { + private val loaderConstructor by lazy { + ServiceLoader::class.java.getDeclaredConstructor(Class::class.java, ModuleLayer::class.java, Class::class.java).also { + it.isAccessible = true + } + } + + internal fun findServices(layer: ModuleLayer, + service: KClass, + requestor: KClass<*>): ServiceLoader { + return loaderConstructor.newInstance(requestor.java, layer, service.java) as ServiceLoader + } + } + + override fun findModule(name: String): FleetModule? { + return layer + .findModule(name) + .map(::JvmFleetModule) + .orElse(null) + } + + override fun findServices(service: KClass, requestor: KClass<*>): Iterable { + return findServices(layer, service, requestor) + } + + override val modules: Set + get() = layer.modules().map(::JvmFleetModule).toSet() + +} diff --git a/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayerLoader.kt b/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayerLoader.kt new file mode 100644 index 000000000000..8af1638a986c --- /dev/null +++ b/fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayerLoader.kt @@ -0,0 +1,71 @@ +package fleet.modules.jvm + +import fleet.modules.api.FleetModuleInfo +import fleet.modules.api.FleetModuleLayer +import fleet.modules.api.FleetModuleLayerLoader +import fleet.util.logging.KLoggers +import fleet.util.modules.FleetModuleFinderLogger +import fleet.util.modules.ModuleInfo +import fleet.util.modules.ModuleLayers +import fleet.util.modules.ModuleLayers.deserializeModuleDescriptor +import java.util.concurrent.ConcurrentHashMap +import java.util.function.Supplier + + +private val logger by lazy { KLoggers.logger(JvmFleetModuleLayerLoader::class) } + +/** + * Returns java module layer for a module path and a list of parents + */ +object JvmFleetModuleLayerLoader { + private val moduleFinderLogger = object : FleetModuleFinderLogger { + override fun warn(message: Supplier) { + logger.warn(message.get()) + } + + override fun error(t: Throwable?, message: Supplier) { + logger.error(t, message.get()) + } + } + + fun jvmModulePath(modulePath: Set): Collection { + return modulePath.map { moduleInfo -> + when (moduleInfo) { + is FleetModuleInfo.Path -> ModuleInfo.Path(moduleInfo.path) + is FleetModuleInfo.WithDescriptor -> { + runCatching { + val jvmDescriptor = deserializeModuleDescriptor(moduleInfo.serializedModuleDescriptor) + ModuleInfo.WithDescriptor(jvmDescriptor, moduleInfo.path) + }.getOrElse { t -> + logger.warn(t) { "Cannot deserialize module descriptor $moduleInfo" } + ModuleInfo.Path(moduleInfo.path) + } + } + } + } + } + + fun production(): FleetModuleLayerLoader = FleetModuleLayerLoader { parentLayers, modulePath -> + val jvmParentLayers = parentLayers.map { (it as JvmFleetModuleLayer).layer } + val jvmModulePath = jvmModulePath(modulePath) + JvmFleetModuleLayer(ModuleLayers.moduleLayer(jvmParentLayers, jvmModulePath, moduleFinderLogger)) + }.memoizing() + + fun test(modulePath: List): FleetModuleLayerLoader { + val layer = ModuleLayers.moduleLayer(emptyList(), modulePath, moduleFinderLogger) // TODO: this operation is very slow, we need to investigate it + val testModuleLayer = TestJvmFleetModuleLayer(layer, modulePath) // share heavy calculations in `TestJvmFleetModuleLayer` amongst all caller of the loader + return FleetModuleLayerLoader { _, _ -> testModuleLayer } + } + + private fun FleetModuleLayerLoader.memoizing(): FleetModuleLayerLoader { + data class Key(@JvmField val parents: List, @JvmField val modulePath: Set) + + val layerByModuleInfo = ConcurrentHashMap() + return FleetModuleLayerLoader { parentLayers, modulePath -> + layerByModuleInfo.computeIfAbsent(Key(parentLayers, modulePath)) { + moduleLayer(parentLayers, modulePath) + } + } + } + +} \ No newline at end of file diff --git a/fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModule.kt b/fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModule.kt new file mode 100644 index 000000000000..89a34874c577 --- /dev/null +++ b/fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModule.kt @@ -0,0 +1,116 @@ +package fleet.modules.jvm + +import fleet.modules.api.FleetModule +import fleet.modules.api.FleetModuleLayer +import fleet.util.logging.KLoggers +import fleet.util.modules.ModuleInfo +import java.io.InputStream +import java.util.jar.JarFile +import kotlin.io.path.* +import kotlin.reflect.KClass + +private val logger by lazy { KLoggers.logger(TestJvmFleetModule::class) } + +data class TestJvmFleetModule( + private val moduleName: String, + private val moduleLayer: FleetModuleLayer, + private val moduleInfo: ModuleInfo? = null, +) : FleetModule { + + /** + * The unnamed module of the system classloader. + * + * In tests, we operate with `--classpath`, both in Gradle and especially in IDE's gutter run where we do not control the `java` call. + * So, all our classes are loaded in the AppClassloader and so [fleet.testlib.core.TestJvmFleetModule] delegates to it in the relevant places. + */ + private val classpathUniqueModule: Module + get() = ClassLoader.getSystemClassLoader().unnamedModule + + override val name: String + get() = moduleName + + override val layer: FleetModuleLayer + get() = moduleLayer + + @Deprecated("Get rid of it as soon as we drop entities auto-registration") + override fun getEntityTypeProvider(providerName: String): Any? { + val providerClass = classpathUniqueModule.classLoader.loadClass(providerName) + return providerClass.getField("INSTANCE").get(null) + } + + override fun getResource(path: String): ByteArray? = + when (val codeLocation = moduleInfo?.codeLocation()) { + null -> null + is CodeLocation.Directory -> Path(codeLocation.path).resolve(path).takeIf { it.exists() }?.readBytes() + is CodeLocation.Jar -> JarFile(codeLocation.path).use { jar -> jar.readBytesOfJarEntry(path) } + } + + // caches provided services resolving, could be slow when it involves reading from a file from the jar + private val providedServices: Map> by lazy { + moduleInfo?.providedServices() ?: emptyMap() + } + + override fun findServices(service: KClass, requestor: KClass<*>): Iterable = + when (moduleInfo) { + null -> { + logger.warn("Trying to find implementation for service '${service.qualifiedName}' in module '${name}' but that module had no module info") + emptyList() + } + else -> providedServices[service.qualifiedName]?.map { loadService(it) } ?: emptyList() + } + + // we need to load the class from `classpathUniqueModule.classLoader`, so using ServiceLoader and an ephemeral classloader here would be redundant + private fun loadService(serviceClass: String): T = + classpathUniqueModule.classLoader.loadClass(serviceClass).getDeclaredConstructor().newInstance() as T +} + +private fun JarFile.readBytesOfJarEntry(path: String): ByteArray? = when (val resourceFile = getJarEntry(path)) { + null -> null + else -> getInputStream(resourceFile).use { it.readBytes() } +} + +private fun ModuleInfo.providedServices(): Map> = when (this) { + is ModuleInfo.Path -> codeLocation().readServices() + is ModuleInfo.WithDescriptor -> descriptor.provides().associate { it.service() to it.providers() } +} + +private fun ModuleInfo.codeLocation(): CodeLocation { + val ppath = when (this) { + is ModuleInfo.Path -> path + is ModuleInfo.WithDescriptor -> path + } + return when { + Path(ppath).isDirectory() -> CodeLocation.Directory(ppath) + ppath.endsWith(".jar") -> CodeLocation.Jar(ppath) + else -> error("Unsupported code location: $ppath, must be a directory or a jar file") + } +} + +private sealed class CodeLocation(val path: String) { + class Jar(path: String) : CodeLocation(path) + class Directory(path: String) : CodeLocation(path) +} + +/** + * Manually reads services provided by a JAR or a compialtion output directory containing META-INF/ + */ +private fun CodeLocation.readServices(): Map> { + val servicesDirectory = "META-INF/services/" + return when (this) { + is CodeLocation.Directory -> Path(path).resolve(servicesDirectory).takeIf { it.exists() }?.listDirectoryEntries()?.associate { entry -> + entry.fileName.toString() to entry.inputStream().readServiceImplementations() + } ?: emptyMap() + is CodeLocation.Jar -> JarFile(path).use { jar -> + jar.entries().asSequence().filter { entry -> + !entry.isDirectory && entry.name.startsWith(servicesDirectory) && entry.name.length > servicesDirectory.length + }.associate { entry -> + entry.name.removePrefix(servicesDirectory) to jar.getInputStream(entry).readServiceImplementations() + } + } + } +} + +private fun InputStream.readServiceImplementations(): List = bufferedReader().useLines { lines -> + lines.map { it.trim() }.filter { it.isNotEmpty() && !it.startsWith("#") } +}.toList() + diff --git a/fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModuleLayer.kt b/fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModuleLayer.kt new file mode 100644 index 000000000000..29993703e0e4 --- /dev/null +++ b/fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModuleLayer.kt @@ -0,0 +1,62 @@ +package fleet.modules.jvm + +import fleet.modules.api.FleetModule +import fleet.modules.api.FleetModuleLayer +import fleet.util.modules.ModuleInfo +import java.lang.module.ModuleFinder +import java.util.ServiceLoader +import kotlin.io.path.Path +import kotlin.reflect.KClass + +/** + * The module layer used in Fleet's test runner. + * Fleet's runtime is modularized to ensure isolation between Dock, SHIP and plugins, however Fleet modules/jars are not modularized (do not + * have `module-info.class`). + * This module layer abstraction bridges between the non-modularized classpath used in tests, and the modularized application definitions + * (plugin descriptors, init module descriptors). + * + * The provided [layer] is only used to construct a set of [TestJvmFleetModule], and not as a backing module layer for loading services, + * resources, etc. + * Indeed, packages of modules in that module layer are already loaded in the SystemClassLoader's unnamed module. + * Even if we created a module layer backed by this class loader, it would for example fail with + * `Package kotlinx/coroutines/test for module kotlinx.coroutines.test is already in the unnamed module defined to the class loader`. + * So instead we implement the delegation ourselves as part of the [TestJvmFleetModule]'s abstraction. + * + * @param layer the module layer containing every test modules and Fleet runtime modules + * @param modulePath the module path of the provided layer + */ +class TestJvmFleetModuleLayer( + val layer: ModuleLayer, + private val modulePath: List, +) : FleetModuleLayer { + private val moduleInfosByName: Map by lazy { + modulePath.mapNotNull { it.name()?.let { name -> name to it } }.toMap() // TODO: check duplicates + } + + private val moduleByName: Map by lazy { + layer.modules().mapNotNull { + TestJvmFleetModule( + moduleName = it.name, + moduleLayer = this, + moduleInfo = moduleInfosByName[it.name], + ) + }.toSet().associateBy { it.name } + } + + private val cachedModules by lazy { + moduleByName.values.toSet() + } + + override val modules: Set + get() = cachedModules + + override fun findModule(name: String): FleetModule? = moduleByName[name] + + override fun findServices(service: KClass, requestor: KClass<*>): Iterable = + modules.flatMap { it.findServices(service, requestor) } // TODO: could we do better in terms of performance here? +} + +private fun ModuleInfo.name(): String? = when (this) { + is ModuleInfo.Path -> ModuleFinder.of(Path(path)).findAll().singleOrNull()?.descriptor()?.name() + is ModuleInfo.WithDescriptor -> descriptor.name() +}