[fleet] Move fleet.modules.jvm to Community

(cherry picked from commit 1629064851e1af29723dc85cae7c8dfa24d99755)

FLEET-MR-6894

GitOrigin-RevId: dc14c3fc8814e23c03602a77cfc0158789d313d7

.idea/modules.xml

@@ -11,6 +11,7 @@
       <module fileurl="file://$PROJECT_DIR$/fleet/ktor/network/tls/fleet.ktor.network.tls.iml" filepath="$PROJECT_DIR$/fleet/ktor/network/tls/fleet.ktor.network.tls.iml" />
       <module fileurl="file://$PROJECT_DIR$/fleet/lsp.protocol/fleet.lsp.protocol.iml" filepath="$PROJECT_DIR$/fleet/lsp.protocol/fleet.lsp.protocol.iml" />
       <module fileurl="file://$PROJECT_DIR$/fleet/modules/api/fleet.modules.api.iml" filepath="$PROJECT_DIR$/community/fleet/modules/api/fleet.modules.api.iml" />
+      <module fileurl="file://$PROJECT_DIR$/fleet/modules/jvm/fleet.modules.jvm.iml" filepath="$PROJECT_DIR$/community/fleet/modules/jvm/fleet.modules.jvm.iml" />
       <module fileurl="file://$PROJECT_DIR$/fleet/multiplatform.shims/fleet.multiplatform.shims.iml" filepath="$PROJECT_DIR$/fleet/multiplatform.shims/fleet.multiplatform.shims.iml" />
       <module fileurl="file://$PROJECT_DIR$/fleet/preferences/fleet.preferences.iml" filepath="$PROJECT_DIR$/community/fleet/preferences/fleet.preferences.iml" />
       <module fileurl="file://$PROJECT_DIR$/fleet/reporting/api/fleet.reporting.api.iml" filepath="$PROJECT_DIR$/fleet/reporting/api/fleet.reporting.api.iml" />

build/bazel-generated-file-list.txt

@@ -228,6 +228,7 @@ fleet/kernel
 fleet/ktor/network/tls
 fleet/lsp.protocol
 fleet/modules/api
+fleet/modules/jvm
 fleet/multiplatform.shims
 fleet/preferences
 fleet/reporting/api

fleet/modules/jvm/BUILD.bazel

@@ -0,0 +1,27 @@
+### auto-generated section `build fleet.modules.jvm` start
+load("//build:compiler-options.bzl", "create_kotlinc_options")
+load("@rules_jvm//:jvm.bzl", "jvm_library")
+
+create_kotlinc_options(
+  name = "custom_jvm",
+  x_consistent_data_class_copy_visibility = True,
+  x_context_parameters = True,
+  x_jvm_default = "all-compatibility",
+  x_lambdas = "class"
+)
+
+jvm_library(
+  name = "jvm",
+  module_name = "fleet.modules.jvm",
+  visibility = ["//visibility:public"],
+  srcs = glob(["src/**/*.kt", "src/**/*.java", "src/**/*.form"], allow_empty = True, exclude = ["**/module-info.java"]),
+  kotlinc_opts = ":custom_jvm",
+  deps = [
+    "@lib//:kotlin-stdlib",
+    "//fleet/modules/api",
+    "@lib//:jetbrains-annotations",
+    "//fleet/util/modules",
+    "//fleet/util/logging/api",
+  ]
+)
+### auto-generated section `build fleet.modules.jvm` end

fleet/modules/jvm/fleet.modules.jvm.iml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="FacetManager">
+    <facet type="kotlin-language" name="Kotlin">
+      <configuration version="5" platform="JVM 21" allPlatforms="JVM [21]" useProjectSettings="false">
+        <compilerSettings>
+          <option name="additionalArguments" value="-Xlambdas=class -Xconsistent-data-class-copy-visibility -Xcontext-parameters -XXLanguage:+AllowEagerSupertypeAccessibilityChecks" />
+        </compilerSettings>
+        <compilerArguments>
+          <stringArguments>
+            <stringArg name="jvmTarget" arg="21" />
+            <stringArg name="apiVersion" arg="2.2" />
+            <stringArg name="languageVersion" arg="2.2" />
+          </stringArguments>
+        </compilerArguments>
+      </configuration>
+    </facet>
+  </component>
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="kotlin-stdlib" level="project" />
+    <orderEntry type="module" module-name="fleet.modules.api" />
+    <orderEntry type="library" name="jetbrains-annotations" level="project" />
+    <orderEntry type="module" module-name="fleet.util.modules" />
+    <orderEntry type="module" module-name="fleet.util.logging.api" />
+  </component>
+</module>

fleet/modules/jvm/gradlebuild/build.gradle.kts

@@ -0,0 +1,59 @@
+// IMPORT__MARKER_START
+import fleet.buildtool.conventions.configureAtMostOneJvmTargetOrThrow
+import fleet.buildtool.conventions.withJavaSourceSet
+// IMPORT__MARKER_END
+plugins {
+  alias(libs.plugins.kotlin.multiplatform)
+  id("fleet.project-module-conventions")
+  id("fleet.toolchain-conventions")
+  id("fleet.module-publishing-conventions")
+  id("fleet.sdk-repositories-publishing-conventions")
+  id("fleet.open-source-module-conventions")
+  alias(libs.plugins.dokka)
+  // GRADLE_PLUGINS__MARKER_START
+  id("fleet-module")
+  // GRADLE_PLUGINS__MARKER_END
+}
+
+fleetModule {
+  module {
+    name = "fleet.modules.jvm"
+    importedFromJps {}
+  }
+}
+
+@OptIn(org.jetbrains.kotlin.gradle.ExperimentalWasmDsl::class)
+kotlin {
+  // KOTLIN__MARKER_START
+  compilerOptions.freeCompilerArgs = listOf(
+    "-Xlambdas=class",
+    "-Xconsistent-data-class-copy-visibility",
+    "-Xcontext-parameters",
+    "-XXLanguage:+AllowEagerSupertypeAccessibilityChecks",
+  )
+  jvm {}
+  sourceSets.jvmMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../src")) }
+  configureAtMostOneJvmTargetOrThrow { compilations.named("main") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../src")) } } }
+  sourceSets.commonMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcCommonMain")) }
+  sourceSets.commonMain.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesCommonMain")) }
+  sourceSets.commonTest.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcCommonTest")) }
+  sourceSets.commonTest.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesCommonTest")) }
+  sourceSets.jvmMain.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcJvmMain")) }
+  configureAtMostOneJvmTargetOrThrow { compilations.named("main") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../srcJvmMain")) } } }
+  sourceSets.jvmMain.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesJvmMain")) }
+  sourceSets.jvmTest.configure { kotlin.srcDir(layout.projectDirectory.dir("../srcJvmTest")) }
+  configureAtMostOneJvmTargetOrThrow { compilations.named("test") { withJavaSourceSet { javaSourceSet -> javaSourceSet.java.srcDir(layout.projectDirectory.dir("../srcJvmTest")) } } }
+  sourceSets.jvmTest.configure { resources.srcDir(layout.projectDirectory.dir("../resourcesJvmTest")) }
+  sourceSets.commonMain.dependencies {
+    implementation(jps.org.jetbrains.kotlin.kotlin.stdlib1993400674.get().let { "${it.group}:${it.name}:${it.version}" }) {
+      exclude(group = "org.jetbrains", module = "annotations")
+    }
+    implementation(jps.org.jetbrains.annotations1504825916.get())
+    implementation(project(":fleet.modules.api"))
+    implementation(project(":fleet.util.logging.api"))
+  }
+  sourceSets.jvmMain.dependencies {
+    implementation(project(":fleet.util.modules"))
+  }
+  // KOTLIN__MARKER_END
+}

fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModule.kt

@@ -0,0 +1,33 @@
+package fleet.modules.jvm
+
+import fleet.modules.api.FleetModule
+import fleet.modules.api.FleetModuleLayer
+import java.util.ServiceLoader
+import kotlin.reflect.KClass
+import kotlin.streams.asSequence
+
+data class JvmFleetModule(val module: Module) : FleetModule {
+  override val name: String
+    get() = module.name
+
+  override val layer: FleetModuleLayer
+    get() = JvmFleetModuleLayer(module.getLayer())
+
+  override fun getEntityTypeProvider(providerName: String): Any? {
+    val providerClass = module.classLoader.loadClass(providerName)
+    return providerClass.getField("INSTANCE").get(null)
+  }
+
+  override fun getResource(path: String): ByteArray? {
+    return module.getResourceAsStream(path)?.readBytes()
+  }
+
+  override fun <T : Any> findServices(service: KClass<T>, requestor: KClass<*>): Iterable<T> {
+    val moduleLayer = module.layer
+    return JvmFleetModuleLayer.findServices(moduleLayer, service, requestor).stream().asSequence().takeWhile {
+      it.type().module.layer == moduleLayer
+    }.filter {
+      it.type().module == module
+    }.map(ServiceLoader.Provider<T>::get).asIterable()
+  }
+}

fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayer.kt

@@ -0,0 +1,37 @@
+package fleet.modules.jvm
+
+import fleet.modules.api.FleetModule
+import fleet.modules.api.FleetModuleLayer
+import java.util.*
+import kotlin.reflect.KClass
+
+data class JvmFleetModuleLayer(val layer: ModuleLayer) : FleetModuleLayer {
+  companion object {
+    private val loaderConstructor by lazy {
+      ServiceLoader::class.java.getDeclaredConstructor(Class::class.java, ModuleLayer::class.java, Class::class.java).also {
+        it.isAccessible = true
+      }
+    }
+
+    internal fun <T : Any> findServices(layer: ModuleLayer,
+                                        service: KClass<T>,
+                                        requestor: KClass<*>): ServiceLoader<T> {
+      return loaderConstructor.newInstance(requestor.java, layer, service.java) as ServiceLoader<T>
+    }
+  }
+
+  override fun findModule(name: String): FleetModule? {
+    return layer
+      .findModule(name)
+      .map(::JvmFleetModule)
+      .orElse(null)
+  }
+
+  override fun <T : Any> findServices(service: KClass<T>, requestor: KClass<*>): Iterable<T> {
+    return findServices(layer, service, requestor)
+  }
+
+  override val modules: Set<FleetModule>
+    get() = layer.modules().map(::JvmFleetModule).toSet()
+
+}

fleet/modules/jvm/src/fleet/modules/jvm/JvmFleetModuleLayerLoader.kt

@@ -0,0 +1,71 @@
+package fleet.modules.jvm
+
+import fleet.modules.api.FleetModuleInfo
+import fleet.modules.api.FleetModuleLayer
+import fleet.modules.api.FleetModuleLayerLoader
+import fleet.util.logging.KLoggers
+import fleet.util.modules.FleetModuleFinderLogger
+import fleet.util.modules.ModuleInfo
+import fleet.util.modules.ModuleLayers
+import fleet.util.modules.ModuleLayers.deserializeModuleDescriptor
+import java.util.concurrent.ConcurrentHashMap
+import java.util.function.Supplier
+
+
+private val logger by lazy { KLoggers.logger(JvmFleetModuleLayerLoader::class) }
+
+/**
+ * Returns java module layer for a module path and a list of parents
+ */
+object JvmFleetModuleLayerLoader {
+  private val moduleFinderLogger = object : FleetModuleFinderLogger {
+    override fun warn(message: Supplier<String>) {
+      logger.warn(message.get())
+    }
+
+    override fun error(t: Throwable?, message: Supplier<String>) {
+      logger.error(t, message.get())
+    }
+  }
+
+  fun jvmModulePath(modulePath: Set<FleetModuleInfo>): Collection<ModuleInfo> {
+    return modulePath.map { moduleInfo ->
+      when (moduleInfo) {
+        is FleetModuleInfo.Path -> ModuleInfo.Path(moduleInfo.path)
+        is FleetModuleInfo.WithDescriptor -> {
+          runCatching {
+            val jvmDescriptor = deserializeModuleDescriptor(moduleInfo.serializedModuleDescriptor)
+            ModuleInfo.WithDescriptor(jvmDescriptor, moduleInfo.path)
+          }.getOrElse { t ->
+            logger.warn(t) { "Cannot deserialize module descriptor $moduleInfo" }
+            ModuleInfo.Path(moduleInfo.path)
+          }
+        }
+      }
+    }
+  }
+
+  fun production(): FleetModuleLayerLoader = FleetModuleLayerLoader { parentLayers, modulePath ->
+    val jvmParentLayers = parentLayers.map { (it as JvmFleetModuleLayer).layer }
+    val jvmModulePath = jvmModulePath(modulePath)
+    JvmFleetModuleLayer(ModuleLayers.moduleLayer(jvmParentLayers, jvmModulePath, moduleFinderLogger))
+  }.memoizing()
+
+  fun test(modulePath: List<ModuleInfo>): FleetModuleLayerLoader {
+    val layer = ModuleLayers.moduleLayer(emptyList(), modulePath, moduleFinderLogger) // TODO: this operation is very slow, we need to investigate it
+    val testModuleLayer = TestJvmFleetModuleLayer(layer, modulePath) // share heavy calculations in `TestJvmFleetModuleLayer` amongst all caller of the loader
+    return FleetModuleLayerLoader { _, _ -> testModuleLayer }
+  }
+
+  private fun FleetModuleLayerLoader.memoizing(): FleetModuleLayerLoader {
+    data class Key(@JvmField val parents: List<FleetModuleLayer>, @JvmField val modulePath: Set<FleetModuleInfo>)
+
+    val layerByModuleInfo = ConcurrentHashMap<Key, FleetModuleLayer>()
+    return FleetModuleLayerLoader { parentLayers, modulePath ->
+      layerByModuleInfo.computeIfAbsent(Key(parentLayers, modulePath)) {
+        moduleLayer(parentLayers, modulePath)
+      }
+    }
+  }
+
+}

fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModule.kt

@@ -0,0 +1,116 @@
+package fleet.modules.jvm
+
+import fleet.modules.api.FleetModule
+import fleet.modules.api.FleetModuleLayer
+import fleet.util.logging.KLoggers
+import fleet.util.modules.ModuleInfo
+import java.io.InputStream
+import java.util.jar.JarFile
+import kotlin.io.path.*
+import kotlin.reflect.KClass
+
+private val logger by lazy { KLoggers.logger(TestJvmFleetModule::class) }
+
+data class TestJvmFleetModule(
+  private val moduleName: String,
+  private val moduleLayer: FleetModuleLayer,
+  private val moduleInfo: ModuleInfo? = null,
+) : FleetModule {
+
+  /**
+   * The unnamed module of the system classloader.
+   *
+   * In tests, we operate with `--classpath`, both in Gradle and especially in IDE's gutter run where we do not control the `java` call.
+   * So, all our classes are loaded in the AppClassloader and so [fleet.testlib.core.TestJvmFleetModule] delegates to it in the relevant places.
+   */
+  private val classpathUniqueModule: Module
+    get() = ClassLoader.getSystemClassLoader().unnamedModule
+
+  override val name: String
+    get() = moduleName
+
+  override val layer: FleetModuleLayer
+    get() = moduleLayer
+
+  @Deprecated("Get rid of it as soon as we drop entities auto-registration")
+  override fun getEntityTypeProvider(providerName: String): Any? {
+    val providerClass = classpathUniqueModule.classLoader.loadClass(providerName)
+    return providerClass.getField("INSTANCE").get(null)
+  }
+
+  override fun getResource(path: String): ByteArray? =
+    when (val codeLocation = moduleInfo?.codeLocation()) {
+      null -> null
+      is CodeLocation.Directory -> Path(codeLocation.path).resolve(path).takeIf { it.exists() }?.readBytes()
+      is CodeLocation.Jar -> JarFile(codeLocation.path).use { jar -> jar.readBytesOfJarEntry(path) }
+    }
+
+  // caches provided services resolving, could be slow when it involves reading from a file from the jar
+  private val providedServices: Map<String, List<String>> by lazy {
+    moduleInfo?.providedServices() ?: emptyMap()
+  }
+
+  override fun <T : Any> findServices(service: KClass<T>, requestor: KClass<*>): Iterable<T> =
+    when (moduleInfo) {
+      null -> {
+        logger.warn("Trying to find implementation for service '${service.qualifiedName}' in module '${name}' but that module had no module info")
+        emptyList()
+      }
+      else -> providedServices[service.qualifiedName]?.map { loadService(it) } ?: emptyList()
+    }
+
+  // we need to load the class from `classpathUniqueModule.classLoader`, so using ServiceLoader and an ephemeral classloader here would be redundant
+  private fun <T> loadService(serviceClass: String): T =
+    classpathUniqueModule.classLoader.loadClass(serviceClass).getDeclaredConstructor().newInstance() as T
+}
+
+private fun JarFile.readBytesOfJarEntry(path: String): ByteArray? = when (val resourceFile = getJarEntry(path)) {
+  null -> null
+  else -> getInputStream(resourceFile).use { it.readBytes() }
+}
+
+private fun ModuleInfo.providedServices(): Map<String, List<String>> = when (this) {
+  is ModuleInfo.Path -> codeLocation().readServices()
+  is ModuleInfo.WithDescriptor -> descriptor.provides().associate { it.service() to it.providers() }
+}
+
+private fun ModuleInfo.codeLocation(): CodeLocation {
+  val ppath = when (this) {
+    is ModuleInfo.Path -> path
+    is ModuleInfo.WithDescriptor -> path
+  }
+  return when {
+    Path(ppath).isDirectory() -> CodeLocation.Directory(ppath)
+    ppath.endsWith(".jar") -> CodeLocation.Jar(ppath)
+    else -> error("Unsupported code location: $ppath, must be a directory or a jar file")
+  }
+}
+
+private sealed class CodeLocation(val path: String) {
+  class Jar(path: String) : CodeLocation(path)
+  class Directory(path: String) : CodeLocation(path)
+}
+
+/**
+ * Manually reads services provided by a JAR or a compialtion output directory containing META-INF/
+ */
+private fun CodeLocation.readServices(): Map<String, List<String>> {
+  val servicesDirectory = "META-INF/services/"
+  return when (this) {
+    is CodeLocation.Directory -> Path(path).resolve(servicesDirectory).takeIf { it.exists() }?.listDirectoryEntries()?.associate { entry ->
+      entry.fileName.toString() to entry.inputStream().readServiceImplementations()
+    } ?: emptyMap()
+    is CodeLocation.Jar -> JarFile(path).use { jar ->
+      jar.entries().asSequence().filter { entry ->
+        !entry.isDirectory && entry.name.startsWith(servicesDirectory) && entry.name.length > servicesDirectory.length
+      }.associate { entry ->
+        entry.name.removePrefix(servicesDirectory) to jar.getInputStream(entry).readServiceImplementations()
+      }
+    }
+  }
+}
+
+private fun InputStream.readServiceImplementations(): List<String> = bufferedReader().useLines { lines ->
+  lines.map { it.trim() }.filter { it.isNotEmpty() && !it.startsWith("#") }
+}.toList()
+

fleet/modules/jvm/src/fleet/modules/jvm/TestJvmFleetModuleLayer.kt

@@ -0,0 +1,62 @@
+package fleet.modules.jvm
+
+import fleet.modules.api.FleetModule
+import fleet.modules.api.FleetModuleLayer
+import fleet.util.modules.ModuleInfo
+import java.lang.module.ModuleFinder
+import java.util.ServiceLoader
+import kotlin.io.path.Path
+import kotlin.reflect.KClass
+
+/**
+ * The module layer used in Fleet's test runner.
+ * Fleet's runtime is modularized to ensure isolation between Dock, SHIP and plugins, however Fleet modules/jars are not modularized (do not
+ * have `module-info.class`).
+ * This module layer abstraction bridges between the non-modularized classpath used in tests, and the modularized application definitions
+ * (plugin descriptors, init module descriptors).
+ *
+ * The provided [layer] is only used to construct a set of [TestJvmFleetModule], and not as a backing module layer for loading services,
+ * resources, etc.
+ * Indeed, packages of modules in that module layer are already loaded in the SystemClassLoader's unnamed module.
+ * Even if we created a module layer backed by this class loader, it would for example fail with
+ * `Package kotlinx/coroutines/test for module kotlinx.coroutines.test is already in the unnamed module defined to the class loader`.
+ * So instead we implement the delegation ourselves as part of the [TestJvmFleetModule]'s abstraction.
+ *
+ * @param layer the module layer containing every test modules and Fleet runtime modules
+ * @param modulePath the module path of the provided layer
+ */
+class TestJvmFleetModuleLayer(
+  val layer: ModuleLayer,
+  private val modulePath: List<ModuleInfo>,
+) : FleetModuleLayer {
+  private val moduleInfosByName: Map<String?, ModuleInfo> by lazy {
+    modulePath.mapNotNull { it.name()?.let { name -> name to it } }.toMap() // TODO: check duplicates
+  }
+
+  private val moduleByName: Map<String, TestJvmFleetModule> by lazy {
+    layer.modules().mapNotNull {
+      TestJvmFleetModule(
+        moduleName = it.name,
+        moduleLayer = this,
+        moduleInfo = moduleInfosByName[it.name],
+      )
+    }.toSet().associateBy { it.name }
+  }
+
+  private val cachedModules by lazy {
+    moduleByName.values.toSet()
+  }
+
+  override val modules: Set<FleetModule>
+    get() = cachedModules
+
+  override fun findModule(name: String): FleetModule? = moduleByName[name]
+
+  override fun <T : Any> findServices(service: KClass<T>, requestor: KClass<*>): Iterable<T> =
+    modules.flatMap { it.findServices(service, requestor) } // TODO: could we do better in terms of performance here?
+}
+
+private fun ModuleInfo.name(): String? = when (this) {
+  is ModuleInfo.Path -> ModuleFinder.of(Path(path)).findAll().singleOrNull()?.descriptor()?.name()
+  is ModuleInfo.WithDescriptor -> descriptor.name()
+}