IJPL-149476 PreSharedIndexesGenerator - use dev mode for IDE (part 2 - fix running for community products)

GitOrigin-RevId: 186efdafc2fd7029caf77e77e911c83193d379c1
2026-09-27 10:03:11 +07:00 · 2024-05-15 21:12:48 +00:00
parent a037c78f9b
commit fbc125eefb
14 changed files with 141 additions and 99 deletions
@@ -61,7 +61,7 @@ fun buildDevMain(): Collection<Path> {
            newClassPath = classPath
            homePath = runDir.toString().replace(File.separator, "/")

-            for ((name, value) in getIdeSystemProperties(runDir)) {
+            for ((name, value) in getIdeSystemProperties(runDir).map) {
              System.setProperty(name, value)
            }
          },
@@ -143,7 +143,7 @@ interface BuildContext : CompilationContext {

  suspend fun cleanupJarCache()

-  suspend fun createProductRunner(): IntellijProductRunner
+  suspend fun createProductRunner(additionalPluginModules: List<String> = emptyList()): IntellijProductRunner
 }

 suspend inline fun <T> BuildContext.executeStep(spanBuilder: SpanBuilder,
@@ -8,6 +8,7 @@ import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.toPersistentMap
 import org.jetbrains.annotations.ApiStatus
 import org.jetbrains.annotations.TestOnly
+import org.jetbrains.intellij.build.BuildOptions.Companion.BUILD_STEPS_TO_SKIP_PROPERTY
 import org.jetbrains.intellij.build.dependencies.TeamCityHelper
 import org.jetbrains.jps.api.GlobalOptions
 import java.nio.file.Path
@@ -89,6 +90,8 @@ data class BuildOptions(
  @JvmField val unpackCompiledClassesArchives: Boolean = SystemProperties.getBooleanProperty(INTELLIJ_BUILD_COMPILER_CLASSES_ARCHIVES_UNPACK, true),

  @JvmField internal val validateModuleStructure: Boolean = parseBooleanValue(System.getProperty(VALIDATE_MODULES_STRUCTURE_PROPERTY, "false")),
+
+  @JvmField internal val isUnpackedDist: Boolean = false,
 ) {
  companion object {
    /**
@@ -0,0 +1,33 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build
+
+import java.util.*
+
+@JvmInline
+value class VmProperties(val map: Map<String, String>) {
+  inline fun mutate(operation: MutableMap<String, String>.() -> Unit): VmProperties {
+    val map = LinkedHashMap(map)
+    map.operation()
+    return VmProperties(map.takeIf { it.isNotEmpty() } ?: Collections.emptyMap())
+  }
+
+  fun toJvmArgs(): List<String> {
+    if (map.isEmpty()) {
+      return Collections.emptyList()
+    }
+
+    val result = ArrayList<String>(map.size)
+    for ((key, value) in map) {
+      result.add("-D$key=$value")
+    }
+    return result
+  }
+
+  operator fun plus(other: VmProperties): VmProperties {
+    return when {
+      map.isEmpty() -> other
+      other.map.isEmpty() -> this
+      else -> VmProperties(map + other.map)
+    }
+  }
+}
@@ -9,10 +9,7 @@ import kotlinx.coroutines.withContext
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
-import org.jetbrains.intellij.build.BuildOptions
-import org.jetbrains.intellij.build.JvmArchitecture
-import org.jetbrains.intellij.build.TraceManager
-import org.jetbrains.intellij.build.closeKtorClient
+import org.jetbrains.intellij.build.*
 import java.nio.file.Files
 import java.nio.file.Path
 import java.util.*
@@ -31,7 +28,7 @@ private const val PRODUCTS_PROPERTIES_PATH = "build/dev-build.json"
 private const val CUSTOM_PRODUCT_PROPERTIES_PATH = "idea.product.properties.path"

@Suppress("SpellCheckingInspection")
-fun getIdeSystemProperties(runDir: Path): Map<String, String> {
+fun getIdeSystemProperties(runDir: Path): VmProperties {
  val result = LinkedHashMap<String, String>()

  val properties = Properties()
@@ -52,7 +49,7 @@ fun getIdeSystemProperties(runDir: Path): Map<String, String> {
      "jb.vmOptionsFile" to "${Files.newDirectoryStream(runDir.parent, "*.vmoptions").use { it.singleOrNull() }}"
    )
  )
-  return result
+  return VmProperties(result)
 }

 /** Returns IDE installation directory */
@@ -62,6 +62,8 @@ data class BuildRequest(

  @JvmField val isUnpackedDist: Boolean = System.getProperty("idea.dev.build.unpacked").toBoolean(),

+  @JvmField val writeCoreClasspath: Boolean = true,
+
  @JvmField val buildOptionsTemplate: BuildOptions? = null,
 ) {
  override fun toString(): String =
@@ -147,12 +149,14 @@ internal suspend fun buildProduct(request: BuildRequest, createProductProperties
        )
      }

-      launch(Dispatchers.IO) {
-        val cp = classPath
-          .asSequence()
-          .filter { !excludedLibJars.contains(it.fileName.toString()) }
-          .joinToString(separator = "\n")
-        Files.writeString(runDir.resolve("core-classpath.txt"), cp)
+      if (request.writeCoreClasspath) {
+        launch(Dispatchers.IO) {
+          val cp = classPath
+            .asSequence()
+            .filter { !excludedLibJars.contains(it.fileName.toString()) }
+            .joinToString(separator = "\n")
+          Files.writeString(runDir.resolve("core-classpath.txt"), cp)
+        }
      }

      request.platformClassPathConsumer?.invoke(classPath, runDir)
@@ -405,6 +409,8 @@ private suspend fun createBuildContext(
          outRootDir = runDir,
          compilationLogEnabled = false,
          logDir = buildDir.resolve("log"),
+
+          isUnpackedDist = request.isUnpackedDist,
        )
        options.setTargetOsAndArchToCurrent()
        options.buildStepsToSkip += listOf(
@@ -376,12 +376,11 @@ class BuildContextImpl internal constructor(
    createDevModeProductRunner(this@BuildContextImpl)
  }

-  override suspend fun createProductRunner(): IntellijProductRunner {
-    if (useModularLoader) {
-      return ModuleBasedProductRunner(productProperties.rootModuleForModularLoader!!, this)
-    }
-    else {
-      return devModeProductRunner.await()
+  override suspend fun createProductRunner(additionalPluginModules: List<String>): IntellijProductRunner {
+    when {
+      useModularLoader -> return ModuleBasedProductRunner(productProperties.rootModuleForModularLoader!!, this)
+      additionalPluginModules.isEmpty() -> return devModeProductRunner.await()
+      else -> return createDevModeProductRunner(additionalPluginModules = additionalPluginModules, context = this)
    }
  }
 }
@@ -1,8 +1,9 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+@file:Suppress("ReplacePutWithAssignment")
+
 package org.jetbrains.intellij.build.impl

 import com.intellij.openapi.util.text.StringUtilRt
-import org.jetbrains.intellij.build.impl.BuildUtils.addVmProperty
 import java.nio.file.Files
 import java.nio.file.Path
 import java.util.function.BiConsumer
@@ -25,21 +26,7 @@ object BuildUtils {
    Files.writeString(targetPath, content)
  }

-  internal fun addVmProperty(args: MutableList<String>, key: String, value: String?) {
-    if (value != null) {
-      args.add("-D$key=$value")
-    }
-  }
-
  fun getPluginJars(pluginPath: Path): List<Path> {
    return Files.newDirectoryStream(pluginPath.resolve("lib"), "*.jar").use { it.toList() }
  }
-}
-
-fun propertiesToJvmArgs(properties: Map<String, String>): List<String> {
-  val result = ArrayList<String>(properties.size)
-  for ((key, value) in properties) {
-    addVmProperty(result, key, value)
-  }
-  return result
 }
@@ -35,7 +35,6 @@ import kotlin.io.path.invariantSeparatorsPathString
 import kotlin.io.path.readLines

 private val JAR_NAME_WITH_VERSION_PATTERN = "(.*)-\\d+(?:\\.\\d+)*\\.jar*".toPattern()
-private val isUnpackedDist = System.getProperty("idea.dev.build.unpacked").toBoolean()

 private val libsThatUsedInJps = java.util.Set.of(
  "ASM",
@@ -181,7 +180,7 @@ class JarPackager private constructor(
            cache = cacheManager,
            context = context,
            isCodesignEnabled = isCodesignEnabled,
-            useCacheAsTargetFile = !dryRun && isUnpackedDist,
+            useCacheAsTargetFile = !dryRun && context.options.isUnpackedDist,
            dryRun = dryRun,
          )
        }
@@ -264,7 +263,7 @@ class JarPackager private constructor(
    val moduleOutDir = context.getModuleOutputDir(module)
    val extraExcludes = layout?.moduleExcludes?.get(moduleName) ?: emptyList()

-    val packToDir = isUnpackedDist &&
+    val packToDir = context.options.isUnpackedDist &&
                    !item.relativeOutputFile.contains('/') &&
                    (patchedContent.isEmpty() || (patchedContent.size == 1 && patchedContent.containsKey("META-INF/plugin.xml"))) &&
                    patchedDirs.isEmpty() &&
@@ -4,6 +4,7 @@
 package org.jetbrains.intellij.build.productRunner

 import org.jetbrains.intellij.build.BuildContext
+import org.jetbrains.intellij.build.VmProperties
 import org.jetbrains.intellij.build.dev.BuildRequest
 import org.jetbrains.intellij.build.dev.buildProduct
 import org.jetbrains.intellij.build.dev.getIdeSystemProperties
@@ -15,14 +16,15 @@ import kotlin.time.Duration.Companion.seconds
 * Only for use in build scripts, not for dev mode / integrations tests.
 * Use [BuildContext.createProductRunner] instead of calling this function directly.
 */
-internal suspend fun createDevModeProductRunner(context: BuildContext): IntellijProductRunner {
+internal suspend fun createDevModeProductRunner(context: BuildContext, additionalPluginModules: List<String> = emptyList()): IntellijProductRunner {
  var newClassPath: Collection<Path>? = null
  val homeDir = context.paths.projectHome
  val runDir = buildProduct(
    request = BuildRequest(
      isUnpackedDist = true,
+      writeCoreClasspath = false,
      platformPrefix = context.productProperties.platformPrefix ?: "idea",
-      additionalModules = emptyList(),
+      additionalModules = additionalPluginModules,
      projectDir = homeDir,
      devRootDir = context.paths.tempDir.resolve("dev-run"),
      jarCacheDir = homeDir.resolve("out/dev-run/jar-cache"),
@@ -42,14 +44,15 @@ private class DevModeProductRunner(
  private val homePath: Path,
  private val classPath: Collection<String>,
 ) : IntellijProductRunner {
-  override suspend fun runProduct(args: List<String>, additionalSystemProperties: Map<String, String>, isLongRunning: Boolean) {
+  override suspend fun runProduct(args: List<String>, additionalVmProperties: VmProperties, isLongRunning: Boolean) {
    runApplicationStarter(
      context = context,
-      ideClasspath = classPath,
-      arguments = args,
+      classpath = classPath,
+      args = args,
      timeout = if (isLongRunning) DEFAULT_TIMEOUT else 30.seconds,
      homePath = homePath,
-      systemProperties = additionalSystemProperties + getIdeSystemProperties(homePath),
+      vmProperties = additionalVmProperties + getIdeSystemProperties(homePath),
+      isFinalClassPath = true,
    )
  }
 }
@@ -8,9 +8,9 @@ import com.jetbrains.plugin.structure.base.utils.exists
 import io.opentelemetry.api.trace.Span
 import org.jetbrains.intellij.build.BuildContext
 import org.jetbrains.intellij.build.CompilationContext
+import org.jetbrains.intellij.build.VmProperties
 import org.jetbrains.intellij.build.impl.BuildUtils
 import org.jetbrains.intellij.build.impl.getCommandLineArgumentsForOpenPackages
-import org.jetbrains.intellij.build.impl.propertiesToJvmArgs
 import org.jetbrains.intellij.build.io.DEFAULT_TIMEOUT
 import org.jetbrains.intellij.build.io.runJava
 import java.nio.file.Files
@@ -28,31 +28,55 @@ import kotlin.time.Duration
 */
 suspend fun runApplicationStarter(
  context: BuildContext,
-  ideClasspath: Collection<String>,
-  arguments: List<String>,
-  systemProperties: Map<String, String> = emptyMap(),
+  classpath: Collection<String>,
+  args: List<String>,
+  vmProperties: VmProperties = VmProperties(emptyMap()),
  vmOptions: List<String> = emptyList(),
  homePath: Path = context.paths.projectHome,
  timeout: Duration = DEFAULT_TIMEOUT,
+  isFinalClassPath: Boolean = false,
 ) {
-  val tempFileNamePrefix = arguments.firstOrNull() ?: "appStarter"
+  val tempFileNamePrefix = args.firstOrNull() ?: "appStarter"
  val tempDir = createTempDirectory(context.paths.tempDir, tempFileNamePrefix)
  Files.createDirectories(tempDir)
-  val jvmArgs = mutableListOf<String>()
+
+  val jvmArgs = getCommandLineArgumentsForOpenPackages(context).toMutableList()
+
  val systemDir = tempDir.resolve("system")
-  BuildUtils.addVmProperty(jvmArgs, PathManager.PROPERTY_HOME_PATH, homePath.toString())
-  BuildUtils.addVmProperty(jvmArgs, "idea.system.path", systemDir.toString())
-  BuildUtils.addVmProperty(jvmArgs, "idea.config.path", "$tempDir/config")
-  BuildUtils.addVmProperty(jvmArgs, "idea.builtin.server.disabled", "true")
-  BuildUtils.addVmProperty(jvmArgs, "java.system.class.loader", "com.intellij.util.lang.PathClassLoader")
-  BuildUtils.addVmProperty(jvmArgs, "idea.platform.prefix", context.productProperties.platformPrefix)
-  jvmArgs.addAll(propertiesToJvmArgs(systemProperties))
+  jvmArgs.addAll(vmProperties.mutate {
+    put(PathManager.PROPERTY_HOME_PATH, homePath.toString())
+
+    put("idea.system.path", systemDir.toString())
+    put("idea.config.path", "$tempDir/config")
+
+    put("idea.builtin.server.disabled", "true")
+    put("java.system.class.loader", "com.intellij.util.lang.PathClassLoader")
+    context.productProperties.platformPrefix?.let {
+      put("idea.platform.prefix", it)
+    }
+
+    put("ij.dir.lock.debug", "true")
+    put("intellij.log.to.json.stdout", "true")
+  }.toJvmArgs())
  jvmArgs.addAll(vmOptions.takeIf { it.isNotEmpty() } ?: listOf("-Xmx2g"))
-  System.getProperty("intellij.build.${arguments.first()}.debug.port")?.let {
+  System.getProperty("intellij.build.${args.first()}.debug.port")?.let {
    jvmArgs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:$it")
  }

-  val effectiveIdeClasspath = LinkedHashSet(ideClasspath)
+  val effectiveIdeClasspath = if (isFinalClassPath) classpath else prepareFlatClasspath(classpath = classpath, tempDir = tempDir, context = context)
+  runJava(mainClass = context.ideMainClassName, args = args, jvmArgs = jvmArgs, classPath = effectiveIdeClasspath, javaExe = context.stableJavaExecutable, timeout = timeout) {
+    val logFile = findLogFile(systemDir)
+    if (logFile != null) {
+      val logFileToPublish = Files.createTempFile(tempFileNamePrefix, ".log")
+      Files.copy(logFile, logFileToPublish, StandardCopyOption.REPLACE_EXISTING)
+      context.notifyArtifactBuilt(logFileToPublish)
+      Span.current().addEvent("log file $logFileToPublish attached to build artifacts")
+    }
+  }
+}
+
+private fun prepareFlatClasspath(classpath: Collection<String>, tempDir: Path, context: BuildContext): LinkedHashSet<String> {
+  val effectiveIdeClasspath = LinkedHashSet(classpath)

  val additionalPluginPaths = context.productProperties.getAdditionalPluginPaths(context)
  val additionalPluginIds = LinkedHashSet<String>()
@@ -66,23 +90,10 @@ suspend fun runApplicationStarter(
      }
    }
  }
+
  disableCompatibleIgnoredPlugins(context = context, configDir = tempDir.resolve("config"), explicitlyEnabledPlugins = additionalPluginIds)
-  runJavaForIntellijModule(
-    context = context,
-    mainClass = context.ideMainClassName,
-    args = arguments,
-    jvmArgs = jvmArgs,
-    classPath = effectiveIdeClasspath.toList(),
-    timeout = timeout
-  ) {
-    val logFile = findLogFile(systemDir)
-    if (logFile != null) {
-      val logFileToPublish = Files.createTempFile(tempFileNamePrefix, ".log")
-      Files.copy(logFile, logFileToPublish, StandardCopyOption.REPLACE_EXISTING)
-      context.notifyArtifactBuilt(logFileToPublish)
-      Span.current().addEvent("log file $logFileToPublish attached to build artifacts")
-    }
-  }
+
+  return effectiveIdeClasspath
 }

@OptIn(ExperimentalPathApi::class)
@@ -1,10 +1,12 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.intellij.build.productRunner

+import org.jetbrains.intellij.build.VmProperties
+
 /**
 * Provides a way to run an IDE which distribution is currently being built by the build scripts.
- * This can be used to obtain some resources and include them in the distribution.
+ * This can be used to collect some resources and include them in the distribution.
 */
 interface IntellijProductRunner {
-  suspend fun runProduct(args: List<String>, additionalSystemProperties: Map<String, String> = emptyMap(), isLongRunning: Boolean = false)
+  suspend fun runProduct(args: List<String>, additionalVmProperties: VmProperties = VmProperties(emptyMap()), isLongRunning: Boolean = false)
 }
@@ -3,33 +3,35 @@ package org.jetbrains.intellij.build.productRunner

 import com.intellij.platform.runtime.repository.RuntimeModuleId
 import org.jetbrains.intellij.build.BuildContext
+import org.jetbrains.intellij.build.VmProperties
 import org.jetbrains.intellij.build.impl.VmOptionsGenerator
 import kotlin.io.path.pathString

 /**
- * Runs the product using the module-based loader which will take class-files from module output directories. 
+ * Runs the product using the module-based loader which will take class-files from module output directories.
 */
 internal class ModuleBasedProductRunner(private val rootModuleForModularLoader: String, private val context: BuildContext) : IntellijProductRunner {
-  override suspend fun runProduct(args: List<String>, additionalSystemProperties: Map<String, String>, isLongRunning: Boolean) {
-    val systemProperties = mutableMapOf(
-      "intellij.platform.runtime.repository.path" to context.originalModuleRepository.repositoryPath.pathString,
-      "intellij.platform.root.module" to rootModuleForModularLoader,
-      "intellij.platform.product.mode" to context.productProperties.productMode.id,
-      "idea.vendor.name" to context.applicationInfo.shortCompanyName,
+  override suspend fun runProduct(args: List<String>, additionalVmProperties: VmProperties, isLongRunning: Boolean) {
+    val systemProperties = VmProperties(
+      mapOf(
+        "intellij.platform.runtime.repository.path" to context.originalModuleRepository.repositoryPath.pathString,
+        "intellij.platform.root.module" to rootModuleForModularLoader,
+        "intellij.platform.product.mode" to context.productProperties.productMode.id,
+        "idea.vendor.name" to context.applicationInfo.shortCompanyName,
+      )
    )

    //todo include jna.boot.library.path, pty4j.preferred.native.folder and related properties? 
    val loaderModule = context.originalModuleRepository.repository.getModule(RuntimeModuleId.module("intellij.platform.runtime.loader"))
    val ideClasspath = loaderModule.moduleClasspath.map { it.pathString }
-    systemProperties.putAll(additionalSystemProperties)
    runApplicationStarter(
-      context,
-      ideClasspath = ideClasspath,
-      arguments = args,
-      systemProperties = systemProperties,
-      vmOptions =  VmOptionsGenerator.computeVmOptions(context) 
-                   + context.productProperties.additionalIdeJvmArguments
-                   + context.productProperties.getAdditionalContextDependentIdeJvmArguments(context),
+      context = context,
+      classpath = ideClasspath,
+      args = args,
+      vmProperties = systemProperties + additionalVmProperties,
+      vmOptions = VmOptionsGenerator.computeVmOptions(context) +
+                  context.productProperties.additionalIdeJvmArguments +
+                  context.productProperties.getAdditionalContextDependentIdeJvmArguments(context),
    )
  }
 }
@@ -53,7 +53,7 @@ internal fun readSearchableOptionIndex(baseDir: Path): SearchableOptionSetDescri
  }
 }

-suspend fun buildSearchableOptions(context: BuildContext, systemProperties: Map<String, String> = emptyMap()): SearchableOptionSetDescriptor? {
+suspend fun buildSearchableOptions(context: BuildContext, systemProperties: VmProperties = VmProperties(emptyMap())): SearchableOptionSetDescriptor? {
  return buildSearchableOptions(productRunner = context.createProductRunner(), context = context, systemProperties = systemProperties)
 }

@@ -63,7 +63,7 @@ suspend fun buildSearchableOptions(context: BuildContext, systemProperties: Map<
 internal suspend fun buildSearchableOptions(
  productRunner: IntellijProductRunner,
  context: BuildContext,
-  systemProperties: Map<String, String> = emptyMap(),
+  systemProperties: VmProperties = VmProperties(emptyMap()),
 ): SearchableOptionSetDescriptor? {
  val span = Span.current()
  if (context.isStepSkipped(BuildOptions.SEARCHABLE_OPTIONS_INDEX_STEP)) {
@@ -104,7 +104,7 @@ internal suspend fun buildSearchableOptions(
    // It'll process all UI elements in the `Settings` dialog and build an index for them.
    productRunner.runProduct(
      args = listOf("traverseUI", targetDirectory.toString(), "true"),
-      additionalSystemProperties = systemProperties + getSystemPropertiesForSearchableOptions(langTag),
+      additionalVmProperties = systemProperties + getSystemPropertiesForSearchableOptions(langTag),
      isLongRunning = true,
    )
  }
@@ -116,14 +116,14 @@ internal suspend fun buildSearchableOptions(
  return index
 }

-private fun getSystemPropertiesForSearchableOptions(langTag: String): Map<String, String> {
+private fun getSystemPropertiesForSearchableOptions(langTag: String): VmProperties {
  if (Locale.ENGLISH.toLanguageTag().equals(langTag)) {
-    return emptyMap()
+    return VmProperties(emptyMap())
  }
  else {
-    return mapOf(
+    return VmProperties(mapOf(
      "intellij.searchableOptions.i18n.enabled" to "true",
      "intellij.searchableOptions.i18n.locale" to langTag //TODO: use corresponding property after IJPL-148813
-    )
+    ))
  }
 }