do not require building a project to run product in dev mode (part 2)

GitOrigin-RevId: f99fe5b22300aef6d3e9d89c6c71c93030b51b2a

platform/build-scripts/codeOptimizer/src/optimizeLib.kt

@@ -26,6 +26,14 @@ internal object FastutilInstall {
   }
 }
 
+@Suppress("unused")
+internal object FastutilDeploy {
+  @JvmStatic
+  fun main(args: Array<String>) {
+    publishToMaven(fastUtil, deploy = true)
+  }
+}
+
 @Suppress("SpellCheckingInspection", "RedundantSuppression", "SameParameterValue")
 private fun publishToMaven(
   @Suppress("SameParameterValue") lib: LibDescriptor,

platform/build-scripts/dev-server/src/IdeBuilder.kt

@@ -18,8 +18,8 @@ import kotlinx.coroutines.*
 import kotlinx.coroutines.selects.onTimeout
 import kotlinx.coroutines.selects.select
 import kotlinx.serialization.ExperimentalSerializationApi
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.encodeToByteArray
+import kotlinx.serialization.builtins.SetSerializer
+import kotlinx.serialization.builtins.serializer
 import kotlinx.serialization.protobuf.ProtoBuf
 import org.jetbrains.intellij.build.*
 import org.jetbrains.intellij.build.TraceManager.spanBuilder
@@ -195,7 +195,8 @@ internal suspend fun buildProduct(productConfiguration: ProductConfiguration, re
 private suspend fun compileIfNeeded(context: BuildContext) {
   val port = System.getProperty("compile.server.port")?.toIntOrNull()
   val project = System.getProperty("compile.server.project")
-  if (port == null || project == null) {
+  val token = System.getProperty("compile.server.token")
+  if (port == null || project == null || token == null) {
     return
   }
 
@@ -209,15 +210,12 @@ private suspend fun compileIfNeeded(context: BuildContext) {
     result
   }
 
-  val url = "http://127.0.0.1:$port/devkit/build?project-hash=$project&skip-save=true"
+  val url = "http://127.0.0.1:$port/devkit/make?project-hash=$project&token=$token"
   TraceManager.flush()
   spanBuilder("compile modules").setAttribute("url", url).useWithScope {
     coroutineScope {
       val task = launch {
-        postData(url, ProtoBuf.encodeToByteArray(listOf(BuildScopeDescription(
-          targetType = "java-production",
-          targetIds = modulesToCompile,
-        ))))
+        postData(url, ProtoBuf.encodeToByteArray(SetSerializer(String.serializer()), modulesToCompile))
       }
 
       var count = 0
@@ -559,11 +557,4 @@ private fun getCommunityHomePath(homePath: Path): BuildDependenciesCommunityRoot
     }
   }
   return BuildDependenciesCommunityRoot(if (Files.isDirectory(communityDotIdea)) communityDotIdea.parent else homePath)
-}
-
-@Serializable
-private data class BuildScopeDescription(
-  val targetType: String,
-  val targetIds: Collection<String>,
-  val forceBuild: Boolean = false,
-)
+}

plugins/devkit/devkit-core/intellij.devkit.core.iml

@@ -78,5 +78,6 @@
     <orderEntry type="module" module-name="intellij.json" />
     <orderEntry type="library" name="kotlinx-serialization-protobuf" level="project" />
     <orderEntry type="module" module-name="intellij.platform.builtInServer" />
+    <orderEntry type="library" name="caffeine" level="project" />
   </component>
 </module>

plugins/devkit/devkit-core/resources/intellij.devkit.core.xml

@@ -22,6 +22,7 @@
     <automaticRenamerFactory implementation="org.jetbrains.idea.devkit.refactoring.InspectionAutomaticRenamerFactory"/>
     <httpRequestHandler implementation="org.jetbrains.idea.devkit.requestHandlers.HttpDebugListener"/>
     <httpRequestHandler implementation="org.jetbrains.idea.devkit.requestHandlers.BuildHttpRequestHandler"/>
+    <httpRequestHandler implementation="org.jetbrains.idea.devkit.requestHandlers.CompileHttpRequestHandler"/>
 
     <junitPatcher implementation="org.jetbrains.idea.devkit.run.JUnitDevKitPatcher"/>
     <runConfigurationExtension implementation="org.jetbrains.idea.devkit.run.DevKitApplicationPatcher"/>

plugins/devkit/devkit-core/src/requestHandlers/BuildHttpRequestHandler.kt

@@ -4,7 +4,6 @@
 package org.jetbrains.idea.devkit.requestHandlers
 
 import com.intellij.compiler.CompilerMessageImpl
-import com.intellij.compiler.impl.CompileDriver
 import com.intellij.compiler.impl.CompileScopeUtil
 import com.intellij.compiler.impl.CompositeScope
 import com.intellij.openapi.application.EDT
@@ -21,9 +20,7 @@ import com.intellij.openapi.diagnostic.logger
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ProjectManager
 import com.intellij.project.stateStore
-import io.netty.buffer.ByteBuf
 import io.netty.buffer.ByteBufInputStream
-import io.netty.buffer.ByteBufUtil
 import io.netty.buffer.Unpooled
 import io.netty.channel.Channel
 import io.netty.channel.ChannelHandlerContext
@@ -32,13 +29,12 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.SerializationException
-import kotlinx.serialization.decodeFromByteArray
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.decodeFromStream
-import kotlinx.serialization.protobuf.ProtoBuf
 import org.jetbrains.ide.HttpRequestHandler
 import org.jetbrains.idea.devkit.util.PsiUtil
 import org.jetbrains.io.send
@@ -60,54 +56,49 @@ private class BuildHttpRequestHandler : HttpRequestHandler() {
     return request.method() == HttpMethod.POST && request.uri().startsWith(PREFIX)
   }
 
-  @Suppress("OPT_IN_USAGE")
   override fun process(urlDecoder: QueryStringDecoder, request: FullHttpRequest, context: ChannelHandlerContext): Boolean {
     val query = urlDecoder.parameters()
     val projectHash = query.get("project-hash")?.firstOrNull()
-    val skipSave = query.get("skip-save")?.firstOrNull()?.toBoolean() ?: false
     val project: Project?
-    val decoder: (ByteBuf) -> List<BuildScopeDescription>
     if (projectHash == null) {
       val projectPath = query.get("project-path")?.firstOrNull()
       project = ProjectManager.getInstance().openProjects.find {
         it.stateStore.projectBasePath.invariantSeparatorsPathString == projectPath
       }
-      decoder = { Json.decodeFromStream<List<BuildScopeDescription>>(ByteBufInputStream(it)) }
     }
     else {
       project = ProjectManager.getInstance().findOpenProjectByHash(projectHash)
-      decoder = { ProtoBuf.decodeFromByteArray<List<BuildScopeDescription>>(ByteBufUtil.getBytes(it)) }
     }
-
     if (project == null) {
       LOG.info("Project is not found (query=$query)")
       HttpResponseStatus.NOT_FOUND.send(context.channel(), request)
       return true
     }
 
-    val scopeDescriptions = try {
-      decoder(request.content())
-    }
-    catch (e: SerializationException) {
-      LOG.info(e)
-      HttpResponseStatus.BAD_REQUEST.send(context.channel(), request)
-      return true
-    }
-
     if (!PsiUtil.isIdeaProject(project)) {
       LOG.info("Build requests are currently handled for 'intellij' project only, so request won't be processed (query=$query)")
       HttpResponseStatus.NOT_FOUND.send(context.channel(), request)
       return true
     }
 
-    project.service<BuildRequestAsyncHandler>().handle(request, context.channel(), scopeDescriptions, skipSave)
+    project.service<BuildRequestAsyncHandler>().handle(request, context.channel())
     return true
   }
 }
 
 @Service(Service.Level.PROJECT)
 private class BuildRequestAsyncHandler(private val project: Project, private val coroutineScope: CoroutineScope) {
-  fun handle(request: FullHttpRequest, channel: Channel, scopeDescriptions: List<BuildScopeDescription>, skipSave: Boolean) {
+  @OptIn(ExperimentalSerializationApi::class)
+  fun handle(request: FullHttpRequest, channel: Channel) {
+    val scopeDescriptions = try {
+      Json.decodeFromStream<List<BuildScopeDescription>>(ByteBufInputStream(request.content()))
+    }
+    catch (e: SerializationException) {
+      LOG.info(e)
+      HttpResponseStatus.BAD_REQUEST.send(channel, request)
+      return
+    }
+
     coroutineScope.launch {
       val compilerManager = project.serviceAsync<CompilerManager>()
       val scope = readAction {
@@ -117,17 +108,7 @@ private class BuildRequestAsyncHandler(private val project: Project, private val
         })
         base
       }
-
-      // the proper implementation - convert CompilerDriver to kotlin and use coroutines/modern API
-      if (skipSave) {
-        scope.putUserData(CompileDriver.SKIP_SAVE, true)
-      }
-
-      var context = Dispatchers.EDT
-      if (skipSave) {
-        context += ModalityState.any().asContextElement()
-      }
-      withContext(context) {
+      withContext(Dispatchers.EDT + ModalityState.any().asContextElement()) {
         compilerManager.make(scope, CompileStatusNotification { aborted, errors, _, compileContext ->
           when {
             aborted -> HttpResponseStatus.INTERNAL_SERVER_ERROR.send(channel, request, description = "Build cancelled")
@@ -155,7 +136,7 @@ private class BuildRequestAsyncHandler(private val project: Project, private val
 @Serializable
 private data class BuildScopeDescription(
   val targetType: String,
-  val targetIds: Collection<String>,
+  val targetIds: List<String>,
   val forceBuild: Boolean = false,
 )
 

plugins/devkit/devkit-core/src/requestHandlers/CompileHttpRequestHandler.kt

@@ -0,0 +1,115 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+@file:Suppress("ReplaceGetOrSet")
+
+package org.jetbrains.idea.devkit.requestHandlers
+
+import com.github.benmanes.caffeine.cache.Caffeine
+import com.intellij.openapi.components.Service
+import com.intellij.openapi.components.service
+import com.intellij.openapi.diagnostic.logger
+import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.project.ProjectManager
+import com.intellij.task.ProjectTaskManager
+import com.intellij.util.io.DigestUtil
+import io.netty.buffer.ByteBufUtil
+import io.netty.buffer.Unpooled
+import io.netty.channel.ChannelHandlerContext
+import io.netty.handler.codec.http.*
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.decodeFromByteArray
+import kotlinx.serialization.protobuf.ProtoBuf
+import org.jetbrains.ide.HttpRequestHandler
+import org.jetbrains.idea.devkit.util.PsiUtil
+import org.jetbrains.io.send
+import java.util.concurrent.TimeUnit
+
+private const val PREFIX = "/devkit/make"
+private val LOG = logger<CompileHttpRequestHandler>()
+
+@Service
+internal class CompileHttpRequestHandlerToken {
+  // build of dev-mode make take a while, so, 15 minutes
+  // (run configuration -> IDE make for configuration is started -> external process started to execute)
+  private val tokens = Caffeine.newBuilder().expireAfterAccess(15, TimeUnit.MINUTES).build<String, Boolean>()
+
+  fun acquireToken(): String {
+    var token = tokens.asMap().keys.firstOrNull()
+    if (token == null) {
+      token = DigestUtil.randomToken()
+      tokens.put(token, true)
+    }
+    return token
+  }
+
+  fun hasToken(token: String): Boolean = tokens.getIfPresent(token) == true
+}
+
+/**
+ * Starts JPS build for targets passed in the content in JSON format (array of [BuildScopeDescription] objects).
+ *
+ * Currently, it's enabled for 'intellij' project only, and can be used to build additional required modules when a developer runs a test or
+ * an application from the IDE.
+ */
+@Suppress("unused")
+private class CompileHttpRequestHandler : HttpRequestHandler() {
+  override fun isSupported(request: FullHttpRequest): Boolean {
+    return request.method() == HttpMethod.POST && request.uri().startsWith(PREFIX)
+  }
+
+  @Suppress("OPT_IN_USAGE")
+  override fun process(urlDecoder: QueryStringDecoder, request: FullHttpRequest, context: ChannelHandlerContext): Boolean {
+    val channel = context.channel()
+
+    val query = urlDecoder.parameters()
+    val token = query.get("token")?.firstOrNull()
+    if (token == null || !service<CompileHttpRequestHandlerToken>().hasToken(token)) {
+      HttpResponseStatus.FORBIDDEN.send(channel, request)
+      return true
+    }
+
+    val projectHash = query.get("project-hash")?.firstOrNull()
+    val project = ProjectManager.getInstance().findOpenProjectByHash(projectHash)
+    if (project == null) {
+      LOG.info("Project is not found (query=$query)")
+      HttpResponseStatus.NOT_FOUND.send(channel, request)
+      return true
+    }
+
+    if (!PsiUtil.isIdeaProject(project)) {
+      LOG.info("Build requests are currently handled for 'intellij' project only, so request won't be processed (query=$query)")
+      HttpResponseStatus.FORBIDDEN.send(channel, request)
+      return true
+    }
+
+    val modules = try {
+      ProtoBuf.decodeFromByteArray<List<String>>(ByteBufUtil.getBytes(request.content()))
+    }
+    catch (e: SerializationException) {
+      LOG.info(e)
+      HttpResponseStatus.BAD_REQUEST.send(channel, request)
+      return true
+    }
+
+    val projectTaskManager = ProjectTaskManager.getInstance(project)
+    val moduleManager = ModuleManager.getInstance(project)
+    val projectTask = projectTaskManager.createModulesBuildTask(
+      /* modules = */ modules.map { moduleManager.findModuleByName(it) }.toTypedArray(),
+      /* isIncrementalBuild = */ true,
+      /* includeDependentModules = */ false,
+      /* includeRuntimeDependencies = */ false,
+      /* includeTests = */ false,
+    )
+    projectTaskManager.run(projectTask)
+      .onSuccess { taskResult ->
+        val content = Unpooled.copiedBuffer("{hasErrors: ${taskResult.hasErrors()}, isAborted: ${taskResult.isAborted}}", Charsets.UTF_8)
+        val response = DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, content)
+        response.headers().set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON)
+        response.send(channel, request)
+      }
+      .onError { error ->
+        HttpResponseStatus.INTERNAL_SERVER_ERROR.send(channel, request, description = "Build cancelled")
+        LOG.warn(error)
+      }
+    return true
+  }
+}

plugins/devkit/devkit-core/src/run/DevKitApplicationPatcher.kt

@@ -9,11 +9,13 @@ import com.intellij.execution.configurations.ParametersList
 import com.intellij.execution.configurations.RunConfigurationBase
 import com.intellij.execution.configurations.RunnerSettings
 import com.intellij.openapi.application.PathManager
+import com.intellij.openapi.components.service
 import com.intellij.openapi.util.io.FileUtilRt
 import com.intellij.util.PlatformUtils
 import com.intellij.util.lang.UrlClassLoader
 import com.intellij.util.system.CpuArch
 import org.jetbrains.ide.BuiltInServerManager
+import org.jetbrains.idea.devkit.requestHandlers.CompileHttpRequestHandlerToken
 import org.jetbrains.idea.devkit.util.PsiUtil
 import java.nio.file.Files
 import java.nio.file.NoSuchFileException
@@ -85,6 +87,7 @@ internal class DevKitApplicationPatcher : RunConfigurationExtension() {
     if (appConfiguration.beforeRunTasks.none { it.providerId === MakeProjectStepBeforeRun.ID }) {
       vmParameters.addProperty("compile.server.port", BuiltInServerManager.getInstance().port.toString())
       vmParameters.addProperty("compile.server.project", project.locationHash)
+      vmParameters.addProperty("compile.server.token", service<CompileHttpRequestHandlerToken>().acquireToken())
     }
 
     var productClassifier = vmParameters.getPropertyValue("idea.platform.prefix")