From f3ab5a172edcd5429d9537a3f08634dd80c17d2c Mon Sep 17 00:00:00 2001 From: Nikolay Chashnikov Date: Thu, 27 Jun 2024 18:39:11 +0200 Subject: [PATCH] [jps build] provide a way to pass workspace model to the JPS build process in serialized way (IJPL-409) If newly introduced 'jps.build.use.workspace.model' registry option is enabled, required modules and libraries are added to the JPS build classpath, paths to global and project cache file for the workspace model are passed to the build process, and it loads the project model from them instead of reading the project configuration files. It's an experimental approach, and it isn't fully implemented yet: * it doesn't ensure that caches are up-to-date; * the cache currently cannot be loaded by the build process because it references entities and other classes aren't available in the build process (e.g. ScratchRootsEntity and VirtualFileUrlBridge). GitOrigin-RevId: 54a4a49566ca9dd0433c7a13251c2e28fc051e78 --- .../impl/intellij.java.compiler.impl.iml | 3 +++ .../compiler/server/BuildManager.java | 16 +++++++++++ .../impl/BuildProcessClasspathManager.kt | 27 ++++++++++++++++++- .../compiler/BaseCompilerTestCase.java | 7 ++++- java/java-impl/src/META-INF/JavaPlugin.xml | 1 + .../com/intellij/compiler/CompilerTests.kt | 24 +++++++++++++++++ .../testFramework/CompilerTester.java | 2 ++ .../src/GlobalWorkspaceModelCache.kt | 6 +++++ .../workspace/src/WorkspaceModelCache.kt | 3 +++ .../ide/impl/GlobalWorkspaceModelCacheImpl.kt | 7 ++++- .../ide/impl/WorkspaceModelCacheImpl.kt | 3 ++- 11 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 java/testFramework/src/com/intellij/compiler/CompilerTests.kt diff --git a/java/compiler/impl/intellij.java.compiler.impl.iml b/java/compiler/impl/intellij.java.compiler.impl.iml index db349a6e6bde..3b5b9853add1 100644 --- a/java/compiler/impl/intellij.java.compiler.impl.iml +++ b/java/compiler/impl/intellij.java.compiler.impl.iml @@ -47,6 +47,9 @@ + + + diff --git a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java index 468f9d92623c..407e6ec0c95e 100644 --- a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java +++ b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java @@ -78,6 +78,8 @@ import com.intellij.openapi.vfs.VirtualFileManager; import com.intellij.openapi.vfs.newvfs.BulkFileListener; import com.intellij.openapi.vfs.newvfs.events.VFileEvent; import com.intellij.openapi.wm.IdeFrame; +import com.intellij.platform.backend.workspace.GlobalWorkspaceModelCache; +import com.intellij.platform.backend.workspace.WorkspaceModelCache; import com.intellij.serviceContainer.AlreadyDisposedException; import com.intellij.ui.ComponentUtil; import com.intellij.util.*; @@ -1414,6 +1416,20 @@ public final class BuildManager implements Disposable { //noinspection SpellCheckingInspection cmdLine.addParameter("-Djna.noclasspath=true"); } + if (Registry.is("jps.build.use.workspace.model")) { + cmdLine.addParameter("-Dintellij.jps.use.workspace.model=true"); + WorkspaceModelCache cache = WorkspaceModelCache.getInstance(project); + GlobalWorkspaceModelCache globalCache = GlobalWorkspaceModelCache.getInstance(); + if (cache != null && globalCache != null) { + //todo ensure that caches are up-to-date or use a different way to pass serialized workspace model to the build process + cmdLine.addParameter("-Djps.workspace.storage.project.cache.path=" + cache.getCacheFile()); + cmdLine.addParameter("-Djps.workspace.storage.global.cache.path=" + globalCache.getCacheFile()); + } + else { + LOG.info("Workspace model caches aren't available and won't be used in the build process"); + } + cmdLine.addParameter("--add-opens=java.base/java.util=ALL-UNNAMED");//used by com.esotericsoftware.kryo.kryo5.serializers.CachedFields.addField + } if (sdkVersion != null) { if (sdkVersion.compareTo(JavaSdkVersion.JDK_1_9) < 0) { diff --git a/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt b/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt index 258487f8bbcf..1ff2ba9b6ec4 100644 --- a/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt +++ b/java/compiler/impl/src/com/intellij/compiler/server/impl/BuildProcessClasspathManager.kt @@ -1,8 +1,10 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.compiler.server.impl +import com.google.common.collect.HashBiMap import com.intellij.compiler.server.BuildProcessParametersProvider import com.intellij.compiler.server.CompileServerPlugin +import com.intellij.diagnostic.Activity import com.intellij.diagnostic.PluginException import com.intellij.ide.plugins.IdeaPluginDescriptor import com.intellij.ide.plugins.PluginManagerCore @@ -14,8 +16,12 @@ import com.intellij.openapi.diagnostic.logger import com.intellij.openapi.project.Project import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.io.FileUtilRt +import com.intellij.openapi.util.registry.Registry import com.intellij.openapi.util.text.StringUtil import com.intellij.openapi.util.text.StringUtilRt +import com.intellij.platform.diagnostic.telemetry.TracerLevel +import com.intellij.platform.workspace.jps.serialization.impl.JpsProjectSerializers +import com.intellij.platform.workspace.storage.WorkspaceEntity import com.intellij.util.PathUtilRt import com.intellij.util.io.URLUtil import com.intellij.util.text.VersionComparatorUtil @@ -42,7 +48,8 @@ class BuildProcessClasspathManager(parentDisposable: Disposable) { } fun getBuildProcessClasspath(project: Project): List { - val appClassPath = ClasspathBootstrap.getBuildProcessApplicationClasspath() + val appClassPath = ClasspathBootstrap.getBuildProcessApplicationClasspath() + + getAdditionalApplicationClasspath() val pluginClassPath = getBuildProcessPluginsClasspath(project) val rawClasspath = appClassPath + pluginClassPath synchronized(lastClasspathLock) { @@ -64,6 +71,24 @@ class BuildProcessClasspathManager(parentDisposable: Disposable) { } } + private fun getAdditionalApplicationClasspath(): List { + return if (Registry.`is`("jps.build.use.workspace.model")) { + listOf( + PathManager.getJarPathForClass(WorkspaceEntity::class.java)!!, //intellij.platform.workspace.storage + PathManager.getJarPathForClass(JpsProjectSerializers::class.java)!!, //intellij.platform.workspace.jps + PathManager.getJarPathForClass(TracerLevel::class.java)!!, //intellij.platform.diagnostic.telemetry + PathManager.getJarPathForClass(Activity::class.java)!!, //intellij.platform.diagnostic + PathManager.getJarPathForClass(HashBiMap::class.java)!!, //Guava + PathManager.getJarPathForClass(kotlinx.coroutines.CoroutineScope::class.java)!!, //kotlinx-coroutines-core + PathManager.getJarPathForClass(kotlin.reflect.full.NoSuchPropertyException::class.java)!!, //kotlin-reflect + PathManager.getJarPathForClass(io.opentelemetry.api.OpenTelemetry::class.java)!!, //opentelemetry + PathManager.getJarPathForClass(io.opentelemetry.context.propagation.ContextPropagators::class.java)!!, //opentelemetry + PathManager.getJarPathForClass(com.esotericsoftware.kryo.kryo5.Kryo::class.java)!!, //Kryo5 + ) + } + else emptyList() + } + /** * For internal use only, use [getBuildProcessClasspath] to get full classpath instead. */ diff --git a/java/compiler/tests/com/intellij/compiler/BaseCompilerTestCase.java b/java/compiler/tests/com/intellij/compiler/BaseCompilerTestCase.java index 352d26e55358..f5ff390c7394 100644 --- a/java/compiler/tests/com/intellij/compiler/BaseCompilerTestCase.java +++ b/java/compiler/tests/com/intellij/compiler/BaseCompilerTestCase.java @@ -13,7 +13,9 @@ import com.intellij.openapi.module.Module; import com.intellij.openapi.module.ModuleManager; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl; -import com.intellij.openapi.roots.*; +import com.intellij.openapi.roots.CompilerModuleExtension; +import com.intellij.openapi.roots.CompilerProjectExtension; +import com.intellij.openapi.roots.ModuleRootModificationUtil; import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; @@ -31,6 +33,7 @@ import com.intellij.util.io.DirectoryContentSpec; import com.intellij.util.io.DirectoryContentSpecKt; import com.intellij.util.io.TestFileSystemBuilder; import com.intellij.util.ui.UIUtil; +import com.intellij.workspaceModel.ide.impl.WorkspaceModelCacheImpl; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.util.JpsPathUtil; @@ -58,6 +61,7 @@ public abstract class BaseCompilerTestCase extends JavaModuleTestCase { protected void setUp() throws Exception { super.setUp(); CompilerTestUtil.enableExternalCompiler(); + WorkspaceModelCacheImpl.forceEnableCaching(getTestRootDisposable()); } protected void forceFSRescan() { @@ -236,6 +240,7 @@ public abstract class BaseCompilerTestCase extends JavaModuleTestCase { PlatformTestUtil.saveProject(myProject); CompilerTestUtil.saveApplicationSettings(); + CompilerTests.saveWorkspaceModelCaches(myProject); ApplicationManager.getApplication().invokeAndWait(() -> { CompilerTester.enableDebugLogging(); action.accept(new CompileStatusNotification() { diff --git a/java/java-impl/src/META-INF/JavaPlugin.xml b/java/java-impl/src/META-INF/JavaPlugin.xml index 754fe61831a0..9128616ee57e 100644 --- a/java/java-impl/src/META-INF/JavaPlugin.xml +++ b/java/java-impl/src/META-INF/JavaPlugin.xml @@ -389,6 +389,7 @@ + diff --git a/java/testFramework/src/com/intellij/compiler/CompilerTests.kt b/java/testFramework/src/com/intellij/compiler/CompilerTests.kt new file mode 100644 index 000000000000..7ae8e66d2bbf --- /dev/null +++ b/java/testFramework/src/com/intellij/compiler/CompilerTests.kt @@ -0,0 +1,24 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:JvmName("CompilerTests") +package com.intellij.compiler + +import com.intellij.openapi.project.Project +import com.intellij.openapi.util.registry.Registry +import com.intellij.platform.backend.workspace.GlobalWorkspaceModelCache.Companion.getInstance +import com.intellij.platform.backend.workspace.WorkspaceModel +import com.intellij.platform.backend.workspace.WorkspaceModelCache +import com.intellij.platform.ide.progress.runWithModalProgressBlocking + +fun saveWorkspaceModelCaches(project: Project) { + if (Registry.`is`("jps.build.use.workspace.model")) { + val cache = WorkspaceModelCache.getInstance(project) + checkNotNull(cache) { "Workspace model cache is not enabled for project" } + cache.setVirtualFileUrlManager(WorkspaceModel.getInstance(project).getVirtualFileUrlManager()) + cache.saveCacheNow() + val globalCache = getInstance() + checkNotNull(globalCache) { "Workspace model cache is not enabled for global storage" } + runWithModalProgressBlocking(project, "Save workspace model cache") { + globalCache.saveCacheNow() + } + } +} diff --git a/java/testFramework/src/com/intellij/testFramework/CompilerTester.java b/java/testFramework/src/com/intellij/testFramework/CompilerTester.java index 15e4f5dc55b5..668c5b374d71 100644 --- a/java/testFramework/src/com/intellij/testFramework/CompilerTester.java +++ b/java/testFramework/src/com/intellij/testFramework/CompilerTester.java @@ -3,6 +3,7 @@ package com.intellij.testFramework; import com.intellij.compiler.CompilerManagerImpl; import com.intellij.compiler.CompilerTestUtil; +import com.intellij.compiler.CompilerTests; import com.intellij.compiler.server.BuildManager; import com.intellij.diagnostic.ThreadDumper; import com.intellij.execution.wsl.WslPath; @@ -194,6 +195,7 @@ public final class CompilerTester { ErrorReportingCallback callback = new ErrorReportingCallback(semaphore); PlatformTestUtil.saveProject(getProject(), false); CompilerTestUtil.saveApplicationSettings(); + CompilerTests.saveWorkspaceModelCaches(getProject()); EdtTestUtil.runInEdtAndWait(() -> { // for now directory based project is used for external storage if (!ProjectKt.isDirectoryBased(myProject)) { diff --git a/platform/backend/workspace/src/GlobalWorkspaceModelCache.kt b/platform/backend/workspace/src/GlobalWorkspaceModelCache.kt index 5aa9229a391b..cd34c836ab4e 100644 --- a/platform/backend/workspace/src/GlobalWorkspaceModelCache.kt +++ b/platform/backend/workspace/src/GlobalWorkspaceModelCache.kt @@ -6,16 +6,22 @@ import com.intellij.openapi.components.service import com.intellij.platform.workspace.storage.MutableEntityStorage import com.intellij.platform.workspace.storage.url.VirtualFileUrlManager import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.TestOnly +import java.nio.file.Path @ApiStatus.Internal public interface GlobalWorkspaceModelCache { + public val cacheFile: Path public fun loadCache(): MutableEntityStorage? public fun scheduleCacheSave() + @TestOnly + public suspend fun saveCacheNow(); public fun invalidateCaches() public fun setVirtualFileUrlManager(vfuManager: VirtualFileUrlManager) public companion object { + @JvmStatic public fun getInstance(): GlobalWorkspaceModelCache? = ApplicationManager.getApplication().getService(GlobalWorkspaceModelCache::class.java) } diff --git a/platform/backend/workspace/src/WorkspaceModelCache.kt b/platform/backend/workspace/src/WorkspaceModelCache.kt index 5b33119d4def..6a8d66fc13bf 100644 --- a/platform/backend/workspace/src/WorkspaceModelCache.kt +++ b/platform/backend/workspace/src/WorkspaceModelCache.kt @@ -6,6 +6,7 @@ import com.intellij.platform.workspace.storage.MutableEntityStorage import com.intellij.platform.workspace.storage.url.VirtualFileUrlManager import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.TestOnly +import java.nio.file.Path /** * Plugins aren't supposed to use this interface directly, the cache is loaded and saved automatically by [WorkspaceModel]. @@ -13,6 +14,7 @@ import org.jetbrains.annotations.TestOnly @ApiStatus.Internal public interface WorkspaceModelCache { public val enabled: Boolean + public val cacheFile: Path public fun loadCache(): MutableEntityStorage? public fun loadUnloadedEntitiesCache(): MutableEntityStorage? @@ -26,6 +28,7 @@ public interface WorkspaceModelCache { public fun saveCacheNow() public companion object { + @JvmStatic public fun getInstance(project: Project): WorkspaceModelCache? = project.getService(WorkspaceModelCache::class.java)?.takeIf { it.enabled } } } diff --git a/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/GlobalWorkspaceModelCacheImpl.kt b/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/GlobalWorkspaceModelCacheImpl.kt index 57bbbee48fad..9d314e5527fa 100644 --- a/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/GlobalWorkspaceModelCacheImpl.kt +++ b/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/GlobalWorkspaceModelCacheImpl.kt @@ -14,13 +14,14 @@ import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.debounce import java.nio.file.Files +import java.nio.file.Path import java.util.concurrent.atomic.AtomicBoolean import kotlin.time.Duration.Companion.milliseconds @OptIn(FlowPreview::class) internal class GlobalWorkspaceModelCacheImpl(coroutineScope: CoroutineScope) : GlobalWorkspaceModelCache { private val saveRequests = MutableSharedFlow(replay=1, onBufferOverflow = BufferOverflow.DROP_OLDEST) - private val cacheFile by lazy { PathManager.getSystemDir().resolve("$DATA_DIR_NAME/cache.data") } + override val cacheFile: Path by lazy { PathManager.getSystemDir().resolve("$DATA_DIR_NAME/cache.data") } private lateinit var virtualFileUrlManager: VirtualFileUrlManager private val urlRelativizer = @@ -60,6 +61,10 @@ internal class GlobalWorkspaceModelCacheImpl(coroutineScope: CoroutineScope) : G check(saveRequests.tryEmit(Unit)) } + override suspend fun saveCacheNow() { + doCacheSaving() + } + override fun invalidateCaches() { Companion.invalidateCaches() } diff --git a/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/WorkspaceModelCacheImpl.kt b/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/WorkspaceModelCacheImpl.kt index 8e39b2bc4a01..ef3e3e681fc4 100644 --- a/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/WorkspaceModelCacheImpl.kt +++ b/platform/platform-impl/src/com/intellij/workspaceModel/ide/impl/WorkspaceModelCacheImpl.kt @@ -46,7 +46,7 @@ class WorkspaceModelCacheImpl(private val project: Project, coroutineScope: Coro private val saveRequests = MutableSharedFlow(replay=1, onBufferOverflow = BufferOverflow.DROP_OLDEST) private lateinit var virtualFileUrlManager: VirtualFileUrlManager - private val cacheFile by lazy { initCacheFile() } + override val cacheFile by lazy { initCacheFile() } private val unloadedEntitiesCacheFile by lazy { project.getProjectDataPath(DATA_DIR_NAME).resolve("unloaded-entities-cache.data") } private val invalidateProjectCacheMarkerFile by lazy { project.getProjectDataPath(DATA_DIR_NAME).resolve(".invalidate") } @@ -177,6 +177,7 @@ class WorkspaceModelCacheImpl(private val project: Project, coroutineScope: Coro invalidateCaches(cachesInvalidated, invalidateCachesMarkerFile) } + @JvmStatic fun forceEnableCaching(disposable: Disposable) { forceEnableCaching = true Disposer.register(disposable) { forceEnableCaching = false }