From 366f9dcc27dbb83179570822d4d861301a70ad6d Mon Sep 17 00:00:00 2001 From: "Andrei.Kuznetsov" Date: Thu, 26 Sep 2024 18:01:21 +0200 Subject: [PATCH] IJPL-157852,IJPL-16008: introduce `ide.workspace.model.write.external.files.directly` registry option When enabled, WSM bypasses module's configuration stores on project save and writes module xml files directly to 'cache' directory (iml files are still saved through module's configuration stores) GitOrigin-RevId: de34f4534b62ebb5bba957651c36e49d9d9bbf3c --- .../src/ProjectStoreBridge.kt | 139 ++++++++++++++++-- .../src/META-INF/WorkspaceModelExtensions.xml | 3 + 2 files changed, 127 insertions(+), 15 deletions(-) diff --git a/platform/configuration-store-impl/src/ProjectStoreBridge.kt b/platform/configuration-store-impl/src/ProjectStoreBridge.kt index ff687691cbcb..005183a52196 100644 --- a/platform/configuration-store-impl/src/ProjectStoreBridge.kt +++ b/platform/configuration-store-impl/src/ProjectStoreBridge.kt @@ -22,6 +22,7 @@ import com.intellij.openapi.util.Pair import com.intellij.openapi.util.SystemInfoRt import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.io.FileUtilRt +import com.intellij.openapi.util.registry.Registry import com.intellij.openapi.vfs.VirtualFileManager import com.intellij.platform.backend.workspace.WorkspaceModelCache import com.intellij.platform.diagnostic.telemetry.helpers.MillisecondsMeasurer @@ -29,6 +30,7 @@ import com.intellij.platform.workspace.jps.JpsProjectConfigLocation import com.intellij.platform.workspace.jps.serialization.impl.JpsFileContentWriter import com.intellij.platform.workspace.jps.serialization.impl.isExternalModuleFile import com.intellij.project.stateStore +import com.intellij.util.LineSeparator import com.intellij.util.PathUtil import com.intellij.util.PathUtilRt import com.intellij.util.containers.HashingStrategy @@ -40,18 +42,26 @@ import com.intellij.workspaceModel.ide.impl.jps.serialization.JpsProjectModelSyn import com.intellij.workspaceModel.ide.impl.jps.serialization.ProjectStoreWithJpsContentReader import com.intellij.workspaceModel.ide.impl.jpsMetrics import io.opentelemetry.api.metrics.Meter +import org.jdom.Attribute import org.jdom.Element import org.jetbrains.annotations.ApiStatus import org.jetbrains.jps.util.JpsPathUtil import java.io.IOException import java.nio.file.Path +import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.function.Supplier +import kotlin.Throws +import kotlin.io.path.Path import kotlin.io.path.invariantSeparatorsPathString +import kotlin.io.path.nameWithoutExtension @ApiStatus.Internal open class ProjectWithModuleStoreImpl(project: Project) : ProjectStoreImpl(project), ProjectStoreWithJpsContentReader { + private fun shouldWriteExternalFilesDirectly(): Boolean = Registry.`is`("ide.workspace.model.write.external.files.directly", false) + final override suspend fun saveModules( saveSessions: MutableList, saveResult: SaveResult, @@ -60,7 +70,11 @@ open class ProjectWithModuleStoreImpl(project: Project) : ProjectStoreImpl(proje ) { projectSessionManager as ProjectWithModulesSaveSessionProducerManager - val writer = JpsStorageContentWriter(session = projectSessionManager, store = this, project = project) + val writer = if (shouldWriteExternalFilesDirectly()) + HalfDirectJpsStorageContentWriter(session = projectSessionManager, store = this, project = project) + else + DelegatingJpsStorageContentWriter(session = projectSessionManager, store = this, project = project) + project.serviceAsync().saveChangedProjectEntities(writer) (project.serviceAsync() as WorkspaceModelCacheImpl).doCacheSavingOnProjectClose() @@ -71,6 +85,8 @@ open class ProjectWithModuleStoreImpl(project: Project) : ProjectStoreImpl(proje projectSessionManager.commitComponents(moduleStore = moduleStore, moduleSaveSessionManager = moduleSessionManager) moduleSessionManager.collectSaveSessions(saveSessions) } + + writer.writeFilesToDisk() } final override fun createSaveSessionProducerManager(): ProjectSaveSessionProducerManager { @@ -82,10 +98,106 @@ open class ProjectWithModuleStoreImpl(project: Project) : ProjectStoreImpl(proje } } -private class JpsStorageContentWriter( - private val session: ProjectWithModulesSaveSessionProducerManager, - private val store: IProjectStore, - private val project: Project, +private class DelegatingJpsStorageContentWriter(session: ProjectWithModulesSaveSessionProducerManager, store: IProjectStore, project: Project) + : JpsStorageContentWriter(session, store, project) { + + override fun saveInternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) { + session.setModuleComponentState(imlFilePath = filePath, componentName = componentName, componentTag = componentTag) + } + + override fun saveExternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) { + session.setExternalModuleComponentState( + moduleFileName = FileUtilRt.getNameWithoutExtension(PathUtilRt.getFileName(filePath)), + componentName = componentName, + componentTag = componentTag, + ) + } + + // This writer has nothing to write. All the components will be committed to component stores, and files will be updated from these stores later + override suspend fun writeFilesToDisk() = Unit +} + +// Half- because we store external xml files directly, and internal iml files via stores +// (because we want the store to generate VFS events in order to keep iml files up-to-date) +private class HalfDirectJpsStorageContentWriter(session: ProjectWithModulesSaveSessionProducerManager, store: IProjectStore, project: Project) + : JpsStorageContentWriter(session, store, project) { + + // we expect that externalFileComponents might be accessed concurrently from different threads, but each file (=value) + // is only accessed from a single thread (i.e. one file is populated from one thread, but several different files + // might be populated from different threads). + private val externalFileComponents: MutableMap = ConcurrentHashMap() + + override fun saveInternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) { + session.setModuleComponentState(imlFilePath = filePath, componentName = componentName, componentTag = componentTag) + } + + override fun saveExternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) { + // componentTag == null is to remove component from iml/xml. We don't care about removing, because we always start with an empty file + if (componentTag != null) { + val fileComponents = externalFileComponents.getOrPut(filePath) { WritableImlFileContent(filePath) } + if (componentTag.name != "component") { + componentTag.attributes.add(0, Attribute("name", componentTag.name)) + componentTag.name = "component" + } + fileComponents.saveComponent(componentName, componentTag) + } + } + + @Throws(IOException::class) + override suspend fun writeFilesToDisk() { + val exceptions = CopyOnWriteArrayList() + val moduleManager = project.serviceAsync() + + // todo (IJPL-157852): we can use several threads + externalFileComponents.forEach { (_, components) -> + try { + components.flush(moduleManager) + } + catch (e: IOException) { + exceptions.add(e) + } + } + + if (exceptions.isNotEmpty()) { + throw IOException().also { e -> + exceptions.forEach(e::addSuppressed) + } + } + } + + // This class is not thread-safe. Same file should not be populated from different threads. + private class WritableImlFileContent( + private val filePath: String, + ) { + private val components = /*sorted*/TreeMap() + + fun saveComponent(componentName: String, componentTag: Element?) { + if (componentTag != null) { + components[componentName] = componentTag + } + } + + fun flush(moduleManager: ModuleManager) { + val path = Path(filePath) + val moduleName = path.nameWithoutExtension + val module = moduleManager.findModuleByName(moduleName) + val pathMacroManager = if (module != null) { + PathMacroManager.getInstance(module) + } + else { + LOG.error("Could not find module with name $moduleName. Paths will not be substituted.") + null + } + val writer = XmlDataWriter("module", components.values.toList(), rootAttributes = emptyMap(), pathMacroManager, filePath) + writer.writeTo(path, requestor = null, LineSeparator.getSystemLineSeparator(), false) + } + } +} + +private abstract class JpsStorageContentWriter( + val session: ProjectWithModulesSaveSessionProducerManager, + val store: IProjectStore, + val project: Project, ) : JpsFileContentWriter { override fun saveComponent(fileUrl: String, componentName: String, componentTag: Element?) { val filePath = JpsPathUtil.urlToPath(fileUrl) @@ -100,17 +212,11 @@ private class JpsStorageContentWriter( } } - private fun saveInternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) { - session.setModuleComponentState(imlFilePath = filePath, componentName = componentName, componentTag = componentTag) - } + abstract fun saveInternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) - private fun saveExternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) { - session.setExternalModuleComponentState( - moduleFileName = FileUtilRt.getNameWithoutExtension(PathUtilRt.getFileName(filePath)), - componentName = componentName, - componentTag = componentTag, - ) - } + abstract fun saveExternalFileModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) + + abstract suspend fun writeFilesToDisk() private fun saveNonModuleComponent(filePath: @NlsSafe String, componentName: String, componentTag: Element?) { val stateStorage = getProjectStateStorage(filePath = filePath, store = store, project = project) @@ -126,6 +232,9 @@ private class JpsStorageContentWriter( override fun getReplacePathMacroMap(fileUrl: String): PathMacroMap { val filePath = JpsPathUtil.urlToPath(fileUrl) if (FileUtilRt.extensionEquals(filePath, "iml") || isExternalModuleFile(filePath)) { + // TODO: this is not correct for external xml files, because $MODULE_DIR$ should always point + // to a dir with `iml` file. (filePath points to xml file when this method is called for external module files, + // so $MODULE_DIR$ resolves to incorrect dir here) return ModulePathMacroManager.createInstance(project::getProjectFilePath, Supplier { filePath }).replacePathMap } else { diff --git a/platform/workspace/jps/src/META-INF/WorkspaceModelExtensions.xml b/platform/workspace/jps/src/META-INF/WorkspaceModelExtensions.xml index c8c6358f9867..15dfd999720b 100644 --- a/platform/workspace/jps/src/META-INF/WorkspaceModelExtensions.xml +++ b/platform/workspace/jps/src/META-INF/WorkspaceModelExtensions.xml @@ -23,6 +23,9 @@ description="Enables various functionality in IDE to work via the new reactive API for the workspace model"/> +