External project storage — store libraries

java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java

@@ -24,6 +24,7 @@ import com.intellij.compiler.impl.javaCompiler.BackendCompiler;
 import com.intellij.compiler.impl.javaCompiler.javac.JavacConfiguration;
 import com.intellij.compiler.server.impl.BuildProcessClasspathManager;
 import com.intellij.concurrency.JobScheduler;
+import com.intellij.configurationStore.StreamProviderKt;
 import com.intellij.execution.ExecutionException;
 import com.intellij.execution.ExecutionListener;
 import com.intellij.execution.ExecutionManager;
@@ -1139,7 +1140,7 @@ public class BuildManager implements Disposable {
       cmdLine.addParameter("-Dpreload.config.path=" + FileUtil.toCanonicalPath(PathManager.getOptionsPath()));
     }
 
-    if (ProjectUtil.isExternalStorageEnabled()) {
+    if (StreamProviderKt.isExternalStorageEnabled()) {
       cmdLine.addParameter("-Dexternal.project.config=" + ProjectUtil.getExternalConfigurationDir(project));
     }
 

platform/configuration-store-impl/src/FileBasedStorage.kt

@@ -26,7 +26,6 @@ import com.intellij.openapi.components.StoragePathMacros
 import com.intellij.openapi.components.TrackingPathMacroSubstitutor
 import com.intellij.openapi.diagnostic.debug
 import com.intellij.openapi.fileEditor.impl.LoadTextUtil
-import com.intellij.openapi.project.ProjectBundle
 import com.intellij.openapi.util.JDOMUtil
 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream
 import com.intellij.openapi.util.io.FileUtilRt
@@ -42,9 +41,7 @@ import com.intellij.util.loadElement
 import com.intellij.util.toBufferExposingByteArray
 import org.jdom.Element
 import org.jdom.JDOMException
-import java.io.FileNotFoundException
 import java.io.IOException
-import java.nio.ByteBuffer
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.nio.file.NoSuchFileException
@@ -58,8 +55,8 @@ open class FileBasedStorage(file: Path,
                             roamingType: RoamingType? = null,
                             provider: StreamProvider? = null) : XmlElementStorage(fileSpec, rootElementName, pathMacroManager, roamingType, provider) {
   private @Volatile var cachedVirtualFile: VirtualFile? = null
-  private var lineSeparator: LineSeparator? = null
-  private var blockSavingTheContent = false
+  protected var lineSeparator: LineSeparator? = null
+  protected var blockSavingTheContent = false
 
   @Volatile var file = file
     private set
@@ -115,9 +112,9 @@ open class FileBasedStorage(file: Path,
       return cachedVirtualFile
     }
 
-  private inline fun runAndHandleExceptions(task: () -> Unit) {
+  protected inline fun <T> runAndHandleExceptions(task: () -> T): T? {
     try {
-      task()
+      return task()
     }
     catch (e: JDOMException) {
       processReadException(e)
@@ -125,6 +122,7 @@ open class FileBasedStorage(file: Path,
     catch (e: IOException) {
       processReadException(e)
     }
+    return null
   }
 
   fun preloadStorageData(isEmpty: Boolean) {
@@ -138,34 +136,7 @@ open class FileBasedStorage(file: Path,
 
   override fun loadLocalData(): Element? {
     blockSavingTheContent = false
-    // use VFS to load module file because it is refreshed and loaded into VFS in any case
-    val isModuleFile = fileSpec == StoragePathMacros.MODULE_FILE
-    if (!isModuleFile) {
-      runAndHandleExceptions {
-        return loadLocalDataUsingIo()
-      }
-    }
-
-    var virtualFile = cachedVirtualFile
-    if (virtualFile == null) {
-      virtualFile = LocalFileSystem.getInstance().findFileByPath(file.systemIndependentPath)
-      if (virtualFile == null || !virtualFile.exists()) {
-        throw FileNotFoundException(ProjectBundle.message("module.file.does.not.exist.error", file.systemIndependentPath))
-      }
-      cachedVirtualFile = virtualFile
-    }
-
-    if (virtualFile.length == 0L) {
-      processReadException(null)
-    }
-    else {
-      runAndHandleExceptions {
-        val charBuffer = StandardCharsets.UTF_8.decode(ByteBuffer.wrap(virtualFile!!.contentsToByteArray()))
-        lineSeparator = detectLineSeparators(charBuffer, if (isUseXmlProlog) null else LineSeparator.LF)
-        return loadElement(charBuffer)
-      }
-    }
-    return null
+    return runAndHandleExceptions { loadLocalDataUsingIo() }
   }
 
   private fun loadLocalDataUsingIo(): Element? {
@@ -192,7 +163,7 @@ open class FileBasedStorage(file: Path,
     return null
   }
 
-  private fun processReadException(e: Exception?) {
+  protected fun processReadException(e: Exception?) {
     val contentTruncated = e == null
     blockSavingTheContent = !contentTruncated && (PROJECT_FILE == fileSpec || fileSpec.startsWith(PROJECT_CONFIG_DIR) || fileSpec == StoragePathMacros.MODULE_FILE || fileSpec == StoragePathMacros.WORKSPACE_FILE)
     if (!ApplicationManager.getApplication().isUnitTestMode && !ApplicationManager.getApplication().isHeadlessEnvironment) {

platform/configuration-store-impl/src/ModuleStateStorageManager.kt

@@ -15,17 +15,23 @@
  */
 package com.intellij.configurationStore
 
-import com.intellij.openapi.components.StateStorage
-import com.intellij.openapi.components.StateStorageOperation
-import com.intellij.openapi.components.StoragePathMacros
-import com.intellij.openapi.components.TrackingPathMacroSubstitutor
+import com.intellij.openapi.components.*
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.module.ModuleManager
 import com.intellij.openapi.module.impl.ModuleEx
 import com.intellij.openapi.module.impl.ModuleManagerImpl
 import com.intellij.openapi.module.impl.getModuleNameByFilePath
+import com.intellij.openapi.project.ProjectBundle
 import com.intellij.openapi.vfs.newvfs.events.VFileEvent
+import com.intellij.util.LineSeparator
+import com.intellij.util.io.systemIndependentPath
+import com.intellij.util.loadElement
 import org.jdom.Element
+import java.io.FileNotFoundException
+import java.nio.ByteBuffer
+import java.nio.charset.StandardCharsets
+import java.nio.file.Path
+import java.nio.file.Paths
 
 internal class ModuleStateStorageManager(macroSubstitutor: TrackingPathMacroSubstitutor, module: Module) : StateStorageManagerImpl("module", macroSubstitutor, module) {
   override fun getOldStorageSpec(component: Any, componentName: String, operation: StateStorageOperation) = StoragePathMacros.MODULE_FILE
@@ -73,4 +79,42 @@ internal class ModuleStateStorageManager(macroSubstitutor: TrackingPathMacroSubs
     // need be last for compat reasons
     element.setAttribute(ProjectStateStorageManager.VERSION_OPTION, "4")
   }
+
+  override fun createFileBasedStorage(path: String, collapsedPath: String, roamingType: RoamingType, rootTagName: String?): StateStorage
+    = ModuleFileStorage(this, Paths.get(path), collapsedPath, rootTagName, roamingType, getMacroSubstitutor(collapsedPath), if (roamingType == RoamingType.DISABLED) null else compoundStreamProvider)
+
+  private class ModuleFileStorage(storageManager: StateStorageManagerImpl,
+                                  file: Path,
+                                  fileSpec: String,
+                                  rootElementName: String?,
+                                  roamingType: RoamingType,
+                                  pathMacroManager: TrackingPathMacroSubstitutor? = null,
+                                  provider: StreamProvider? = null) : MyFileStorage(storageManager, file, fileSpec, rootElementName, roamingType, pathMacroManager, provider) {
+    // use VFS to load module file because it is refreshed and loaded into VFS in any case
+    override fun loadLocalData(): Element? {
+      blockSavingTheContent = false
+      val virtualFile = virtualFile
+      if (virtualFile == null || !virtualFile.exists()) {
+        // only on first load
+        if (storageDataRef.get() == null) {
+          throw FileNotFoundException(ProjectBundle.message("module.file.does.not.exist.error", file.systemIndependentPath))
+        }
+        else {
+          return null
+        }
+      }
+
+      if (virtualFile.length == 0L) {
+        processReadException(null)
+      }
+      else {
+        runAndHandleExceptions {
+          val charBuffer = StandardCharsets.UTF_8.decode(ByteBuffer.wrap(virtualFile.contentsToByteArray()))
+          lineSeparator = detectLineSeparators(charBuffer, if (isUseXmlProlog) null else LineSeparator.LF)
+          return loadElement(charBuffer)
+        }
+      }
+      return null
+    }
+  }
 }

platform/configuration-store-impl/src/ProjectStoreImpl.kt

@@ -187,7 +187,6 @@ abstract class ProjectStoreBase(override final val project: ProjectImpl) : Compo
     if (isDirectoryBased) {
       var result: MutableList<Storage>? = null
       for (storage in storages) {
-        @Suppress("DEPRECATION")
         if (storage.path != PROJECT_FILE) {
           if (result == null) {
             result = SmartList()
@@ -201,6 +200,13 @@ abstract class ProjectStoreBase(override final val project: ProjectImpl) : Compo
       }
       else {
         result!!.sortWith(deprecatedComparator)
+        StreamProviderFactory.EP_NAME.getExtensions(project).computeIfAny {
+          LOG.runAndLogException { it.customizeStorageSpecs(component, storageManager.componentManager!!, result!!, operation) }
+        }?.let {
+          // yes, DEPRECATED_PROJECT_FILE_STORAGE_ANNOTATION is not added in this case
+          return it
+        }
+
         // if we create project from default, component state written not to own storage file, but to project file,
         // we don't have time to fix it properly, so, ancient hack restored
         result.add(DEPRECATED_PROJECT_FILE_STORAGE_ANNOTATION)

platform/configuration-store-impl/src/StateStorageManagerImpl.kt

@@ -21,7 +21,6 @@ import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.*
 import com.intellij.openapi.components.StateStorage.SaveSession
 import com.intellij.openapi.components.StateStorageChooserEx.Resolution
-import com.intellij.openapi.project.isExternalStorageEnabled
 import com.intellij.openapi.roots.ProjectModelElement
 import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.util.io.FileUtilRt
@@ -239,7 +238,12 @@ open class StateStorageManagerImpl(private val rootTagName: String,
     if (storageClass != StateStorage::class.java) {
       val constructor = storageClass.constructors.first()
       constructor.isAccessible = true
-      return constructor.newInstance(componentManager!!, this) as StateStorage
+      if (constructor.parameterCount == 2) {
+        return constructor.newInstance(componentManager!!, this) as StateStorage
+      }
+      else {
+        return constructor.newInstance(collapsedPath, componentManager!!, this) as StateStorage
+      }
     }
 
     val effectiveRoamingType = getEffectiveRoamingType(roamingType, collapsedPath)
@@ -279,13 +283,13 @@ open class StateStorageManagerImpl(private val rootTagName: String,
   private class MyDirectoryStorage(override val storageManager: StateStorageManagerImpl, file: Path, @Suppress("DEPRECATION") splitter: StateSplitter) :
     DirectoryBasedStorage(file, splitter, storageManager.macroSubstitutor), StorageVirtualFileTracker.TrackedStorage
 
-  private class MyFileStorage(override val storageManager: StateStorageManagerImpl,
-                              file: Path,
-                              fileSpec: String,
-                              rootElementName: String?,
-                              roamingType: RoamingType,
-                              pathMacroManager: TrackingPathMacroSubstitutor? = null,
-                              provider: StreamProvider? = null) : FileBasedStorage(file, fileSpec, rootElementName, pathMacroManager, roamingType, provider), StorageVirtualFileTracker.TrackedStorage {
+  protected open class MyFileStorage(override val storageManager: StateStorageManagerImpl,
+                                     file: Path,
+                                     fileSpec: String,
+                                     rootElementName: String?,
+                                     roamingType: RoamingType,
+                                     pathMacroManager: TrackingPathMacroSubstitutor? = null,
+                                     provider: StreamProvider? = null) : FileBasedStorage(file, fileSpec, rootElementName, pathMacroManager, roamingType, provider), StorageVirtualFileTracker.TrackedStorage {
     override val isUseXmlProlog: Boolean
       get() = rootElementName != null && storageManager.isUseXmlProlog
 

platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStorage.kt

@@ -20,16 +20,19 @@ import com.intellij.configurationStore.StateStorageManager
 import com.intellij.configurationStore.StreamProviderFactory
 import com.intellij.configurationStore.XmlElementStorage
 import com.intellij.openapi.components.RoamingType
+import com.intellij.openapi.components.StateSplitterEx
 import com.intellij.openapi.components.StoragePathMacros
 import com.intellij.openapi.module.Module
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.JDOMUtil
 import org.jdom.Element
 
-internal class ExternalProjectStorage(private val module: Module, storageManager: StateStorageManager) : XmlElementStorage(StoragePathMacros.MODULE_FILE, "module", storageManager.macroSubstitutor, RoamingType.DISABLED) {
+internal class ExternalModuleStorage(private val module: Module, storageManager: StateStorageManager) : XmlElementStorage(StoragePathMacros.MODULE_FILE, "module", storageManager.macroSubstitutor, RoamingType.DISABLED) {
   private val manager = StreamProviderFactory.EP_NAME.getExtensions(module.project).first { it is ExternalSystemStreamProviderFactory } as ExternalSystemStreamProviderFactory
 
   override fun loadLocalData() = manager.readModuleData(module.name)
 
-  override fun createSaveSession(states: StateMap) = object : XmlElementStorageSaveSession<ExternalProjectStorage>(states, this) {
+  override fun createSaveSession(states: StateMap) = object : XmlElementStorageSaveSession<ExternalModuleStorage>(states, this) {
     override fun saveLocally(element: Element?) {
       // our customizeStorageSpecs on write will not return our storage for not applicable module, so, we don't need to check it here
       if (element == null) {
@@ -40,4 +43,32 @@ internal class ExternalProjectStorage(private val module: Module, storageManager
       }
     }
   }
+}
+
+// for libraries only for now - we use null rootElementName because the only component is expected (libraryTable)
+internal class ExternalProjectStorage(fileSpec: String, project: Project, storageManager: StateStorageManager) : XmlElementStorage(fileSpec, null, storageManager.macroSubstitutor, RoamingType.DISABLED) {
+  private val manager = StreamProviderFactory.EP_NAME.getExtensions(project).first { it is ExternalSystemStreamProviderFactory } as ExternalSystemStreamProviderFactory
+
+  override fun loadLocalData() = manager.fileStorage.read(fileSpec)
+
+  override fun createSaveSession(states: StateMap) = object : XmlElementStorageSaveSession<ExternalProjectStorage>(states, this) {
+    override fun saveLocally(element: Element?) {
+      var isEmpty = true
+      if (element != null) {
+        for (child in element.children) {
+          if (child.getAttribute(StateSplitterEx.EXTERNAL_SYSTEM_ID_ATTRIBUTE) != null) {
+            isEmpty = false
+            break
+          }
+        }
+      }
+
+      if (element == null || isEmpty) {
+        manager.fileStorage.remove(fileSpec)
+      }
+      else {
+        manager.fileStorage.write(fileSpec, element, JDOMUtil.ElementOutputFilter { childElement, level -> level != 1 || childElement.getAttribute(StateSplitterEx.EXTERNAL_SYSTEM_ID_ATTRIBUTE) != null })
+      }
+    }
+  }
 }

platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStreamProviderFactory.kt

@@ -18,6 +18,7 @@ package com.intellij.openapi.externalSystem.configurationStore
 import com.intellij.ProjectTopics
 import com.intellij.configurationStore.FileStorageAnnotation
 import com.intellij.configurationStore.StreamProviderFactory
+import com.intellij.configurationStore.isExternalStorageEnabled
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.*
 import com.intellij.openapi.diagnostic.logger
@@ -26,18 +27,18 @@ import com.intellij.openapi.module.Module
 import com.intellij.openapi.project.ModuleListener
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ex.ProjectEx
-import com.intellij.openapi.project.isExternalStorageEnabled
 import com.intellij.openapi.roots.ProjectModelElement
 import com.intellij.util.Function
 import org.jdom.Element
 import java.util.*
 
-private val EXTERNAL_STORAGE_ANNOTATION = FileStorageAnnotation(StoragePathMacros.MODULE_FILE, false, ExternalProjectStorage::class.java)
+private val EXTERNAL_MODULE_STORAGE_ANNOTATION = FileStorageAnnotation(StoragePathMacros.MODULE_FILE, false, ExternalModuleStorage::class.java)
 private val LOG = logger<ExternalSystemStreamProviderFactory>()
 
 // todo handle module rename
 internal class ExternalSystemStreamProviderFactory(private val project: Project) : StreamProviderFactory {
-  val moduleStorage = FileSystemExternalSystemStorage(project)
+  val moduleStorage = ModuleFileSystemExternalSystemStorage(project)
+  val fileStorage = ProjectFileSystemExternalSystemStorage(project)
 
   private var isStorageFlushInProgress = false
 
@@ -72,7 +73,21 @@ internal class ExternalSystemStreamProviderFactory(private val project: Project)
   }
 
   override fun customizeStorageSpecs(component: PersistentStateComponent<*>, componentManager: ComponentManager, storages: List<Storage>, operation: StateStorageOperation): List<Storage>? {
-    if (componentManager !is Module || component !is ProjectModelElement || !isExternalStorageEnabled) {
+    if (!isExternalStorageEnabled) {
+      return null
+    }
+
+    if (componentManager is Project) {
+      val fileSpec = storages.firstOrNull()?.value
+      if (fileSpec == "libraries") {
+        val result = ArrayList<Storage>(storages.size + 1)
+        result.add(FileStorageAnnotation("$fileSpec.xml", false, ExternalProjectStorage::class.java))
+        result.addAll(storages)
+        return result
+      }
+    }
+
+    if (component !is ProjectModelElement) {
       return null
     }
 
@@ -88,7 +103,12 @@ internal class ExternalSystemStreamProviderFactory(private val project: Project)
     // on write default storages also returned, because default FileBasedStorage will remove data if component has external source
 
     val result = ArrayList<Storage>(storages.size + 1)
-    result.add(EXTERNAL_STORAGE_ANNOTATION)
+    if (componentManager is Project) {
+      result.add(FileStorageAnnotation(storages.get(0).value, false, ExternalProjectStorage::class.java))
+    }
+    else {
+      result.add(EXTERNAL_MODULE_STORAGE_ANNOTATION)
+    }
     result.addAll(storages)
     return result
   }

platform/external-system-impl/src/com/intellij/openapi/externalSystem/configurationStore/PersistentMapManager.kt

@@ -15,91 +15,18 @@
  */
 package com.intellij.openapi.externalSystem.configurationStore
 
-import com.intellij.configurationStore.deserializeElementFromBinary
-import com.intellij.configurationStore.serializeElementToBinary
-import com.intellij.openapi.Disposable
 import com.intellij.openapi.diagnostic.logger
 import com.intellij.openapi.externalSystem.service.project.manage.ExternalProjectsDataStorage
-import com.intellij.openapi.externalSystem.service.project.manage.ExternalProjectsManager
 import com.intellij.openapi.project.Project
-import com.intellij.openapi.startup.StartupManager
-import com.intellij.openapi.util.Disposer
-import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream
-import com.intellij.openapi.util.io.ByteSequence
+import com.intellij.openapi.util.JDOMUtil
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.util.io.*
 import com.intellij.util.loadElement
 import com.intellij.util.write
 import org.jdom.Element
-import java.io.ByteArrayInputStream
-import java.io.IOException
 import java.nio.file.Path
 
-private val LOG = logger<PersistentMapManager<Any>>()
-
-class PersistentMapManager<VALUE>(name: String, dir: Path, valueExternalizer: DataExternalizer<VALUE>, parentDisposable: Disposable, formatVersion: Int, corruptedHandler: () -> Unit) {
-  private val file = dir.resolve(name)
-
-  private val fileInitiallyExisted = file.exists()
-
-  @Volatile
-  private var storageCreated = false
-
-  fun get(key: String) = if (fileInitiallyExisted || storageCreated) storage.get(key) else null
-
-  fun remove(key: String) {
-    put(key, null)
-  }
-
-  fun put(key: String, value: VALUE?) {
-    if (value != null) {
-      storage.put(key, value)
-    }
-    else if (fileInitiallyExisted || storageCreated) {
-      storage.remove(key)
-    }
-  }
-
-  val isDirty: Boolean
-    get() = if (storageCreated) storage.isDirty else false
-
-  private val storage by lazy {
-    val versionFile = dir.resolve("$name.version")
-
-    fun createMap() = PersistentHashMap(file.toFile(), EnumeratorStringDescriptor.INSTANCE, valueExternalizer)
-
-    fun deleteFileAndWriteLatestFormatVersion() {
-      dir.deleteChildrenStartingWith(file.fileName.toString())
-      file.delete()
-      versionFile.outputStream().use { it.write(formatVersion) }
-
-      corruptedHandler()
-    }
-
-    val data = try {
-      val fileVersion = versionFile.inputStreamIfExists()?.use { it.read() } ?: -1
-      if (fileVersion != formatVersion) {
-        deleteFileAndWriteLatestFormatVersion()
-      }
-      createMap()
-    }
-    catch (e: IOException) {
-      LOG.info(e)
-      deleteFileAndWriteLatestFormatVersion()
-      createMap()
-    }
-
-    Disposer.register(parentDisposable, Disposable { data.close() })
-    storageCreated = true
-    data
-  }
-
-  fun forceSave() {
-    if (storageCreated) {
-      storage.force()
-    }
-  }
-}
+private val LOG = logger<FileSystemExternalSystemStorage>()
 
 internal interface ExternalSystemStorage {
   val isDirty: Boolean
@@ -108,19 +35,28 @@ internal interface ExternalSystemStorage {
 
   fun read(name: String): Element?
 
-  fun write(name: String, element: Element)
+  fun write(name: String, element: Element, filter: JDOMUtil.ElementOutputFilter? = null)
 
   fun forceSave()
 
   fun rename(oldName: String, newName: String)
 }
 
-private fun nameToFilename(name: String) = "${FileUtil.sanitizeFileName(name, false)}.xml"
+internal class ModuleFileSystemExternalSystemStorage(project: Project) : FileSystemExternalSystemStorage("modules", project) {
+  companion object {
+    private fun nameToFilename(name: String) = "${FileUtil.sanitizeFileName(name, false)}.xml"
+  }
 
-internal class FileSystemExternalSystemStorage(project: Project) : ExternalSystemStorage {
+  override fun nameToPath(name: String) = super.nameToPath(nameToFilename(name))
+}
+
+internal class ProjectFileSystemExternalSystemStorage(project: Project) : FileSystemExternalSystemStorage("project", project) {
+}
+
+internal abstract class FileSystemExternalSystemStorage(dirName: String, project: Project) : ExternalSystemStorage {
   override val isDirty = false
 
-  private val dir = ExternalProjectsDataStorage.getProjectConfigurationDir(project).resolve("modules")
+  protected val dir: Path = ExternalProjectsDataStorage.getProjectConfigurationDir(project).resolve(dirName)
 
   private var hasSomeData: Boolean
 
@@ -140,7 +76,7 @@ internal class FileSystemExternalSystemStorage(project: Project) : ExternalSyste
     }
   }
 
-  private fun nameToPath(name: String) = dir.resolve(nameToFilename(name))
+  protected open fun nameToPath(name: String): Path = dir.resolve(name)
 
   override fun forceSave() {
   }
@@ -163,9 +99,9 @@ internal class FileSystemExternalSystemStorage(project: Project) : ExternalSyste
     }
   }
 
-  override fun write(name: String, element: Element) {
+  override fun write(name: String, element: Element, filter: JDOMUtil.ElementOutputFilter?) {
     hasSomeData = true
-    element.write(nameToPath(name))
+    element.write(nameToPath(name), filter = filter)
   }
 
   override fun rename(oldName: String, newName: String) {
@@ -178,48 +114,4 @@ internal class FileSystemExternalSystemStorage(project: Project) : ExternalSyste
       oldFile.move(nameToPath(newName))
     }
   }
-}
-
-// not used for now, https://upsource.jetbrains.com/IDEA/review/IDEA-CR-20673, later PersistentHashMap will be not used.
-@Suppress("unused")
-internal class BinaryExternalSystemStorage(project: Project) : ExternalSystemStorage {
-  override fun forceSave() {
-    moduleStorage.forceSave()
-  }
-
-  override val isDirty: Boolean
-    get() = moduleStorage.isDirty
-
-  @Suppress("INTERFACE_STATIC_METHOD_CALL_FROM_JAVA6_TARGET")
-  val moduleStorage = PersistentMapManager("modules", ExternalProjectsDataStorage.getProjectConfigurationDir(project), ByteSequenceDataExternalizer.INSTANCE, project, 0) {
-    StartupManager.getInstance(project).runWhenProjectIsInitialized {
-      val externalProjectManager = ExternalProjectsManager.getInstance(project)
-      externalProjectManager.runWhenInitialized {
-        externalProjectManager.externalProjectsWatcher.markDirtyAllExternalProjects()
-      }
-    }
-  }
-
-  override fun remove(name: String) {
-    moduleStorage.remove(name)
-  }
-
-  override fun read(name: String): Element? {
-    val data = moduleStorage.get(name) ?: return null
-    return ByteArrayInputStream(data.bytes, data.offset, data.length).use { deserializeElementFromBinary(it) }
-
-  }
-
-  override fun write(name: String, element: Element) {
-    val byteOut = BufferExposingByteArrayOutputStream()
-    serializeElementToBinary(element, byteOut)
-    moduleStorage.put(name, ByteSequence(byteOut.internalBuffer, 0, byteOut.size()))
-  }
-
-  override fun rename(oldName: String, newName: String) {
-    moduleStorage.get(oldName)?.let {
-      moduleStorage.remove(oldName)
-      moduleStorage.put(newName, it)
-    }
-  }
 }

platform/external-system-impl/testSrc/com/intellij/openapi/externalSystem/configurationStore/ExternalSystemStorageTest.kt

@@ -16,10 +16,10 @@
 package com.intellij.openapi.externalSystem.configurationStore
 
 import com.intellij.configurationStore.ESCAPED_MODULE_DIR
+import com.intellij.configurationStore.IS_EXTERNAL_STORAGE_ENABLED
 import com.intellij.configurationStore.createModule
 import com.intellij.configurationStore.useAndDispose
 import com.intellij.openapi.externalSystem.service.project.manage.ExternalProjectsDataStorage
-import com.intellij.openapi.project.IS_EXTERNAL_STORAGE_ENABLED
 import com.intellij.openapi.roots.ExternalProjectSystemRegistry
 import com.intellij.openapi.roots.ModuleRootModificationUtil
 import com.intellij.testFramework.*

platform/platform-api/src/com/intellij/openapi/project/ProjectUtil.kt

@@ -31,13 +31,11 @@ import com.intellij.openapi.module.ModuleManager
 import com.intellij.openapi.roots.ModuleRootManager
 import com.intellij.openapi.roots.ProjectRootManager
 import com.intellij.openapi.util.io.FileUtil
-import com.intellij.openapi.util.registry.Registry
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFilePathWrapper
 import com.intellij.util.PathUtilRt
 import com.intellij.util.io.exists
 import com.intellij.util.text.trimMiddle
-import org.jetbrains.annotations.TestOnly
 import java.nio.file.InvalidPathException
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -169,12 +167,6 @@ fun Project.getExternalConfigurationDir(): Path {
   return getProjectCachePath("external_build_system")
 }
 
-@set:TestOnly
-var IS_EXTERNAL_STORAGE_ENABLED = false
-
-val isExternalStorageEnabled: Boolean
-  get() = Registry.`is`("store.imported.project.elements.separately", false) || IS_EXTERNAL_STORAGE_ENABLED
-
 /**
  * Use parameters only for migration purposes, once all usages will be migrated, parameters will be removed
  */

platform/projectModel-api/src/com/intellij/configurationStore/StreamProvider.kt

@@ -17,9 +17,16 @@ package com.intellij.configurationStore
 
 import com.intellij.openapi.components.RoamingType
 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream
+import com.intellij.openapi.util.registry.Registry
 import org.jetbrains.annotations.TestOnly
 import java.io.InputStream
 
+@set:TestOnly
+var IS_EXTERNAL_STORAGE_ENABLED = false
+
+val isExternalStorageEnabled: Boolean
+  get() = Registry.`is`("store.imported.project.elements.separately", false) || IS_EXTERNAL_STORAGE_ENABLED
+
 interface StreamProvider {
   val enabled: Boolean
     get() = true

platform/projectModel-api/src/com/intellij/openapi/components/StateSplitterEx.java

@@ -15,6 +15,7 @@
  */
 package com.intellij.openapi.components;
 
+import com.intellij.configurationStore.StreamProviderKt;
 import com.intellij.openapi.util.JDOMUtil;
 import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.io.FileUtil;
@@ -27,6 +28,8 @@ import java.util.List;
 
 @SuppressWarnings("deprecation")
 public abstract class StateSplitterEx implements StateSplitter {
+  public static final String EXTERNAL_SYSTEM_ID_ATTRIBUTE = "external-system-id";
+
   @Override
   public abstract List<Pair<Element, String>> splitState(@NotNull Element state);
 
@@ -41,10 +44,18 @@ public abstract class StateSplitterEx implements StateSplitter {
 
   @NotNull
   protected static List<Pair<Element, String>> splitState(@NotNull Element state, @NotNull String attributeName) {
+    return splitState(state, attributeName, false);
+  }
+
+  @NotNull
+  protected static List<Pair<Element, String>> splitState(@NotNull Element state, @NotNull String attributeName, boolean filterOutExternalElements) {
     UniqueNameGenerator generator = new UniqueNameGenerator();
     List<Pair<Element, String>> result = new SmartList<>();
+    boolean isExternalStorageEnabled = filterOutExternalElements && StreamProviderKt.isExternalStorageEnabled();
     for (Element subState : state.getChildren()) {
-      result.add(createItem(generator, subState, attributeName));
+      if (!isExternalStorageEnabled || subState.getAttribute(EXTERNAL_SYSTEM_ID_ATTRIBUTE) == null) {
+        result.add(createItem(generator, subState, attributeName));
+      }
     }
     return result;
   }

platform/projectModel-api/src/com/intellij/openapi/components/Storage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2016 JetBrains s.r.o.
+ * Copyright 2000-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ public @interface Storage {
   RoamingType roamingType() default RoamingType.DEFAULT;
 
   /**
-   * Class must have constructor {@code (ComponentManager componentManager, StateStorageManager storageManager)}.
+   * Class must have constructor {@code (String fileSpec, ComponentManager componentManager, StateStorageManager storageManager)}.
    * {@code componentManager} parameter can have more concrete type - e.g. Module (if storage intended to support only one type).
    */
   Class<? extends StateStorage> storageClass() default StateStorage.class;

platform/projectModel-api/src/com/intellij/util/jdom.kt

@@ -61,18 +61,18 @@ private fun getSaxBuilder(): SAXBuilder {
 
 @JvmOverloads
 @Throws(IOException::class)
-fun Parent.write(file: Path, lineSeparator: String = "\n") {
-  write(file.outputStream(), lineSeparator)
+fun Parent.write(file: Path, lineSeparator: String = "\n", filter: JDOMUtil.ElementOutputFilter? = null) {
+  write(file.outputStream(), lineSeparator, filter)
 }
 
 @JvmOverloads
-fun Parent.write(output: OutputStream, lineSeparator: String = "\n") {
+fun Parent.write(output: OutputStream, lineSeparator: String = "\n", filter: JDOMUtil.ElementOutputFilter? = null) {
   output.bufferedWriter().use { writer ->
     if (this is Document) {
       JDOMUtil.writeDocument(this, writer, lineSeparator)
     }
     else {
-      JDOMUtil.writeElement(this as Element, writer, lineSeparator)
+      JDOMUtil.writeElement(this as Element, writer, JDOMUtil.createOutputter(lineSeparator, filter))
     }
   }
 }

platform/projectModel-impl/src/com/intellij/configurationStore/StreamProviderFactory.kt

@@ -35,9 +35,7 @@ interface StreamProviderFactory {
   fun createProvider(componentManager: ComponentManager, storageManager: StateStorageManager): StreamProvider? = null
 
   /**
-   * For now called only for Module.
    * `storages` are preprocessed by component store - not raw from state spec.
-   *
    * @return null if not applicable
    */
   fun customizeStorageSpecs(component: PersistentStateComponent<*>, componentManager: ComponentManager, storages: List<Storage>, operation: StateStorageOperation): List<Storage>? = null

platform/projectModel-impl/src/com/intellij/openapi/roots/impl/libraries/LibraryImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2016 JetBrains s.r.o.
+ * Copyright 2000-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -48,6 +48,7 @@ import org.jetbrains.annotations.Nullable;
 
 import java.util.*;
 
+import static com.intellij.openapi.components.StateSplitterEx.EXTERNAL_SYSTEM_ID_ATTRIBUTE;
 import static com.intellij.openapi.vfs.VirtualFileVisitor.ONE_LEVEL_DEEP;
 import static com.intellij.openapi.vfs.VirtualFileVisitor.SKIP_ROOT;
 
@@ -63,7 +64,6 @@ public class LibraryImpl extends TraceableDisposable implements LibraryEx.Modifi
   @NonNls public static final String PROPERTIES_ELEMENT = "properties";
   private static final SkipDefaultValuesSerializationFilters SERIALIZATION_FILTERS = new SkipDefaultValuesSerializationFilters();
   private static final String EXCLUDED_ROOTS_TAG = "excluded";
-  private static final String EXTERNAL_SYSTEM_ID_ATTRIBUTE = "external-system-id";
   private String myName;
   private final LibraryTable myLibraryTable;
   private final Map<OrderRootType, VirtualFilePointerContainer> myRoots;

platform/projectModel-impl/src/com/intellij/openapi/roots/impl/libraries/ProjectLibraryTable.kt

@@ -39,7 +39,7 @@ open class ProjectLibraryTable : LibraryTableBase() {
   override fun getPresentation() = PROJECT_LIBRARY_TABLE_PRESENTATION
 
   class LibraryStateSplitter : StateSplitterEx() {
-    override fun splitState(state: Element): MutableList<Pair<Element, String>> = StateSplitterEx.splitState(state, LibraryImpl.LIBRARY_NAME_ATTR)
+    override fun splitState(state: Element): MutableList<Pair<Element, String>> = StateSplitterEx.splitState(state, LibraryImpl.LIBRARY_NAME_ATTR, true)
   }
 }
 

platform/util/src/com/intellij/openapi/util/JDOMUtil.java

@@ -433,7 +433,10 @@ public class JDOMUtil {
   }
 
   public static void writeElement(@NotNull Element element, Writer writer, String lineSeparator) throws IOException {
-    XMLOutputter xmlOutputter = createOutputter(lineSeparator);
+    writeElement(element, writer, createOutputter(lineSeparator));
+  }
+
+  public static void writeElement(@NotNull Element element, @NotNull Writer writer, @NotNull XMLOutputter xmlOutputter) throws IOException {
     try {
       xmlOutputter.output(element, writer);
     }
@@ -483,7 +486,12 @@ public class JDOMUtil {
 
   @NotNull
   public static XMLOutputter createOutputter(String lineSeparator) {
-    XMLOutputter xmlOutputter = new MyXMLOutputter();
+    return createOutputter(lineSeparator, null);
+  }
+
+  @NotNull
+  public static XMLOutputter createOutputter(String lineSeparator, @Nullable ElementOutputFilter elementOutputFilter) {
+    XMLOutputter xmlOutputter = new MyXMLOutputter(elementOutputFilter);
     Format format = Format.getCompactFormat().
       setIndent("  ").
       setTextMode(Format.TextMode.TRIM).
@@ -555,6 +563,16 @@ public class JDOMUtil {
   }
 
   public static class MyXMLOutputter extends XMLOutputter {
+    private final ElementOutputFilter myElementOutputFilter;
+
+    public MyXMLOutputter(@Nullable ElementOutputFilter filter) {
+      myElementOutputFilter = filter;
+    }
+
+    public MyXMLOutputter() {
+      this(null);
+    }
+
     @Override
     @NotNull
     public String escapeAttributeEntities(@NotNull String str) {
@@ -566,6 +584,17 @@ public class JDOMUtil {
     public String escapeElementEntities(@NotNull String str) {
       return escapeText(str, false, false);
     }
+
+    @Override
+    protected void printElement(Writer out, Element element, int level, NamespaceStack namespaces) throws IOException {
+      if (myElementOutputFilter == null || myElementOutputFilter.accept(element, level)) {
+        super.printElement(out, element, level, namespaces);
+      }
+    }
+  }
+
+  public interface ElementOutputFilter {
+    boolean accept(@NotNull Element element, int level);
   }
 
   private static void printDiagnostics(@NotNull Element element, String prefix) {