project configuration: automatically unload newly added modules

If some modules were unloaded IDEA will process modules which appear after reloading state (e.g. after update from VCS),  automatically decide which of them can be also unloaded, and show notification about that (IDEA-180193).

java/java-tests/testSrc/com/intellij/roots/AutomaticModuleUnloaderTest.kt

@@ -0,0 +1,137 @@
+// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.roots
+
+import com.intellij.openapi.application.runWriteAction
+import com.intellij.openapi.components.stateStore
+import com.intellij.openapi.module.Module
+import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.module.StdModuleTypes
+import com.intellij.openapi.module.impl.ModuleManagerImpl
+import com.intellij.openapi.module.impl.ModulePath
+import com.intellij.openapi.module.impl.UnloadedModuleDescriptionImpl
+import com.intellij.openapi.project.ex.ProjectManagerEx
+import com.intellij.openapi.project.impl.ProjectManagerImpl
+import com.intellij.openapi.roots.ModuleRootModificationUtil
+import com.intellij.openapi.util.Disposer
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.testFramework.ModuleTestCase
+import java.io.File
+import java.util.*
+
+/**
+ * @author nik
+ */
+class AutomaticModuleUnloaderTest : ModuleTestCase() {
+  fun `test unload simple module`() {
+    createModule("a")
+    createModule("b")
+    val moduleManager = ModuleManager.getInstance(project)
+    moduleManager.setUnloadedModules(listOf("a"))
+    createModule("c")
+
+    val moduleFiles = createNewModuleFiles(listOf("d")) {}
+    reloadProjectWithNewModules(moduleFiles)
+
+    ModuleTestCase.assertSameElements(moduleManager.unloadedModuleDescriptions.map { it.name }, "a", "d")
+  }
+
+  fun `test unload modules with dependencies between them`() {
+    createModule("a")
+    createModule("b")
+    doTest("a", listOf("c", "d"), { modules ->
+      ModuleRootModificationUtil.updateModel(modules["c"]!!) {
+        it.addModuleOrderEntry(modules["d"]!!)
+      }
+    },"a", "c", "d")
+  }
+
+  fun `test do not unload module if loaded module depends on it`() {
+    createModule("a")
+    val b = createModule("b")
+    ModuleRootModificationUtil.updateModel(b) {
+      it.addInvalidModuleEntry("d")
+    }
+    doTest("a", listOf("d"), {}, "a")
+  }
+
+  fun `test unload module if only unloaded module depends on it`() {
+    val a = createModule("a")
+    createModule("b")
+    ModuleRootModificationUtil.updateModel(a) {
+      it.addInvalidModuleEntry("d")
+    }
+    doTest("a", listOf("d"), {}, "a", "d")
+  }
+
+  fun `test do not unload modules if loaded module depends on them transitively`() {
+    createModule("a")
+    val b = createModule("b")
+    ModuleRootModificationUtil.updateModel(b) {
+      it.addInvalidModuleEntry("d")
+    }
+
+    doTest("a", listOf("c", "d"), { modules ->
+      ModuleRootModificationUtil.updateModel(modules["d"]!!) {
+        it.addModuleOrderEntry(modules["c"]!!)
+      }
+    }, "a")
+  }
+
+  fun `test unload module if loaded module transitively depends on it via previosly unloaded module`() {
+    val a = createModule("a")
+    val b = createModule("b")
+    ModuleRootModificationUtil.addDependency(a, b)
+    ModuleRootModificationUtil.updateModel(b) {
+      it.addInvalidModuleEntry("c")
+    }
+    doTest("b", listOf("c"), {}, "b", "c")
+  }
+
+  private fun doTest(initiallyUnloaded: String,
+                     newModulesName: List<String>,
+                     setup: (Map<String, Module>) -> Unit,
+                     vararg expectedUnloadedModules: String) {
+    val moduleManager = ModuleManager.getInstance(project)
+    moduleManager.setUnloadedModules(listOf(initiallyUnloaded))
+
+    val moduleFiles = createNewModuleFiles(newModulesName, setup)
+    reloadProjectWithNewModules(moduleFiles)
+
+    ModuleTestCase.assertSameElements(moduleManager.unloadedModuleDescriptions.map { it.name }, *expectedUnloadedModules)
+
+  }
+
+  private fun createNewModuleFiles(moduleNames: List<String>, setup: (Map<String, Module>) -> Unit): List<File> {
+    val newModulesProjectDir = FileUtil.createTempDirectory("newModules", "")
+    val moduleFiles = moduleNames.map { File(newModulesProjectDir, "$it.iml") }
+    val projectManager = ProjectManagerEx.getInstanceEx() as ProjectManagerImpl
+    val project = projectManager.createProject("newModules", newModulesProjectDir.absolutePath)!!
+    try {
+      val modules = runWriteAction {
+        moduleFiles.map {
+          ModuleManager.getInstance(project).newModule(it.absolutePath, StdModuleTypes.JAVA.id)
+        }
+      }
+      setup(ModuleManager.getInstance(project).modules.associateBy { it.name })
+      modules.forEach {
+        it.stateStore.save(mutableListOf())
+      }
+    }
+    finally {
+      projectManager.forceCloseProject(project, true)
+      runWriteAction { Disposer.dispose(project) }
+    }
+    return moduleFiles
+  }
+
+  private fun reloadProjectWithNewModules(moduleFiles: List<File>) {
+    val moduleManager = ModuleManagerImpl.getInstanceImpl(myProject)
+    val modulePaths = LinkedHashSet<ModulePath>()
+    moduleManager.modules.forEach { it.stateStore.save(mutableListOf()) }
+    moduleManager.modules.mapTo(modulePaths) { ModulePath(it.moduleFilePath, null) }
+    moduleManager.unloadedModuleDescriptions.mapTo(modulePaths) { (it as UnloadedModuleDescriptionImpl).modulePath }
+    moduleFiles.mapTo(modulePaths) { ModulePath(FileUtil.toSystemIndependentName(it.absolutePath), null) }
+    moduleManager.loadStateFromModulePaths(modulePaths)
+  }
+
+}

platform/lang-impl/src/com/intellij/openapi/module/impl/AutomaticModuleUnloader.kt

@@ -0,0 +1,160 @@
+// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.openapi.module.impl
+
+import com.intellij.notification.Notification
+import com.intellij.notification.NotificationAction
+import com.intellij.notification.NotificationGroup
+import com.intellij.notification.NotificationType
+import com.intellij.openapi.actionSystem.AnActionEvent
+import com.intellij.openapi.components.*
+import com.intellij.openapi.module.ModuleDescription
+import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.roots.ui.configuration.ConfigureUnloadedModulesDialog
+import com.intellij.util.xmlb.annotations.AbstractCollection
+import com.intellij.util.xmlb.annotations.Tag
+import com.intellij.xml.util.XmlStringUtil
+
+/**
+ * If some modules were unloaded and new modules appears after loading project configuration, automatically unloads those which
+ * aren't required for loaded modules.
+ *
+ * @author nik
+ */
+@State(name = "AutomaticModuleUnloader", storages = arrayOf(Storage(StoragePathMacros.WORKSPACE_FILE)))
+class AutomaticModuleUnloader(private val project: Project) : PersistentStateComponent<LoadedModulesListStorage> {
+  private val loadedModulesListStorage = LoadedModulesListStorage()
+
+  fun processNewModules(modulesToLoad: Set<ModulePath>, modulesToUnload: List<UnloadedModuleDescriptionImpl>): UnloadedModulesListChange {
+    val oldLoaded = loadedModulesListStorage.modules.toSet()
+    if (oldLoaded.isEmpty() || modulesToLoad.all { it.moduleName in oldLoaded }) {
+      return UnloadedModulesListChange(emptyList(), emptyList(), emptyList())
+    }
+
+    val moduleDescriptions = LinkedHashMap<String, UnloadedModuleDescriptionImpl>(modulesToLoad.size + modulesToUnload.size)
+    UnloadedModuleDescriptionImpl.createFromPaths(modulesToLoad, project).associateByTo(moduleDescriptions) { it.name }
+    modulesToUnload.associateByTo(moduleDescriptions) { it.name }
+
+    val oldLoadedWithDependencies = HashSet<ModuleDescription>()
+    val explicitlyUnloaded = modulesToUnload.mapTo(HashSet()) { it.name }
+    for (name in oldLoaded) {
+      processTransitiveDependencies(name, moduleDescriptions, explicitlyUnloaded, oldLoadedWithDependencies)
+    }
+
+    val newLoadedNames = oldLoadedWithDependencies.mapTo(LinkedHashSet()) { it.name }
+    val toLoad = modulesToLoad.filter { it.moduleName in newLoadedNames && it.moduleName !in oldLoaded}
+    val toUnload = modulesToLoad.filter { it.moduleName !in newLoadedNames }
+    loadedModulesListStorage.modules.clear()
+    modulesToLoad.filter { it.moduleName in newLoadedNames }.mapTo(loadedModulesListStorage.modules) { it.moduleName }
+    val change = UnloadedModulesListChange(toLoad, toUnload, toUnload.map { moduleDescriptions[it.moduleName]!! })
+    fireNotifications(change)
+    return change
+  }
+
+  private fun processTransitiveDependencies(name: String, moduleDescriptions: Map<String, UnloadedModuleDescriptionImpl>,
+                                            explicitlyUnloaded: Set<String>, result: MutableSet<ModuleDescription>) {
+    if (name in explicitlyUnloaded) return
+
+    val module = moduleDescriptions[name]
+    if (module == null || !result.add(module)) return
+
+    module.dependencyModuleNames.forEach {
+      processTransitiveDependencies(it, moduleDescriptions, explicitlyUnloaded, result)
+    }
+  }
+
+  private fun fireNotifications(change: UnloadedModulesListChange) {
+    if (change.toLoad.isEmpty() && change.toUnload.isEmpty()) return
+
+    val messages = ArrayList<String>()
+    val actions = ArrayList<NotificationAction>()
+    populateNotification(change.toUnload, messages, actions, "Load", {"Load $it back"}, change.toLoad.isEmpty(), {"unloaded"}) {
+      it.removeAll(change.toUnload.map { it.moduleName })
+    }
+    populateNotification(change.toLoad, messages, actions, "Unload", {"Unload $it"}, change.toUnload.isEmpty(), {"loaded because some other modules depend on $it"}) {
+      it.addAll(change.toLoad.map { it.moduleName })
+    }
+    actions.add(object: NotificationAction("Configure Unloaded Modules") {
+      override fun actionPerformed(e: AnActionEvent, notification: Notification) {
+        val ok = ConfigureUnloadedModulesDialog(project, null).showAndGet()
+        if (ok) {
+          notification.expire()
+        }
+      }
+    })
+
+    NOTIFICATION_GROUP.createNotification("New Modules are Added", XmlStringUtil.wrapInHtml(messages.joinToString("<br>")),
+                                          NotificationType.INFORMATION, null)
+      .apply {
+        actions.forEach { addAction(it) }
+      }
+      .notify(project)
+  }
+
+  private fun populateNotification(modules: List<ModulePath>,
+                                   messages: ArrayList<String>,
+                                   actions: ArrayList<NotificationAction>,
+                                   revertActionName: String,
+                                   revertActionShortText: (String) -> String,
+                                   useShortActionText: Boolean,
+                                   statusDescription: (String) -> String,
+                                   revertAction: (MutableList<String>) -> Unit) {
+    when {
+      modules.size == 1 -> {
+        val moduleName = modules.single().moduleName
+        messages.add("Newly added module '$moduleName' was automatically ${statusDescription("it")}.")
+        val text = if (useShortActionText) revertActionShortText("it") else "$revertActionName '$moduleName' module"
+        actions.add(createAction(text, revertAction))
+      }
+      modules.size == 2 -> {
+        val names = "'${modules[0].moduleName}' and '${modules[1].moduleName}'"
+        messages.add("Newly added modules $names were automatically ${statusDescription("them")}.")
+        val text = if (useShortActionText) revertActionShortText("them") else "$revertActionName modules $names"
+        actions.add(createAction(text, revertAction))
+      }
+      modules.size > 2 -> {
+        val names = "'${modules.first().moduleName}' and ${modules.size - 1} more modules"
+        messages.add("$names were automatically ${statusDescription("them")}.")
+        val text = if (useShortActionText) revertActionShortText("them") else "$revertActionName $names"
+        actions.add(createAction(text, revertAction))
+      }
+    }
+  }
+
+  fun createAction(text: String, action: (MutableList<String>) -> Unit) = object : NotificationAction(text) {
+    override fun actionPerformed(e: AnActionEvent, notification: Notification) {
+      val unloaded = ArrayList<String>()
+      val moduleManager = ModuleManager.getInstance(project)
+      moduleManager.unloadedModuleDescriptions.mapTo(unloaded) { it.name }
+      action(unloaded)
+      moduleManager.setUnloadedModules(unloaded)
+      notification.expire()
+    }
+  }
+
+  fun setLoadedModules(modules: List<String>) {
+    loadedModulesListStorage.modules.clear()
+    loadedModulesListStorage.modules.addAll(modules)
+  }
+
+  override fun getState() = loadedModulesListStorage
+
+  override fun loadState(state: LoadedModulesListStorage?) {
+    setLoadedModules(state?.modules ?: emptyList())
+  }
+
+  companion object {
+    @JvmStatic
+    fun getInstance(project: Project) = project.service<AutomaticModuleUnloader>()
+
+    private val NOTIFICATION_GROUP = NotificationGroup.balloonGroup("Automatic Module Unloading")
+  }
+}
+
+class LoadedModulesListStorage {
+  @Tag("loaded-modules")
+  @AbstractCollection(surroundWithTag = false, elementTag = "module", elementValueAttribute = "name")
+  var modules: MutableList<String> = ArrayList()
+}
+
+class UnloadedModulesListChange(val toLoad: List<ModulePath>, val toUnload: List<ModulePath>, val toUnloadDescriptions: List<UnloadedModuleDescriptionImpl>)

platform/lang-impl/src/com/intellij/openapi/module/impl/ModuleManagerComponent.java

@@ -36,7 +36,10 @@ import com.intellij.util.messages.MessageHandler;
 import org.jetbrains.annotations.NotNull;
 
 import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * @author yole
@@ -80,6 +83,13 @@ public class ModuleManagerComponent extends ModuleManagerImpl {
     myMessageBusConnection.subscribe(VirtualFileManager.VFS_CHANGES, new ModuleFileListener(this));
   }
 
+  @Override
+  protected void unloadNewlyAddedModulesIfPossible(Set<ModulePath> modulesToLoad, List<UnloadedModuleDescriptionImpl> modulesToUnload) {
+    UnloadedModulesListChange change = AutomaticModuleUnloader.getInstance(myProject).processNewModules(modulesToLoad, modulesToUnload);
+    modulesToLoad.removeAll(change.getToUnload());
+    modulesToUnload.addAll(change.getToUnloadDescriptions());
+  }
+
   @Override
   protected void showUnknownModuleTypeNotification(@NotNull List<Module> modulesWithUnknownTypes) {
     if (!ApplicationManager.getApplication().isHeadlessEnvironment() && !modulesWithUnknownTypes.isEmpty()) {
@@ -120,6 +130,16 @@ public class ModuleManagerComponent extends ModuleManagerImpl {
     return createModule(filePath);
   }
 
+  @Override
+  protected void setUnloadedModuleNames(@NotNull List<String> unloadedModuleNames) {
+    super.setUnloadedModuleNames(unloadedModuleNames);
+    if (!unloadedModuleNames.isEmpty()) {
+      List<String> loadedModules = new ArrayList<>(myModuleModel.myModules.keySet());
+      loadedModules.removeAll(new HashSet<>(unloadedModuleNames));
+      AutomaticModuleUnloader.getInstance(myProject).setLoadedModules(loadedModules);
+    }
+  }
+
   @Override
   protected boolean isUnknownModuleType(@NotNull Module module) {
     return ModuleType.get(module) instanceof UnknownModuleType;

platform/platform-resources/src/META-INF/LangExtensions.xml

@@ -254,6 +254,7 @@
                     serviceImplementation="com.intellij.facet.impl.invalid.InvalidFacetManagerImpl"/>
     <projectService serviceInterface="com.intellij.openapi.module.ProjectLoadingErrorsNotifier"
                     serviceImplementation="com.intellij.openapi.module.impl.ProjectLoadingErrorsNotifierImpl"/>
+    <projectService serviceImplementation="com.intellij.openapi.module.impl.AutomaticModuleUnloader"/>
 
     <moduleService serviceInterface="com.intellij.facet.FacetModificationTrackingService"
                    serviceImplementation="com.intellij.facet.impl.FacetModificationTrackingServiceImpl"/>

platform/projectModel-impl/src/com/intellij/openapi/module/impl/ModuleManagerImpl.java

@@ -60,6 +60,7 @@ import gnu.trove.TObjectHashingStrategy;
 import org.jdom.Element;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.TestOnly;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -165,8 +166,17 @@ public abstract class ModuleManagerImpl extends ModuleManager implements Disposa
 
   @Override
   public void loadState(Element state) {
+    loadState(getPathsToModuleFiles(state));
+  }
+
+  @TestOnly
+  public void loadStateFromModulePaths(LinkedHashSet<ModulePath> modulePaths) {
+    loadState(modulePaths);
+  }
+
+  private void loadState(LinkedHashSet<ModulePath> modulePaths) {
     boolean isFirstLoadState = myModulePathsToLoad == null;
-    myModulePathsToLoad = getPathsToModuleFiles(state);
+    myModulePathsToLoad = modulePaths;
     Set<String> unloadedModuleNames = new HashSet<>(UnloadedModulesListStorage.getInstance(myProject).getUnloadedModuleNames());
     Iterator<ModulePath> iterator = myModulePathsToLoad.iterator();
     List<ModulePath> unloadedModulePaths = new ArrayList<>();
@@ -177,9 +187,12 @@ public abstract class ModuleManagerImpl extends ModuleManager implements Disposa
         iterator.remove();
       }
     }
-    List<UnloadedModuleDescriptionImpl> descriptions = UnloadedModuleDescriptionImpl.createFromPaths(unloadedModulePaths, this);
+    List<UnloadedModuleDescriptionImpl> unloaded = new ArrayList<>(UnloadedModuleDescriptionImpl.createFromPaths(unloadedModulePaths, this));
+    if (!unloaded.isEmpty()) {
+      unloadNewlyAddedModulesIfPossible(myModulePathsToLoad, unloaded);
+    }
     myUnloadedModules.clear();
-    for (UnloadedModuleDescriptionImpl description : descriptions) {
+    for (UnloadedModuleDescriptionImpl description : unloaded) {
       myUnloadedModules.put(description.getName(), description);
     }
 
@@ -219,6 +232,9 @@ public abstract class ModuleManagerImpl extends ModuleManager implements Disposa
     myModulePathsToLoad.clear();
   }
 
+  protected void unloadNewlyAddedModulesIfPossible(Set<ModulePath> modulesToLoad, List<UnloadedModuleDescriptionImpl> modulesToUnload) {
+  }
+
   @NotNull
   // returns mutable linked hash set
   public static LinkedHashSet<ModulePath> getPathsToModuleFiles(@NotNull Element element) {
@@ -964,7 +980,7 @@ public abstract class ModuleManagerImpl extends ModuleManager implements Disposa
       fireModulesRenamed(modules, oldNames);
       cleanCachedStuff();
       UnloadedModulesListStorage unloadedModulesListStorage = UnloadedModulesListStorage.getInstance(myProject);
-      unloadedModulesListStorage.setUnloadedModuleNames(ContainerUtil.filter(unloadedModulesListStorage.getUnloadedModuleNames(), myUnloadedModules::containsKey));
+      setUnloadedModuleNames(ContainerUtil.filter(unloadedModulesListStorage.getUnloadedModuleNames(), myUnloadedModules::containsKey));
     }, false, true);
   }
 
@@ -1016,8 +1032,8 @@ public abstract class ModuleManagerImpl extends ModuleManager implements Disposa
       return;
     }
 
-    UnloadedModulesListStorage.getInstance(myProject).setUnloadedModuleNames(unloadedModuleNames);
-    
+    setUnloadedModuleNames(unloadedModuleNames);
+
     final ModifiableModuleModel model = getModifiableModel();
     Map<String, UnloadedModuleDescriptionImpl> toLoad = new LinkedHashMap<>(myUnloadedModules);
     myUnloadedModules.clear();
@@ -1049,6 +1065,10 @@ public abstract class ModuleManagerImpl extends ModuleManager implements Disposa
     myModulePathsToLoad.clear();
   }
 
+  protected void setUnloadedModuleNames(@NotNull List<String> unloadedModuleNames) {
+    UnloadedModulesListStorage.getInstance(myProject).setUnloadedModuleNames(unloadedModuleNames);
+  }
+
   public void setModuleGroupPath(Module module, String[] groupPath) {
     myModuleModel.setModuleGroupPath(module, groupPath);
   }

platform/projectModel-impl/src/com/intellij/openapi/module/impl/UnloadedModuleDescriptionImpl.kt

@@ -46,7 +46,7 @@ class UnloadedModuleDescriptionImpl(val modulePath: ModulePath,
 
   companion object {
     @JvmStatic
-    fun createFromPaths(paths: List<ModulePath>, parentDisposable: Disposable): List<UnloadedModuleDescriptionImpl> {
+    fun createFromPaths(paths: Collection<ModulePath>, parentDisposable: Disposable): List<UnloadedModuleDescriptionImpl> {
       val pathVariables = JpsGlobalLoader.computeAllPathVariables(PathManager.getOptionsPath())
       val modules = JpsProjectLoader.loadModules(paths.map { Paths.get(it.path) }, null, pathVariables)
       val pathsByName = paths.associateBy { it.moduleName }