build: clean output and caches of stale targets

If a build target become obsolete (e.g. its a module build target which module was removed or renamed) clean output files and caches for that target when all targets of that type are being built (IDEA-185574).

jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildDataPathsImpl.java

@@ -16,6 +16,7 @@
 package org.jetbrains.jps.builders.impl;
 
 import com.intellij.util.PathUtilRt;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jps.builders.BuildTarget;
 import org.jetbrains.jps.builders.BuildTargetType;
 import org.jetbrains.jps.builders.storage.BuildDataPaths;
@@ -49,9 +50,16 @@ public class BuildDataPathsImpl implements BuildDataPaths {
 
   @Override
   public File getTargetDataRoot(BuildTarget<?> target) {
+    BuildTargetType<?> targetType = target.getTargetType();
     final String targetId = target.getId();
+    return getTargetDataRoot(targetType, targetId);
+  }
+
+  @Override
+  @NotNull
+  public File getTargetDataRoot(@NotNull BuildTargetType<?> targetType, @NotNull String targetId) {
     // targetId may diff from another targetId only in case
     // when used as a file name in case-insensitive file systems, both paths for different targets will point to the same dir
-    return new File(getTargetTypeDataRoot(target.getTargetType()), PathUtilRt.suggestFileName(targetId + "_" + Integer.toHexString(targetId.hashCode()), true, false));
+    return new File(getTargetTypeDataRoot(targetType), PathUtilRt.suggestFileName(targetId + "_" + Integer.toHexString(targetId.hashCode()), true, false));
   }
 }

jps/jps-builders/src/org/jetbrains/jps/builders/storage/BuildDataPaths.java

@@ -15,6 +15,7 @@
  */
 package org.jetbrains.jps.builders.storage;
 
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jps.builders.BuildTarget;
 import org.jetbrains.jps.builders.BuildTargetType;
 
@@ -31,4 +32,7 @@ public interface BuildDataPaths {
   File getTargetTypeDataRoot(BuildTargetType<?> targetType);
 
   File getTargetDataRoot(BuildTarget<?> target);
+
+  @NotNull
+  File getTargetDataRoot(@NotNull BuildTargetType<?> targetType, @NotNull String targetId);
 }

jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScope.java

@@ -53,6 +53,11 @@ public abstract class CompileScope {
    */
   public abstract boolean isWholeTargetAffected(@NotNull BuildTarget<?> target);
 
+  /**
+   * @return {@code true} if all files from all targets of type {@code type} are included into the scope
+   */
+  public abstract boolean isAllTargetsOfTypeAffected(@NotNull BuildTargetType<?> type);
+
   /**
    * @return {@code true} if all files from {@code target} should be recompiled even if they weren't changed since last compilation
    */

jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScopeImpl.java

@@ -15,7 +15,6 @@
  */
 package org.jetbrains.jps.incremental;
 
-import java.util.HashSet;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jps.builders.BuildTarget;
 import org.jetbrains.jps.builders.BuildTargetType;
@@ -26,6 +25,7 @@ import org.jetbrains.jps.model.module.JpsModule;
 
 import java.io.File;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
@@ -70,6 +70,11 @@ public class CompileScopeImpl extends CompileScope {
     return (myTypes.contains(target.getTargetType()) || myTargets.contains(target) || isAffectedByAssociatedModule(target)) && !myFiles.containsKey(target);
   }
 
+  @Override
+  public boolean isAllTargetsOfTypeAffected(@NotNull BuildTargetType<?> type) {
+    return myTypes.contains(type) && myFiles.isEmpty();
+  }
+
   @Override
   public boolean isBuildForced(@NotNull BuildTarget<?> target) {
     return myTypesToForceBuild.contains(target.getTargetType()) && myFiles.isEmpty() && isWholeTargetAffected(target);
@@ -77,7 +82,7 @@ public class CompileScopeImpl extends CompileScope {
 
   @Override
   public boolean isBuildForcedForAllTargets(@NotNull BuildTargetType<?> targetType) {
-    return myTypesToForceBuild.contains(targetType) && myTypes.contains(targetType) && myFiles.isEmpty();
+    return myTypesToForceBuild.contains(targetType) && isAllTargetsOfTypeAffected(targetType);
   }
 
   @Override

jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java

@@ -40,6 +40,7 @@ import org.jetbrains.jps.incremental.messages.*;
 import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration;
 import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping;
 import org.jetbrains.jps.incremental.storage.OutputToTargetRegistry;
+import org.jetbrains.jps.incremental.storage.SourceToOutputMappingImpl;
 import org.jetbrains.jps.indices.ModuleExcludeIndex;
 import org.jetbrains.jps.javac.ExternalJavacManager;
 import org.jetbrains.jps.javac.JavacMain;
@@ -449,6 +450,11 @@ public class IncProjectBuilder {
           }
         }
       }
+      for (BuildTargetType<?> type : TargetTypeRegistry.getInstance().getTargetTypes()) {
+        if (context.getScope().isAllTargetsOfTypeAffected(type)) {
+          cleanOutputOfStaleTargets(type, context);
+        }
+      }
     }
     catch (ProjectBuildException e) {
       ex = e;
@@ -489,11 +495,46 @@ public class IncProjectBuilder {
     }
   }
 
+  private void cleanOutputOfStaleTargets(BuildTargetType<?> type, CompileContext context) {
+    List<Pair<String, Integer>> targetIds = myProjectDescriptor.dataManager.getTargetsState().getStaleTargetIds(type);
+    if (targetIds.isEmpty()) return;
+
+    context.processMessage(new ProgressMessage("Cleaning old output directories..."));
+    for (Pair<String, Integer> ids : targetIds) {
+      String stringId = ids.first;
+      try {
+        SourceToOutputMappingImpl mapping = null;
+        try {
+          mapping = myProjectDescriptor.dataManager.createSourceToOutputMapForStaleTarget(type, stringId);
+          clearOutputFiles(context, mapping, type);
+        }
+        finally {
+          if (mapping != null) {
+            mapping.close();
+          }
+        }
+        FileUtil.delete(myProjectDescriptor.dataManager.getDataPaths().getTargetDataRoot(type, stringId));
+        myProjectDescriptor.dataManager.getTargetsState().cleanStaleTarget(type, stringId);
+      }
+      catch (IOException e) {
+        LOG.warn(e);
+        myMessageDispatcher.processMessage(new CompilerMessage("", BuildMessage.Kind.WARNING, "Failed to delete output files from obsolete '" + stringId + "' target: " + e.toString()));
+      }
+    }
+  }
+
   public static void clearOutputFiles(CompileContext context, BuildTarget<?> target) throws IOException {
     final SourceToOutputMapping map = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
-    final THashSet<File> dirsToDelete = target instanceof ModuleBasedTarget ? new THashSet<>(FileUtil.FILE_HASHING_STRATEGY) : null;
-    for (String srcPath : map.getSources()) {
-      final Collection<String> outs = map.getOutputs(srcPath);
+    BuildTargetType<?> targetType = target.getTargetType();
+    clearOutputFiles(context, map, targetType);
+    registerTargetsWithClearedOutput(context, Collections.singletonList(target));
+  }
+
+  private static void clearOutputFiles(CompileContext context, SourceToOutputMapping mapping, BuildTargetType<?> targetType) throws IOException {
+    final THashSet<File> dirsToDelete = targetType instanceof ModuleBasedBuildTargetType<?>
+                                        ? new THashSet<>(FileUtil.FILE_HASHING_STRATEGY) : null;
+    for (String srcPath : mapping.getSources()) {
+      final Collection<String> outs = mapping.getOutputs(srcPath);
       if (outs != null && !outs.isEmpty()) {
         List<String> deletedPaths = new ArrayList<>();
         for (String out : outs) {
@@ -504,7 +545,6 @@ public class IncProjectBuilder {
         }
       }
     }
-    registerTargetsWithClearedOutput(context, Collections.singletonList(target));
     if (dirsToDelete != null) {
       FSOperations.pruneEmptyDirs(context, dirsToDelete);
     }

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java

@@ -22,6 +22,7 @@ import com.intellij.util.io.PersistentHashMapValueStorage;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.jps.builders.BuildTarget;
+import org.jetbrains.jps.builders.BuildTargetType;
 import org.jetbrains.jps.builders.impl.BuildTargetChunk;
 import org.jetbrains.jps.builders.impl.storage.BuildTargetStorages;
 import org.jetbrains.jps.builders.java.dependencyView.Mappings;
@@ -147,6 +148,10 @@ public class BuildDataManager implements StorageOwner {
     return new SourceToOutputMappingWrapper(sourceToOutputMapping, buildTargetId);
   }
 
+  public SourceToOutputMappingImpl createSourceToOutputMapForStaleTarget(BuildTargetType<?> targetType, String targetId) throws IOException {
+    return new SourceToOutputMappingImpl(new File(getSourceToOutputMapRoot(targetType, targetId), "data"));
+  }
+
   @NotNull
   public <S extends StorageOwner> S getStorage(@NotNull BuildTarget<?> target, @NotNull StorageProvider<S> provider) throws IOException {
     final BuildTargetStorages storages = fetchValue(myTargetStorages, target, TARGET_STORAGES_VALUE_FACTORY);
@@ -330,6 +335,10 @@ public class BuildDataManager implements StorageOwner {
     return new File(myDataPaths.getTargetDataRoot(target), "src-out");
   }
 
+  private File getSourceToOutputMapRoot(BuildTargetType<?> targetType, String targetId) {
+    return new File(myDataPaths.getTargetDataRoot(targetType, targetId), "src-out");
+  }
+
   private File getSourceToFormsRoot() {
     return new File(myDataPaths.getDataStorageRoot(), SRC_TO_FORM_STORAGE);
   }

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetTypeState.java

@@ -16,6 +16,7 @@
 package org.jetbrains.jps.incremental.storage;
 
 import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.util.io.IOUtil;
 import org.jetbrains.jps.builders.BuildTarget;
@@ -23,7 +24,9 @@ import org.jetbrains.jps.builders.BuildTargetLoader;
 import org.jetbrains.jps.builders.BuildTargetType;
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -34,6 +37,7 @@ import java.util.concurrent.ConcurrentMap;
 public class BuildTargetTypeState {
   private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.storage.BuildTargetTypeState");
   private final Map<BuildTarget<?>, Integer> myTargetIds;
+  private final List<Pair<String, Integer>> myStaleTargetIds;
   private final ConcurrentMap<BuildTarget<?>, BuildTargetConfiguration> myConfigurations;
   private final BuildTargetType<?> myTargetType;
   private final BuildTargetsState myTargetsState;
@@ -45,6 +49,7 @@ public class BuildTargetTypeState {
     myTargetsFile = new File(state.getDataPaths().getTargetTypeDataRoot(targetType), "targets.dat");
     myConfigurations = new ConcurrentHashMap<>(16, 0.75f, 1);
     myTargetIds = new HashMap<>();
+    myStaleTargetIds = new ArrayList<>();
     load();
   }
 
@@ -68,7 +73,7 @@ public class BuildTargetTypeState {
             myTargetIds.put(target, intId);
           }
           else {
-            LOG.info("Unknown " + myTargetType.getTypeId() + " target: " + stringId);
+            myStaleTargetIds.add(Pair.create(stringId, intId));
           }
         }
         return true;
@@ -89,11 +94,15 @@ public class BuildTargetTypeState {
       DataOutputStream output = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(myTargetsFile)));
       try {
         output.writeInt(0);
-        output.writeInt(myTargetIds.size());
+        output.writeInt(myTargetIds.size() + myStaleTargetIds.size());
         for (Map.Entry<BuildTarget<?>, Integer> entry : myTargetIds.entrySet()) {
           IOUtil.writeString(entry.getKey().getId(), output);
           output.writeInt(entry.getValue());
         }
+        for (Pair<String, Integer> pair : myStaleTargetIds) {
+          IOUtil.writeString(pair.first, output);
+          output.writeInt(pair.second);
+        }
       }
       finally {
         output.close();
@@ -104,6 +113,14 @@ public class BuildTargetTypeState {
     }
   }
 
+  public synchronized List<Pair<String, Integer>> getStaleTargetIds() {
+    return new ArrayList<>(myStaleTargetIds);
+  }
+
+  public synchronized void removeStaleTarget(String targetId) {
+    myStaleTargetIds.removeIf(pair -> pair.first.equals(targetId));
+  }
+
   public synchronized int getTargetId(BuildTarget<?> target) {
     if (!myTargetIds.containsKey(target)) {
       myTargetIds.put(target, myTargetsState.getFreeId());

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetsState.java

@@ -16,6 +16,7 @@
 package org.jetbrains.jps.incremental.storage;
 
 import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.util.io.FileUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jps.builders.BuildTarget;
@@ -26,6 +27,7 @@ import org.jetbrains.jps.incremental.TargetTypeRegistry;
 import org.jetbrains.jps.model.JpsModel;
 
 import java.io.*;
+import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -96,6 +98,14 @@ public class BuildTargetsState {
     return getTypeState(target.getTargetType()).getConfiguration(target);
   }
 
+  public List<Pair<String, Integer>> getStaleTargetIds(@NotNull BuildTargetType<?> type) {
+    return getTypeState(type).getStaleTargetIds();
+  }
+
+  public void cleanStaleTarget(BuildTargetType<?> type, String targetId) {
+    getTypeState(type).removeStaleTarget(targetId);
+  }
+
   private BuildTargetTypeState getTypeState(BuildTargetType<?> type) {
     BuildTargetTypeState state = myTypeStates.get(type);
     if (state == null) {

jps/jps-builders/testSrc/org/jetbrains/jps/builders/CleanStaleTargetsTest.kt

@@ -0,0 +1,53 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package org.jetbrains.jps.builders
+
+import com.intellij.util.PathUtil
+import com.intellij.util.io.directoryContent
+import org.jetbrains.jps.model.java.JpsJavaExtensionService
+import org.jetbrains.jps.model.module.JpsModule
+import org.jetbrains.jps.util.JpsPathUtil
+
+class CleanStaleTargetsTest : JpsBuildTestCase() {
+  fun `test delete old output when module is deleted`() {
+    doTestDeleteOldOutput {
+      myProject.removeModule(it)
+    }
+  }
+
+  fun `test delete old output when module is renamed`() {
+    doTestDeleteOldOutput {
+      it.name = "a2"
+    }
+  }
+
+  fun `test delete old output when module output is changed`() {
+    doTestDeleteOldOutput {
+      val moduleExtension = JpsJavaExtensionService.getInstance().getOrCreateModuleExtension(it)
+      moduleExtension.isInheritOutput = false
+      moduleExtension.outputUrl = JpsPathUtil.pathToUrl(getAbsolutePath("out/a2"))
+    }
+  }
+
+  private fun doTestDeleteOldOutput(action: (JpsModule) -> Unit) {
+    JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(myProject).outputUrl = JpsPathUtil.pathToUrl(getAbsolutePath("out"))
+    val aRoot = PathUtil.getParentPath(createFile("a/src/A.java", "class A {}"))
+    val aModule = addModule("a", arrayOf(aRoot), null, null, jdk)
+    val bRoot = PathUtil.getParentPath(createFile("b/src/B.java", "class B {}"))
+    val bModule = addModule("b", arrayOf(bRoot), null, null, jdk)
+    rebuildAllModules()
+
+    val aOutput = getModuleOutput(aModule)
+    assertOutput(aOutput.absolutePath, directoryContent { file("A.class") })
+    action(aModule)
+
+    doBuild(CompileScopeTestBuilder.make().module(bModule))
+    //do not clean output when just one other target is built to avoid unexpectedly long builds
+    assertOutput(aOutput.absolutePath, directoryContent { file("A.class") })
+
+    buildAllModules()
+    //clean output of stale targets when all targets of this type are built
+    if (aOutput.exists()) {
+      assertOutput(aOutput.absolutePath, directoryContent { })
+    }
+  }
+}

jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java

@@ -440,16 +440,27 @@ public abstract class JpsBuildTestCase extends UsefulTestCase {
   }
 
   public JpsModule addModule(String moduleName, String... srcPaths) {
+    return addModule(moduleName, srcPaths, getAbsolutePath(getModuleOutputRelativePath(moduleName)), null, getJdk());
+  }
+
+  protected final JpsSdk<JpsDummyElement> getJdk() {
     if (myJdk == null) {
       myJdk = addJdk("1.6");
     }
-    return addModule(moduleName, srcPaths, getAbsolutePath(getModuleOutputRelativePath(moduleName)), null, myJdk);
+    return myJdk;
+  }
+
+  @NotNull
+  protected static File getModuleOutput(JpsModule module) {
+    String outputUrl = JpsJavaExtensionService.getInstance().getOutputUrl(module, false);
+    return JpsPathUtil.urlToFile(outputUrl);
   }
 
   @NotNull
   protected String getModuleOutputRelativePath(JpsModule module) {
     return getModuleOutputRelativePath(module.getName());
   }
+
   @NotNull
   protected String getModuleOutputRelativePath(String moduleName) {
     return "out/production/" + moduleName;

jps/model-api/src/org/jetbrains/jps/model/JpsProject.java

@@ -40,6 +40,8 @@ public interface JpsProject extends JpsCompositeElement, JpsReferenceableElement
 
   void addModule(@NotNull JpsModule module);
 
+  void removeModule(@NotNull JpsModule module);
+
   @NotNull
   List<JpsModule> getModules();
 

jps/model-impl/src/org/jetbrains/jps/model/impl/JpsProjectImpl.java

@@ -114,6 +114,11 @@ public class JpsProjectImpl extends JpsRootElementBase<JpsProjectImpl> implement
     myContainer.getChild(JpsModuleRole.MODULE_COLLECTION_ROLE).addChild(module);
   }
 
+  @Override
+  public void removeModule(@NotNull JpsModule module) {
+    myContainer.getChild(JpsModuleRole.MODULE_COLLECTION_ROLE).removeChild(module);
+  }
+
   @NotNull
   @Override
   public JpsLibraryCollection getLibraryCollection() {