diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildDataPathsImpl.java b/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildDataPathsImpl.java index 1c81ec9f18e2..69dccca81621 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildDataPathsImpl.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildDataPathsImpl.java @@ -16,6 +16,7 @@ package org.jetbrains.jps.builders.impl; import com.intellij.util.PathUtilRt; +import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.BuildTargetType; import org.jetbrains.jps.builders.storage.BuildDataPaths; @@ -49,9 +50,16 @@ public class BuildDataPathsImpl implements BuildDataPaths { @Override public File getTargetDataRoot(BuildTarget target) { + BuildTargetType targetType = target.getTargetType(); final String targetId = target.getId(); + return getTargetDataRoot(targetType, targetId); + } + + @Override + @NotNull + public File getTargetDataRoot(@NotNull BuildTargetType targetType, @NotNull String targetId) { // targetId may diff from another targetId only in case // when used as a file name in case-insensitive file systems, both paths for different targets will point to the same dir - return new File(getTargetTypeDataRoot(target.getTargetType()), PathUtilRt.suggestFileName(targetId + "_" + Integer.toHexString(targetId.hashCode()), true, false)); + return new File(getTargetTypeDataRoot(targetType), PathUtilRt.suggestFileName(targetId + "_" + Integer.toHexString(targetId.hashCode()), true, false)); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/storage/BuildDataPaths.java b/jps/jps-builders/src/org/jetbrains/jps/builders/storage/BuildDataPaths.java index fa5e3fb1ed0f..20c65e83730f 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/storage/BuildDataPaths.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/storage/BuildDataPaths.java @@ -15,6 +15,7 @@ */ package org.jetbrains.jps.builders.storage; +import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.BuildTargetType; @@ -31,4 +32,7 @@ public interface BuildDataPaths { File getTargetTypeDataRoot(BuildTargetType targetType); File getTargetDataRoot(BuildTarget target); + + @NotNull + File getTargetDataRoot(@NotNull BuildTargetType targetType, @NotNull String targetId); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScope.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScope.java index 205d47478f65..2ed8d312a8f4 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScope.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScope.java @@ -53,6 +53,11 @@ public abstract class CompileScope { */ public abstract boolean isWholeTargetAffected(@NotNull BuildTarget target); + /** + * @return {@code true} if all files from all targets of type {@code type} are included into the scope + */ + public abstract boolean isAllTargetsOfTypeAffected(@NotNull BuildTargetType type); + /** * @return {@code true} if all files from {@code target} should be recompiled even if they weren't changed since last compilation */ diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScopeImpl.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScopeImpl.java index d0774d72cd61..cf0bcd34e4ab 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScopeImpl.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileScopeImpl.java @@ -15,7 +15,6 @@ */ package org.jetbrains.jps.incremental; -import java.util.HashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.BuildTargetType; @@ -26,6 +25,7 @@ import org.jetbrains.jps.model.module.JpsModule; import java.io.File; import java.util.Collection; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -70,6 +70,11 @@ public class CompileScopeImpl extends CompileScope { return (myTypes.contains(target.getTargetType()) || myTargets.contains(target) || isAffectedByAssociatedModule(target)) && !myFiles.containsKey(target); } + @Override + public boolean isAllTargetsOfTypeAffected(@NotNull BuildTargetType type) { + return myTypes.contains(type) && myFiles.isEmpty(); + } + @Override public boolean isBuildForced(@NotNull BuildTarget target) { return myTypesToForceBuild.contains(target.getTargetType()) && myFiles.isEmpty() && isWholeTargetAffected(target); @@ -77,7 +82,7 @@ public class CompileScopeImpl extends CompileScope { @Override public boolean isBuildForcedForAllTargets(@NotNull BuildTargetType targetType) { - return myTypesToForceBuild.contains(targetType) && myTypes.contains(targetType) && myFiles.isEmpty(); + return myTypesToForceBuild.contains(targetType) && isAllTargetsOfTypeAffected(targetType); } @Override diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 8a8f76b7eca7..707117d25cd1 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -40,6 +40,7 @@ import org.jetbrains.jps.incremental.messages.*; import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration; import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping; import org.jetbrains.jps.incremental.storage.OutputToTargetRegistry; +import org.jetbrains.jps.incremental.storage.SourceToOutputMappingImpl; import org.jetbrains.jps.indices.ModuleExcludeIndex; import org.jetbrains.jps.javac.ExternalJavacManager; import org.jetbrains.jps.javac.JavacMain; @@ -449,6 +450,11 @@ public class IncProjectBuilder { } } } + for (BuildTargetType type : TargetTypeRegistry.getInstance().getTargetTypes()) { + if (context.getScope().isAllTargetsOfTypeAffected(type)) { + cleanOutputOfStaleTargets(type, context); + } + } } catch (ProjectBuildException e) { ex = e; @@ -489,11 +495,46 @@ public class IncProjectBuilder { } } + private void cleanOutputOfStaleTargets(BuildTargetType type, CompileContext context) { + List> targetIds = myProjectDescriptor.dataManager.getTargetsState().getStaleTargetIds(type); + if (targetIds.isEmpty()) return; + + context.processMessage(new ProgressMessage("Cleaning old output directories...")); + for (Pair ids : targetIds) { + String stringId = ids.first; + try { + SourceToOutputMappingImpl mapping = null; + try { + mapping = myProjectDescriptor.dataManager.createSourceToOutputMapForStaleTarget(type, stringId); + clearOutputFiles(context, mapping, type); + } + finally { + if (mapping != null) { + mapping.close(); + } + } + FileUtil.delete(myProjectDescriptor.dataManager.getDataPaths().getTargetDataRoot(type, stringId)); + myProjectDescriptor.dataManager.getTargetsState().cleanStaleTarget(type, stringId); + } + catch (IOException e) { + LOG.warn(e); + myMessageDispatcher.processMessage(new CompilerMessage("", BuildMessage.Kind.WARNING, "Failed to delete output files from obsolete '" + stringId + "' target: " + e.toString())); + } + } + } + public static void clearOutputFiles(CompileContext context, BuildTarget target) throws IOException { final SourceToOutputMapping map = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target); - final THashSet dirsToDelete = target instanceof ModuleBasedTarget ? new THashSet<>(FileUtil.FILE_HASHING_STRATEGY) : null; - for (String srcPath : map.getSources()) { - final Collection outs = map.getOutputs(srcPath); + BuildTargetType targetType = target.getTargetType(); + clearOutputFiles(context, map, targetType); + registerTargetsWithClearedOutput(context, Collections.singletonList(target)); + } + + private static void clearOutputFiles(CompileContext context, SourceToOutputMapping mapping, BuildTargetType targetType) throws IOException { + final THashSet dirsToDelete = targetType instanceof ModuleBasedBuildTargetType + ? new THashSet<>(FileUtil.FILE_HASHING_STRATEGY) : null; + for (String srcPath : mapping.getSources()) { + final Collection outs = mapping.getOutputs(srcPath); if (outs != null && !outs.isEmpty()) { List deletedPaths = new ArrayList<>(); for (String out : outs) { @@ -504,7 +545,6 @@ public class IncProjectBuilder { } } } - registerTargetsWithClearedOutput(context, Collections.singletonList(target)); if (dirsToDelete != null) { FSOperations.pruneEmptyDirs(context, dirsToDelete); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index 9b76e5706a00..dea70f054089 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -22,6 +22,7 @@ import com.intellij.util.io.PersistentHashMapValueStorage; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.builders.BuildTarget; +import org.jetbrains.jps.builders.BuildTargetType; import org.jetbrains.jps.builders.impl.BuildTargetChunk; import org.jetbrains.jps.builders.impl.storage.BuildTargetStorages; import org.jetbrains.jps.builders.java.dependencyView.Mappings; @@ -147,6 +148,10 @@ public class BuildDataManager implements StorageOwner { return new SourceToOutputMappingWrapper(sourceToOutputMapping, buildTargetId); } + public SourceToOutputMappingImpl createSourceToOutputMapForStaleTarget(BuildTargetType targetType, String targetId) throws IOException { + return new SourceToOutputMappingImpl(new File(getSourceToOutputMapRoot(targetType, targetId), "data")); + } + @NotNull public S getStorage(@NotNull BuildTarget target, @NotNull StorageProvider provider) throws IOException { final BuildTargetStorages storages = fetchValue(myTargetStorages, target, TARGET_STORAGES_VALUE_FACTORY); @@ -330,6 +335,10 @@ public class BuildDataManager implements StorageOwner { return new File(myDataPaths.getTargetDataRoot(target), "src-out"); } + private File getSourceToOutputMapRoot(BuildTargetType targetType, String targetId) { + return new File(myDataPaths.getTargetDataRoot(targetType, targetId), "src-out"); + } + private File getSourceToFormsRoot() { return new File(myDataPaths.getDataStorageRoot(), SRC_TO_FORM_STORAGE); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetTypeState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetTypeState.java index 9010515f3f5b..205cba4291f6 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetTypeState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetTypeState.java @@ -16,6 +16,7 @@ package org.jetbrains.jps.incremental.storage; import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.io.IOUtil; import org.jetbrains.jps.builders.BuildTarget; @@ -23,7 +24,9 @@ import org.jetbrains.jps.builders.BuildTargetLoader; import org.jetbrains.jps.builders.BuildTargetType; import java.io.*; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -34,6 +37,7 @@ import java.util.concurrent.ConcurrentMap; public class BuildTargetTypeState { private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.storage.BuildTargetTypeState"); private final Map, Integer> myTargetIds; + private final List> myStaleTargetIds; private final ConcurrentMap, BuildTargetConfiguration> myConfigurations; private final BuildTargetType myTargetType; private final BuildTargetsState myTargetsState; @@ -45,6 +49,7 @@ public class BuildTargetTypeState { myTargetsFile = new File(state.getDataPaths().getTargetTypeDataRoot(targetType), "targets.dat"); myConfigurations = new ConcurrentHashMap<>(16, 0.75f, 1); myTargetIds = new HashMap<>(); + myStaleTargetIds = new ArrayList<>(); load(); } @@ -68,7 +73,7 @@ public class BuildTargetTypeState { myTargetIds.put(target, intId); } else { - LOG.info("Unknown " + myTargetType.getTypeId() + " target: " + stringId); + myStaleTargetIds.add(Pair.create(stringId, intId)); } } return true; @@ -89,11 +94,15 @@ public class BuildTargetTypeState { DataOutputStream output = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(myTargetsFile))); try { output.writeInt(0); - output.writeInt(myTargetIds.size()); + output.writeInt(myTargetIds.size() + myStaleTargetIds.size()); for (Map.Entry, Integer> entry : myTargetIds.entrySet()) { IOUtil.writeString(entry.getKey().getId(), output); output.writeInt(entry.getValue()); } + for (Pair pair : myStaleTargetIds) { + IOUtil.writeString(pair.first, output); + output.writeInt(pair.second); + } } finally { output.close(); @@ -104,6 +113,14 @@ public class BuildTargetTypeState { } } + public synchronized List> getStaleTargetIds() { + return new ArrayList<>(myStaleTargetIds); + } + + public synchronized void removeStaleTarget(String targetId) { + myStaleTargetIds.removeIf(pair -> pair.first.equals(targetId)); + } + public synchronized int getTargetId(BuildTarget target) { if (!myTargetIds.containsKey(target)) { myTargetIds.put(target, myTargetsState.getFreeId()); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetsState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetsState.java index 1686b17e5440..817fde591e56 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetsState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetsState.java @@ -16,6 +16,7 @@ package org.jetbrains.jps.incremental.storage; import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.io.FileUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.BuildTarget; @@ -26,6 +27,7 @@ import org.jetbrains.jps.incremental.TargetTypeRegistry; import org.jetbrains.jps.model.JpsModel; import java.io.*; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @@ -96,6 +98,14 @@ public class BuildTargetsState { return getTypeState(target.getTargetType()).getConfiguration(target); } + public List> getStaleTargetIds(@NotNull BuildTargetType type) { + return getTypeState(type).getStaleTargetIds(); + } + + public void cleanStaleTarget(BuildTargetType type, String targetId) { + getTypeState(type).removeStaleTarget(targetId); + } + private BuildTargetTypeState getTypeState(BuildTargetType type) { BuildTargetTypeState state = myTypeStates.get(type); if (state == null) { diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/CleanStaleTargetsTest.kt b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/CleanStaleTargetsTest.kt new file mode 100644 index 000000000000..59e3be06a13a --- /dev/null +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/CleanStaleTargetsTest.kt @@ -0,0 +1,53 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.jps.builders + +import com.intellij.util.PathUtil +import com.intellij.util.io.directoryContent +import org.jetbrains.jps.model.java.JpsJavaExtensionService +import org.jetbrains.jps.model.module.JpsModule +import org.jetbrains.jps.util.JpsPathUtil + +class CleanStaleTargetsTest : JpsBuildTestCase() { + fun `test delete old output when module is deleted`() { + doTestDeleteOldOutput { + myProject.removeModule(it) + } + } + + fun `test delete old output when module is renamed`() { + doTestDeleteOldOutput { + it.name = "a2" + } + } + + fun `test delete old output when module output is changed`() { + doTestDeleteOldOutput { + val moduleExtension = JpsJavaExtensionService.getInstance().getOrCreateModuleExtension(it) + moduleExtension.isInheritOutput = false + moduleExtension.outputUrl = JpsPathUtil.pathToUrl(getAbsolutePath("out/a2")) + } + } + + private fun doTestDeleteOldOutput(action: (JpsModule) -> Unit) { + JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(myProject).outputUrl = JpsPathUtil.pathToUrl(getAbsolutePath("out")) + val aRoot = PathUtil.getParentPath(createFile("a/src/A.java", "class A {}")) + val aModule = addModule("a", arrayOf(aRoot), null, null, jdk) + val bRoot = PathUtil.getParentPath(createFile("b/src/B.java", "class B {}")) + val bModule = addModule("b", arrayOf(bRoot), null, null, jdk) + rebuildAllModules() + + val aOutput = getModuleOutput(aModule) + assertOutput(aOutput.absolutePath, directoryContent { file("A.class") }) + action(aModule) + + doBuild(CompileScopeTestBuilder.make().module(bModule)) + //do not clean output when just one other target is built to avoid unexpectedly long builds + assertOutput(aOutput.absolutePath, directoryContent { file("A.class") }) + + buildAllModules() + //clean output of stale targets when all targets of this type are built + if (aOutput.exists()) { + assertOutput(aOutput.absolutePath, directoryContent { }) + } + } +} diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java index 0295f49fca90..0f829cc15ad6 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java @@ -440,16 +440,27 @@ public abstract class JpsBuildTestCase extends UsefulTestCase { } public JpsModule addModule(String moduleName, String... srcPaths) { + return addModule(moduleName, srcPaths, getAbsolutePath(getModuleOutputRelativePath(moduleName)), null, getJdk()); + } + + protected final JpsSdk getJdk() { if (myJdk == null) { myJdk = addJdk("1.6"); } - return addModule(moduleName, srcPaths, getAbsolutePath(getModuleOutputRelativePath(moduleName)), null, myJdk); + return myJdk; + } + + @NotNull + protected static File getModuleOutput(JpsModule module) { + String outputUrl = JpsJavaExtensionService.getInstance().getOutputUrl(module, false); + return JpsPathUtil.urlToFile(outputUrl); } @NotNull protected String getModuleOutputRelativePath(JpsModule module) { return getModuleOutputRelativePath(module.getName()); } + @NotNull protected String getModuleOutputRelativePath(String moduleName) { return "out/production/" + moduleName; diff --git a/jps/model-api/src/org/jetbrains/jps/model/JpsProject.java b/jps/model-api/src/org/jetbrains/jps/model/JpsProject.java index 35834335332d..6619df7ee100 100644 --- a/jps/model-api/src/org/jetbrains/jps/model/JpsProject.java +++ b/jps/model-api/src/org/jetbrains/jps/model/JpsProject.java @@ -40,6 +40,8 @@ public interface JpsProject extends JpsCompositeElement, JpsReferenceableElement void addModule(@NotNull JpsModule module); + void removeModule(@NotNull JpsModule module); + @NotNull List getModules(); diff --git a/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsProjectImpl.java b/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsProjectImpl.java index e43f2b2d86fc..d02379ebacf1 100644 --- a/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsProjectImpl.java +++ b/jps/model-impl/src/org/jetbrains/jps/model/impl/JpsProjectImpl.java @@ -114,6 +114,11 @@ public class JpsProjectImpl extends JpsRootElementBase implement myContainer.getChild(JpsModuleRole.MODULE_COLLECTION_ROLE).addChild(module); } + @Override + public void removeModule(@NotNull JpsModule module) { + myContainer.getChild(JpsModuleRole.MODULE_COLLECTION_ROLE).removeChild(module); + } + @NotNull @Override public JpsLibraryCollection getLibraryCollection() {