From 60e173296860f2b917ceec6b36d25a8285f441af Mon Sep 17 00:00:00 2001 From: Eugene Zhuravlev Date: Mon, 18 Jun 2012 23:09:37 +0200 Subject: [PATCH] external build: handle module output directory changes/module renames: recompile whole module ensure up-to-date project model is used by the build by saving the project before compilation faster rebuild in case of forced storage cleanup --- .../intellij/compiler/impl/CompileDriver.java | 13 ++++- .../compiler/server/BuildManager.java | 11 +++- .../jps/incremental/CompileContext.java | 47 ++++++++++++++- .../jps/incremental/IncProjectBuilder.java | 16 ++++++ .../incremental/storage/BuildDataManager.java | 57 +++++++++++++------ .../storage/ModuleOutputRootsLayout.java | 39 +++++++++++++ 6 files changed, 159 insertions(+), 24 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ModuleOutputRootsLayout.java diff --git a/java/compiler/impl/src/com/intellij/compiler/impl/CompileDriver.java b/java/compiler/impl/src/com/intellij/compiler/impl/CompileDriver.java index da2282d82411..8363e6278bf1 100644 --- a/java/compiler/impl/src/com/intellij/compiler/impl/CompileDriver.java +++ b/java/compiler/impl/src/com/intellij/compiler/impl/CompileDriver.java @@ -648,7 +648,10 @@ public class CompileDriver { new CompilerTask(myProject, compileInBackground, contentName, ApplicationManager.getApplication().isUnitTestMode()); StatusBar.Info.set("", myProject, "Compiler"); - + if (useOutOfProcessBuild() && BuildManager.getInstance().rescanRequired(myProject)) { + // ensure the project model seen by build process is up-to-date + myProject.save(); + } PsiDocumentManager.getInstance(myProject).commitAllDocuments(); FileDocumentManager.getInstance().saveAllDocuments(); @@ -686,7 +689,7 @@ public class CompileDriver { compileContext.addMessage(message); } final Collection paths = fetchFiles(compileContext); - final List modules = paths.isEmpty()? Arrays.asList(compileContext.getCompileScope().getAffectedModules()) : Collections.emptyList(); + final List modules = paths.isEmpty() && !isRebuild && !allProjectModulesAffected(compileContext)? Arrays.asList(compileContext.getCompileScope().getAffectedModules()) : Collections.emptyList(); final Set artifacts = ArtifactCompileScope.getArtifactsToBuild(myProject, compileContext.getCompileScope(), true); final RequestFuture future = compileInExternalProcess(compileContext, modules, artifacts, paths, callback); if (future != null) { @@ -780,6 +783,12 @@ public class CompileDriver { }); } + private static boolean allProjectModulesAffected(CompileContextImpl compileContext) { + final Set allModules = new HashSet(Arrays.asList(compileContext.getProjectCompileScope().getAffectedModules())); + allModules.removeAll(Arrays.asList(compileContext.getCompileScope().getAffectedModules())); + return allModules.isEmpty(); + } + private static List fetchFiles(CompileContextImpl context) { if (context.isRebuild()) { return Collections.emptyList(); diff --git a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java index da89f9948413..e57348f4e328 100644 --- a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java +++ b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java @@ -78,7 +78,8 @@ import org.jetbrains.jps.cmdline.BuildMain; import org.jetbrains.jps.server.ClasspathBootstrap; import org.jetbrains.jps.server.Server; -import javax.tools.*; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -222,6 +223,14 @@ public class BuildManager implements ApplicationComponent{ } } + public boolean rescanRequired(Project project) { + final String projectPath = getProjectPath(project); + synchronized (myProjectDataMap) { + final ProjectData data = myProjectDataMap.get(projectPath); + return data == null || data.myNeedRescan; + } + } + @Nullable private static String getProjectPath(final Project project) { final String path = project.getPresentableUrl(); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java index 49161f672335..a044f91a5510 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java @@ -2,6 +2,8 @@ package org.jetbrains.jps.incremental; import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.UserDataHolderBase; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.EventDispatcher; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -14,6 +16,7 @@ import org.jetbrains.jps.incremental.messages.FileGeneratedEvent; import org.jetbrains.jps.incremental.messages.ProgressMessage; import org.jetbrains.jps.incremental.messages.UptoDateFilesSavedEvent; import org.jetbrains.jps.incremental.storage.BuildDataManager; +import org.jetbrains.jps.incremental.storage.ModuleOutputRootsLayout; import org.jetbrains.jps.incremental.storage.SourceToOutputMapping; import org.jetbrains.jps.incremental.storage.Timestamps; import org.jetbrains.jps.server.ProjectDescriptor; @@ -350,17 +353,20 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler for (Module module : chunk.getModules()) { if (isProjectRebuild()) { markDirtyFiles(module, myTimestamps, true, isCompilingTests() ? DirtyMarkScope.TESTS : DirtyMarkScope.PRODUCTION, null); + updateOutputRootsLayout(module); } else { if (isMake()) { if (myProjectDescriptor.fsState.markInitialScanPerformed(module.getName(), isCompilingTests())) { initModuleFSState(module); + updateOutputRootsLayout(module); } } else { // forced compilation mode if (getScope().isRecompilationForced(module.getName())) { markDirtyFiles(module, myTimestamps, true, isCompilingTests() ? DirtyMarkScope.TESTS : DirtyMarkScope.PRODUCTION, null); + updateOutputRootsLayout(module); } } } @@ -368,9 +374,24 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler } private void initModuleFSState(Module module) throws IOException { - final HashSet currentFiles = new HashSet(); - markDirtyFiles(module, myTimestamps, false, isCompilingTests() ? DirtyMarkScope.TESTS : DirtyMarkScope.PRODUCTION, currentFiles); + boolean forceMarkDirty = false; + final File currentOutput = getProjectPaths().getModuleOutputDir(module, isCompilingTests()); + if (currentOutput != null) { + Pair outputsPair = getDataManager().getOutputRootsLayout().getState(module.getName()); + if (outputsPair != null) { + final String previousPath = isCompilingTests() ? outputsPair.second : outputsPair.first; + forceMarkDirty = StringUtil.isEmpty(previousPath) || !FileUtil.filesEqual(currentOutput, new File(previousPath)); + } + else { + forceMarkDirty = true; + } + } + final HashSet currentFiles = new HashSet(); + markDirtyFiles(module, myTimestamps, forceMarkDirty, isCompilingTests() ? DirtyMarkScope.TESTS : DirtyMarkScope.PRODUCTION, currentFiles); + + // handle deleted paths + myProjectDescriptor.fsState.clearDeletedPaths(module.getName(), isCompilingTests()); final SourceToOutputMapping sourceToOutputMap = getDataManager().getSourceToOutputMap(module.getName(), isCompilingTests()); for (final Iterator it = sourceToOutputMap.getKeysIterator(); it.hasNext();) { final String path = it.next(); @@ -382,6 +403,27 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler } } + private void updateOutputRootsLayout(Module module) throws IOException { + final File currentOutput = getProjectPaths().getModuleOutputDir(module, isCompilingTests()); + if (currentOutput == null) { + return; + } + final ModuleOutputRootsLayout outputRootsLayout = getDataManager().getOutputRootsLayout(); + Pair outputsPair = outputRootsLayout.getState(module.getName()); + // update data + final String productionPath; + final String testPath; + if (isCompilingTests()) { + productionPath = outputsPair != null? outputsPair.first : ""; + testPath = FileUtil.toSystemIndependentName(currentOutput.getPath()); + } + else { + productionPath = FileUtil.toSystemIndependentName(currentOutput.getPath()); + testPath = outputsPair != null? outputsPair.second : ""; + } + outputRootsLayout.update(module.getName(), Pair.create(productionPath, testPath)); + } + public boolean hasRemovedSources() { final Map> removed = Utils.REMOVED_SOURCES_KEY.get(this); return removed != null && !removed.isEmpty(); @@ -447,7 +489,6 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler continue; } myProjectDescriptor.fsState.clearRecompile(rd); - myProjectDescriptor.fsState.clearDeletedPaths(module.getName(), isCompilingTests()); traverseRecursively(rd, rd.root, excludes, tsStorage, forceMarkDirty, currentFiles); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 8d2e56bc3683..de29273874d4 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -218,6 +218,22 @@ public class IncProjectBuilder { context.processMessage(new ProgressMessage("Running 'after' tasks")); runTasks(context, myBuilderRegistry.getAfterTasks()); + + // cleanup output roots layout, commented for efficiency + //final ModuleOutputRootsLayout outputRootsLayout = context.getDataManager().getOutputRootsLayout(); + //try { + // final Iterator keysIterator = outputRootsLayout.getKeysIterator(); + // final Map modules = myProjectDescriptor.project.getModules(); + // while (keysIterator.hasNext()) { + // final String moduleName = keysIterator.next(); + // if (modules.containsKey(moduleName)) { + // outputRootsLayout.remove(moduleName); + // } + // } + //} + //catch (IOException e) { + // throw new ProjectBuildException(e); + //} } finally { for (ProjectLevelBuilder builder : myBuilderRegistry.getProjectLevelBuilders()) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index 8b1e97111783..ad5986f2a5ac 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -18,7 +18,7 @@ import java.util.Map; * Date: 10/7/11 */ public class BuildDataManager implements StorageOwner { - private static final int VERSION = 4; + private static final int VERSION = 5; private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.storage.BuildDataManager"); private static final String SRC_TO_OUTPUTS_STORAGE = "src-out"; private static final String SRC_TO_FORM_STORAGE = "src-form"; @@ -30,6 +30,7 @@ public class BuildDataManager implements StorageOwner { private final SourceToFormMapping mySrcToFormMap; private final ArtifactsBuildData myArtifactsBuildData; + private final ModuleOutputRootsLayout myOutputRootsLayout; private final Mappings myMappings; private final File myDataStorageRoot; private final File myVersionFile; @@ -37,11 +38,16 @@ public class BuildDataManager implements StorageOwner { public BuildDataManager(final File dataStorageRoot, final boolean useMemoryTempCaches) throws IOException { myDataStorageRoot = dataStorageRoot; mySrcToFormMap = new SourceToFormMapping(new File(getSourceToFormsRoot(), "data")); + myOutputRootsLayout = new ModuleOutputRootsLayout(new File(getOutputsLayoutRoot(), "data")); myMappings = new Mappings(getMappingsRoot(), useMemoryTempCaches); myArtifactsBuildData = new ArtifactsBuildData(new File(dataStorageRoot, "artifacts")); myVersionFile = new File(myDataStorageRoot, "version.dat"); } + private File getOutputsLayoutRoot() { + return new File(myDataStorageRoot, "output-roots"); + } + public SourceToOutputMapping getSourceToOutputMap(final String moduleName, final boolean testSources) throws IOException { String lowerCaseModuleName = moduleName.toLowerCase(Locale.US); final Map storageMap = testSources ? myTestSourceToOutputs : myProductionSourceToOutputs; @@ -64,6 +70,10 @@ public class BuildDataManager implements StorageOwner { return mySrcToFormMap; } + public ModuleOutputRootsLayout getOutputRootsLayout() { + return myOutputRootsLayout; + } + public Mappings getMappings() { return myMappings; } @@ -88,14 +98,19 @@ public class BuildDataManager implements StorageOwner { wipeStorage(getSourceToFormsRoot(), mySrcToFormMap); } finally { - final Mappings mappings = myMappings; - if (mappings != null) { - synchronized (mappings) { - mappings.clean(); - } + try { + wipeStorage(getOutputsLayoutRoot(), myOutputRootsLayout); } - else { - FileUtil.delete(getMappingsRoot()); + finally { + final Mappings mappings = myMappings; + if (mappings != null) { + synchronized (mappings) { + mappings.clean(); + } + } + else { + FileUtil.delete(getMappingsRoot()); + } } } } @@ -115,6 +130,7 @@ public class BuildDataManager implements StorageOwner { } } mySrcToFormMap.flush(memoryCachesOnly); + myOutputRootsLayout.flush(memoryCachesOnly); final Mappings mappings = myMappings; if (mappings != null) { synchronized (mappings) { @@ -138,17 +154,22 @@ public class BuildDataManager implements StorageOwner { closeStorage(mySrcToFormMap); } finally { - final Mappings mappings = myMappings; - if (mappings != null) { - try { - mappings.close(); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw ((IOException)cause); + try { + closeStorage(myOutputRootsLayout); + } + finally { + final Mappings mappings = myMappings; + if (mappings != null) { + try { + mappings.close(); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw ((IOException)cause); + } + throw e; } - throw e; } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ModuleOutputRootsLayout.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ModuleOutputRootsLayout.java new file mode 100644 index 000000000000..9f6d977ec08c --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ModuleOutputRootsLayout.java @@ -0,0 +1,39 @@ +package org.jetbrains.jps.incremental.storage; + +import com.intellij.openapi.util.Pair; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.IOUtil; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; + +/** + * @author Eugene Zhuravlev + * Date: 12/29/11 + */ +public class ModuleOutputRootsLayout extends AbstractStateStorage>{ + + public ModuleOutputRootsLayout(File storePath) throws IOException { + super(storePath, new EnumeratorStringDescriptor(), new PairDataExternalizer()); + } + + public void appendData(String s, Pair data) throws IOException { + update(s, data); + } + + private static class PairDataExternalizer implements DataExternalizer> { + public void save(DataOutput out, Pair value) throws IOException { + IOUtil.writeString(value.getFirst(), out); + IOUtil.writeString(value.getSecond(), out); + } + + public Pair read(DataInput in) throws IOException { + final String first = IOUtil.readString(in); + final String second = IOUtil.readString(in); + return new Pair(first, second); + } + } +}