From 3eda313c9dcc923d0d82dc4fdfd29668251f4375 Mon Sep 17 00:00:00 2001 From: Eugene Zhuravlev Date: Tue, 10 Jul 2012 21:10:59 +0200 Subject: [PATCH] parallel compilation: flush storages on a per-chunk group basis --- .../jps/incremental/IncProjectBuilder.java | 173 ++++++++++-------- .../incremental/storage/BuildDataManager.java | 15 +- 2 files changed, 102 insertions(+), 86 deletions(-) diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 7f85cd35e0b9..996c8bc58f32 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -410,69 +410,88 @@ public class IncProjectBuilder { private void buildChunks(final CompileContextImpl context, ProjectChunks chunks) throws ProjectBuildException { final CompileScope scope = context.getScope(); - if (PARALLEL_BUILD_ENABLED) { - final List chunkGroups = buildChunkGroups(context, chunks); - for (ChunkGroup group : chunkGroups) { - final List groupChunks = group.getChunks(); - final int chunkCount = groupChunks.size(); - if (chunkCount == 0) { - continue; - } - if (chunkCount == 1) { - _buildChunk(createContextWrapper(context), scope, groupChunks.iterator().next()); - } - else { - final CountDownLatch latch = new CountDownLatch(chunkCount); - final Ref exRef = new Ref(null); - final StringBuilder logBuilder = new StringBuilder("Building chunks in parallel: "); - for (final ModuleChunk chunk : groupChunks) { - logBuilder.append(chunk.getName()).append("; "); - final CompileContext chunkLocalContext = createContextWrapper(context); - SharedThreadPool.INSTANCE.submitBuildTask(new Runnable() { - @Override - public void run() { - try { - _buildChunk(chunkLocalContext, scope, chunk); - } - catch (Throwable e) { - synchronized (exRef) { - if (exRef.isNull()) { - exRef.set(e); - } - } - LOG.info(e); - } - finally { - latch.countDown(); - } - } - }); + final ProjectDescriptor pd = context.getProjectDescriptor(); + try { + if (PARALLEL_BUILD_ENABLED) { + final List chunkGroups = buildChunkGroups(context, chunks); + for (ChunkGroup group : chunkGroups) { + final List groupChunks = group.getChunks(); + final int chunkCount = groupChunks.size(); + if (chunkCount == 0) { + continue; } - LOG.info(logBuilder.toString()); - try { - latch.await(); - } - catch (InterruptedException e) { - LOG.info(e); - } - - final Throwable exception = exRef.get(); - if (exception != null) { - if (exception instanceof ProjectBuildException) { - throw (ProjectBuildException)exception; + if (chunkCount == 1) { + _buildChunk(createContextWrapper(context), scope, groupChunks.iterator().next()); } else { - throw new ProjectBuildException(exception); + final CountDownLatch latch = new CountDownLatch(chunkCount); + final Ref exRef = new Ref(null); + final StringBuilder logBuilder = new StringBuilder("Building chunks in parallel: "); + for (final ModuleChunk chunk : groupChunks) { + logBuilder.append(chunk.getName()).append("; "); + final CompileContext chunkLocalContext = createContextWrapper(context); + SharedThreadPool.INSTANCE.submitBuildTask(new Runnable() { + @Override + public void run() { + try { + _buildChunk(chunkLocalContext, scope, chunk); + } + catch (Throwable e) { + synchronized (exRef) { + if (exRef.isNull()) { + exRef.set(e); + } + } + LOG.info(e); + } + finally { + latch.countDown(); + } + } + }); + } + LOG.info(logBuilder.toString()); + + try { + latch.await(); + } + catch (InterruptedException e) { + LOG.info(e); + } + + final Throwable exception = exRef.get(); + if (exception != null) { + if (exception instanceof ProjectBuildException) { + throw (ProjectBuildException)exception; + } + else { + throw new ProjectBuildException(exception); + } + } } } + finally { + pd.dataManager.closeSourceToOutputStorages(groupChunks, context.isCompilingTests()); + pd.dataManager.flush(true); + } + } + } + else { + // non-parallel build + for (ModuleChunk chunk : chunks.getChunkList()) { + try { + _buildChunk(context, scope, chunk); + } + finally { + pd.dataManager.closeSourceToOutputStorages(Collections.singleton(chunk), context.isCompilingTests()); + pd.dataManager.flush(true); + } } } } - else { - for (ModuleChunk chunk : chunks.getChunkList()) { - _buildChunk(context, scope, chunk); - } + catch (IOException e) { + throw new ProjectBuildException(e); } } @@ -866,40 +885,34 @@ public class IncProjectBuilder { return false; } - public static void onChunkBuildComplete(CompileContext context, @NotNull ModuleChunk chunk) throws IOException { + private static void onChunkBuildComplete(CompileContext context, @NotNull ModuleChunk chunk) throws IOException { final ProjectDescriptor pd = context.getProjectDescriptor(); final BuildFSState fsState = pd.fsState; fsState.clearContextRoundData(context); fsState.clearContextChunk(context); - try { - if (!Utils.ERRORS_DETECTED_KEY.get(context, Boolean.FALSE) && !context.getCancelStatus().isCanceled()) { - boolean marked = false; - for (Module module : chunk.getModules()) { - if (context.isMake()) { - // ensure non-incremental flag cleared - context.clearNonIncrementalMark(module); - } - if (context.isProjectRebuild()) { - fsState.markInitialScanPerformed(module.getName(), context.isCompilingTests()); - } - final Timestamps timestamps = pd.timestamps.getStorage(); - final List roots = pd.rootsIndex.getModuleRoots(context, module); - for (RootDescriptor rd : roots) { - if (context.isCompilingTests() ? rd.isTestRoot : !rd.isTestRoot) { - marked |= fsState.markAllUpToDate(context.getScope(), rd, timestamps, context.getCompilationStartStamp()); - } - } + if (!Utils.ERRORS_DETECTED_KEY.get(context, Boolean.FALSE) && !context.getCancelStatus().isCanceled()) { + boolean marked = false; + for (Module module : chunk.getModules()) { + if (context.isMake()) { + // ensure non-incremental flag cleared + context.clearNonIncrementalMark(module); } - - if (marked) { - context.processMessage(UptoDateFilesSavedEvent.INSTANCE); + if (context.isProjectRebuild()) { + fsState.markInitialScanPerformed(module.getName(), context.isCompilingTests()); + } + final Timestamps timestamps = pd.timestamps.getStorage(); + final List roots = pd.rootsIndex.getModuleRoots(context, module); + for (RootDescriptor rd : roots) { + if (context.isCompilingTests() ? rd.isTestRoot : !rd.isTestRoot) { + marked |= fsState.markAllUpToDate(context.getScope(), rd, timestamps, context.getCompilationStartStamp()); + } } } - } - finally { - pd.dataManager.closeSourceToOutputStorages(chunk, context.isCompilingTests()); - pd.dataManager.flush(true); + + if (marked) { + context.processMessage(UptoDateFilesSavedEvent.INSTANCE); + } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index 3e2fd8867159..f84199b61b61 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -9,6 +9,7 @@ import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.incremental.artifacts.ArtifactsBuildData; import java.io.*; +import java.util.Collection; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -178,14 +179,16 @@ public class BuildDataManager implements StorageOwner { } } - public void closeSourceToOutputStorages(ModuleChunk chunk, boolean testSources) throws IOException { + public void closeSourceToOutputStorages(Collection chunks, boolean testSources) throws IOException { final Map storageMap = testSources? myTestSourceToOutputs : myProductionSourceToOutputs; synchronized (mySourceToOutputLock) { - for (Module module : chunk.getModules()) { - final String moduleName = module.getName().toLowerCase(Locale.US); - final SourceToOutputMapping mapping = storageMap.remove(moduleName); - if (mapping != null) { - mapping.close(); + for (ModuleChunk chunk : chunks) { + for (Module module : chunk.getModules()) { + final String moduleName = module.getName().toLowerCase(Locale.US); + final SourceToOutputMapping mapping = storageMap.remove(moduleName); + if (mapping != null) { + mapping.close(); + } } } }