From 54c77b6b83016bc35d68e3365a28dddef788828b Mon Sep 17 00:00:00 2001 From: nik Date: Fri, 27 Jan 2012 09:43:54 +0400 Subject: [PATCH] project level builder introduced --- .../jetbrains/jps/incremental/Builder.java | 174 +---------------- .../jps/incremental/BuilderRegistry.java | 6 +- .../jps/incremental/IncProjectBuilder.java | 12 +- .../jps/incremental/ModuleLevelBuilder.java | 177 ++++++++++++++++++ .../jps/incremental/ProjectLevelBuilder.java | 20 ++ .../jps/incremental/groovy/GroovyBuilder.java | 4 +- .../jps/incremental/java/JavaBuilder.java | 2 +- .../resourses/ResourcesBuilder.java | 2 +- 8 files changed, 211 insertions(+), 186 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectLevelBuilder.java diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/Builder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/Builder.java index f7c24ef8c73b..4bfc4b9c294e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/Builder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/Builder.java @@ -1,182 +1,10 @@ package org.jetbrains.jps.incremental; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.Key; -import com.intellij.openapi.util.io.FileUtil; -import org.jetbrains.ether.dependencyView.Mappings; -import org.jetbrains.jps.Module; -import org.jetbrains.jps.ModuleChunk; -import org.jetbrains.jps.incremental.storage.SourceToOutputMapping; -import sun.util.LocaleServiceProviderPool; - -import java.io.File; -import java.io.IOException; -import java.util.*; - /** - * @author Eugene Zhuravlev - * Date: 9/17/11 + * @author nik */ public abstract class Builder { - private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.Builder"); - - private static final Key> ALL_AFFECTED_FILES_KEY = Key.create("_all_affected_files_"); - private static final Key> ALL_COMPILED_FILES_KEY = Key.create("_all_compiled_files_"); - - public static enum ExitCode { - OK, ABORT, ADDITIONAL_PASS_REQUIRED - } - public abstract String getName(); - public abstract ExitCode build(CompileContext context, ModuleChunk chunk) throws ProjectBuildException; - public abstract String getDescription(); - - public void cleanupResources(CompileContext context, ModuleChunk chunk) { - ALL_AFFECTED_FILES_KEY.set(context, null); - ALL_COMPILED_FILES_KEY.set(context, null); - } - - /** - * @param context - * @param delta - * @param chunk - * @param filesToCompile files compiled in this round - * @param successfullyCompiled - * @return true if additional compilation pass is required, false otherwise - * @throws Exception - */ - public final boolean updateMappings(CompileContext context, final Mappings delta, ModuleChunk chunk, Collection filesToCompile, Collection successfullyCompiled) throws Exception { - try { - boolean additionalPassRequired = false; - - final Set removedPaths = getRemovedPaths(context); - - final Mappings globalMappings = context.getDataManager().getMappings(); - - //noinspection SynchronizationOnLocalVariableOrMethodParameter - synchronized (globalMappings) { - if (!context.isProjectRebuild() && context.shouldDifferentiate(chunk, context.isCompilingTests())) { - final Set allCompiledFiles = getAllCompiledFilesContainer(context); - final Set allAffectedFiles = getAllAffectedFilesContainer(context); - - // mark as affected all files that were dirty before compilation - allAffectedFiles.addAll(filesToCompile); - // accumulate all successfully compiled in this round - allCompiledFiles.addAll(successfullyCompiled); - // unmark as affected all successfully compiled - allAffectedFiles.removeAll(successfullyCompiled); - - final HashSet affectedBeforeDif = new HashSet(allAffectedFiles); - - final boolean incremental = globalMappings.differentiate( - delta, removedPaths, successfullyCompiled, allCompiledFiles, allAffectedFiles - ); - - if (LOG.isDebugEnabled()) { - LOG.debug("Differentiate Results:"); - - LOG.debug(" Compiled Files:"); - - for (final File c : allCompiledFiles) { - LOG.debug(" " + c.getAbsolutePath()); - } - - LOG.debug(" Affected Files:"); - - for (final File c : allAffectedFiles) { - LOG.debug(" " + c.getAbsolutePath()); - } - - LOG.debug("End Of Differentiate Results."); - } - - if (incremental) { - final Set newlyAffectedFiles = new HashSet(allAffectedFiles); - newlyAffectedFiles.removeAll(affectedBeforeDif); - newlyAffectedFiles.removeAll(allCompiledFiles); // the diff operation may have affected the class already compiled in thic compilation round - - if (!newlyAffectedFiles.isEmpty()) { - for (File file : newlyAffectedFiles) { - context.markDirty(file); - } - additionalPassRequired = context.isMake() && chunkContainsAffectedFiles(context, chunk, newlyAffectedFiles); - } - } - else { - additionalPassRequired = context.isMake(); - context.markDirtyRecursively(chunk); - } - } - - globalMappings.integrate(delta, successfullyCompiled, removedPaths); - } - - return additionalPassRequired; - } - catch(RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw ((IOException)cause); - } - throw e; - } - } - - // delete all class files that according to mappings correspond to given sources - public static void deleteCorrespondingOutputFiles(CompileContext context, Map sources) throws Exception { - if (!context.isProjectRebuild() && !sources.isEmpty()) { - for (Map.Entry pair : sources.entrySet()) { - final File file = pair.getKey(); - final String srcPath = FileUtil.toSystemIndependentName(file.getPath()); - final String moduleName = pair.getValue().getName().toLowerCase(Locale.US); - final SourceToOutputMapping srcToOut = context.getDataManager().getSourceToOutputMap(moduleName, context.isCompilingTests()); - final Collection outputs = srcToOut.getState(srcPath); - if (outputs != null) { - for (String output : outputs) { - FileUtil.delete(new File(output)); - } - srcToOut.remove(srcPath); - } - } - } - } - - private static boolean chunkContainsAffectedFiles(CompileContext context, ModuleChunk chunk, final Set affected) throws Exception { - final Set chunkModules = new HashSet(chunk.getModules()); - if (!chunkModules.isEmpty()) { - for (File file : affected) { - final RootDescriptor moduleAndRoot = context.getModuleAndRoot(file); - if (moduleAndRoot != null && chunkModules.contains(moduleAndRoot.module)) { - return true; - } - } - } - return false; - } - - private static Set getAllAffectedFilesContainer(CompileContext context) { - Set allAffectedFiles = ALL_AFFECTED_FILES_KEY.get(context); - if (allAffectedFiles == null) { - allAffectedFiles = new HashSet(); - ALL_AFFECTED_FILES_KEY.set(context, allAffectedFiles); - } - return allAffectedFiles; - } - - private static Set getAllCompiledFilesContainer(CompileContext context) { - Set allCompiledFiles = ALL_COMPILED_FILES_KEY.get(context); - if (allCompiledFiles == null) { - allCompiledFiles = new HashSet(); - ALL_COMPILED_FILES_KEY.set(context, allCompiledFiles); - } - return allCompiledFiles; - } - - private static Set getRemovedPaths(CompileContext context) { - final Set removed = Paths.CHUNK_REMOVED_SOURCES_KEY.get(context); - return removed != null? removed : Collections.emptySet(); - } - } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuilderRegistry.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuilderRegistry.java index 7840245b8409..ee20e5a5d8e3 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuilderRegistry.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuilderRegistry.java @@ -16,7 +16,7 @@ public class BuilderRegistry { private static class Holder { static final BuilderRegistry ourInstance = new BuilderRegistry(); } - private final Map> myBuilders = new HashMap>(); + private final Map> myBuilders = new HashMap>(); private ExecutorService myTasksExecutor; public static BuilderRegistry getInstance() { @@ -25,7 +25,7 @@ public class BuilderRegistry { private BuilderRegistry() { for (BuilderCategory category : BuilderCategory.values()) { - myBuilders.put(category, new ArrayList()); + myBuilders.put(category, new ArrayList()); } final Runtime runtime = Runtime.getRuntime(); myTasksExecutor = Executors.newFixedThreadPool(runtime.availableProcessors()); @@ -60,7 +60,7 @@ public class BuilderRegistry { return Collections.emptyList(); // todo } - public List getBuilders(BuilderCategory category){ + public List getBuilders(BuilderCategory category){ return Collections.unmodifiableList(myBuilders.get(category)); // todo } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 65efacbb9bcb..ccfa9e42dc4b 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -314,7 +314,7 @@ public class IncProjectBuilder { finally { try { for (BuilderCategory category : BuilderCategory.values()) { - for (Builder builder : myBuilderRegistry.getBuilders(category)) { + for (ModuleLevelBuilder builder : myBuilderRegistry.getBuilders(category)) { builder.cleanupResources(context, chunk); } } @@ -334,7 +334,7 @@ public class IncProjectBuilder { } private void runBuilders(CompileContext context, ModuleChunk chunk, BuilderCategory category) throws ProjectBuildException { - final List builders = myBuilderRegistry.getBuilders(category); + final List builders = myBuilderRegistry.getBuilders(category); if (builders.isEmpty()) { return; } @@ -347,16 +347,16 @@ public class IncProjectBuilder { do { nextPassRequired = false; context.beforeNextCompileRound(chunk); - for (Builder builder : builders) { - final Builder.ExitCode buildResult = builder.build(context, chunk); + for (ModuleLevelBuilder builder : builders) { + final ModuleLevelBuilder.ExitCode buildResult = builder.build(context, chunk); - if (buildResult == Builder.ExitCode.ABORT) { + if (buildResult == ModuleLevelBuilder.ExitCode.ABORT) { throw new ProjectBuildException("Builder " + builder.getDescription() + " requested build stop"); } if (myCancelStatus.isCanceled()) { throw new ProjectBuildException(CANCELED_MESSAGE); } - if (buildResult == Builder.ExitCode.ADDITIONAL_PASS_REQUIRED) { + if (buildResult == ModuleLevelBuilder.ExitCode.ADDITIONAL_PASS_REQUIRED) { if (!nextPassRequired) { // recalculate basis myModulesProcessed -= (stagesPassed * modulesInChunk) / stageCount; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java new file mode 100644 index 000000000000..314187fa1100 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java @@ -0,0 +1,177 @@ +package org.jetbrains.jps.incremental; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.io.FileUtil; +import org.jetbrains.ether.dependencyView.Mappings; +import org.jetbrains.jps.Module; +import org.jetbrains.jps.ModuleChunk; +import org.jetbrains.jps.incremental.storage.SourceToOutputMapping; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +/** + * @author Eugene Zhuravlev + * Date: 9/17/11 + */ +public abstract class ModuleLevelBuilder extends Builder { + private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.Builder"); + + private static final Key> ALL_AFFECTED_FILES_KEY = Key.create("_all_affected_files_"); + private static final Key> ALL_COMPILED_FILES_KEY = Key.create("_all_compiled_files_"); + + public static enum ExitCode { + OK, ABORT, ADDITIONAL_PASS_REQUIRED + } + + public abstract ExitCode build(CompileContext context, ModuleChunk chunk) throws ProjectBuildException; + + public void cleanupResources(CompileContext context, ModuleChunk chunk) { + ALL_AFFECTED_FILES_KEY.set(context, null); + ALL_COMPILED_FILES_KEY.set(context, null); + } + + /** + * @param context + * @param delta + * @param chunk + * @param filesToCompile files compiled in this round + * @param successfullyCompiled + * @return true if additional compilation pass is required, false otherwise + * @throws Exception + */ + public final boolean updateMappings(CompileContext context, final Mappings delta, ModuleChunk chunk, Collection filesToCompile, Collection successfullyCompiled) throws Exception { + try { + boolean additionalPassRequired = false; + + final Set removedPaths = getRemovedPaths(context); + + final Mappings globalMappings = context.getDataManager().getMappings(); + + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (globalMappings) { + if (!context.isProjectRebuild() && context.shouldDifferentiate(chunk, context.isCompilingTests())) { + final Set allCompiledFiles = getAllCompiledFilesContainer(context); + final Set allAffectedFiles = getAllAffectedFilesContainer(context); + + // mark as affected all files that were dirty before compilation + allAffectedFiles.addAll(filesToCompile); + // accumulate all successfully compiled in this round + allCompiledFiles.addAll(successfullyCompiled); + // unmark as affected all successfully compiled + allAffectedFiles.removeAll(successfullyCompiled); + + final HashSet affectedBeforeDif = new HashSet(allAffectedFiles); + + final boolean incremental = globalMappings.differentiate( + delta, removedPaths, successfullyCompiled, allCompiledFiles, allAffectedFiles + ); + + if (LOG.isDebugEnabled()) { + LOG.debug("Differentiate Results:"); + + LOG.debug(" Compiled Files:"); + + for (final File c : allCompiledFiles) { + LOG.debug(" " + c.getAbsolutePath()); + } + + LOG.debug(" Affected Files:"); + + for (final File c : allAffectedFiles) { + LOG.debug(" " + c.getAbsolutePath()); + } + + LOG.debug("End Of Differentiate Results."); + } + + if (incremental) { + final Set newlyAffectedFiles = new HashSet(allAffectedFiles); + newlyAffectedFiles.removeAll(affectedBeforeDif); + newlyAffectedFiles.removeAll(allCompiledFiles); // the diff operation may have affected the class already compiled in thic compilation round + + if (!newlyAffectedFiles.isEmpty()) { + for (File file : newlyAffectedFiles) { + context.markDirty(file); + } + additionalPassRequired = context.isMake() && chunkContainsAffectedFiles(context, chunk, newlyAffectedFiles); + } + } + else { + additionalPassRequired = context.isMake(); + context.markDirtyRecursively(chunk); + } + } + + globalMappings.integrate(delta, successfullyCompiled, removedPaths); + } + + return additionalPassRequired; + } + catch(RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw ((IOException)cause); + } + throw e; + } + } + + // delete all class files that according to mappings correspond to given sources + public static void deleteCorrespondingOutputFiles(CompileContext context, Map sources) throws Exception { + if (!context.isProjectRebuild() && !sources.isEmpty()) { + for (Map.Entry pair : sources.entrySet()) { + final File file = pair.getKey(); + final String srcPath = FileUtil.toSystemIndependentName(file.getPath()); + final String moduleName = pair.getValue().getName().toLowerCase(Locale.US); + final SourceToOutputMapping srcToOut = context.getDataManager().getSourceToOutputMap(moduleName, context.isCompilingTests()); + final Collection outputs = srcToOut.getState(srcPath); + if (outputs != null) { + for (String output : outputs) { + FileUtil.delete(new File(output)); + } + srcToOut.remove(srcPath); + } + } + } + } + + private static boolean chunkContainsAffectedFiles(CompileContext context, ModuleChunk chunk, final Set affected) throws Exception { + final Set chunkModules = new HashSet(chunk.getModules()); + if (!chunkModules.isEmpty()) { + for (File file : affected) { + final RootDescriptor moduleAndRoot = context.getModuleAndRoot(file); + if (moduleAndRoot != null && chunkModules.contains(moduleAndRoot.module)) { + return true; + } + } + } + return false; + } + + private static Set getAllAffectedFilesContainer(CompileContext context) { + Set allAffectedFiles = ALL_AFFECTED_FILES_KEY.get(context); + if (allAffectedFiles == null) { + allAffectedFiles = new HashSet(); + ALL_AFFECTED_FILES_KEY.set(context, allAffectedFiles); + } + return allAffectedFiles; + } + + private static Set getAllCompiledFilesContainer(CompileContext context) { + Set allCompiledFiles = ALL_COMPILED_FILES_KEY.get(context); + if (allCompiledFiles == null) { + allCompiledFiles = new HashSet(); + ALL_COMPILED_FILES_KEY.set(context, allCompiledFiles); + } + return allCompiledFiles; + } + + private static Set getRemovedPaths(CompileContext context) { + final Set removed = Paths.CHUNK_REMOVED_SOURCES_KEY.get(context); + return removed != null? removed : Collections.emptySet(); + } + +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectLevelBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectLevelBuilder.java new file mode 100644 index 000000000000..c8296590c3aa --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectLevelBuilder.java @@ -0,0 +1,20 @@ +package org.jetbrains.jps.incremental; + +/** + * @author nik + */ +public abstract class ProjectLevelBuilder extends Builder { + private final ProjectLevelBuilderCategory myCategory; + + protected ProjectLevelBuilder(ProjectLevelBuilderCategory category) { + myCategory = category; + } + + public abstract void build(CompileContext context); + + public ProjectLevelBuilderCategory getCategory() { + return myCategory; + } + + public static enum ProjectLevelBuilderCategory { TRANSLATOR, PACKAGER } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java index 30635be989f8..7f8be2cb10a2 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java @@ -29,7 +29,7 @@ import java.util.*; * @author Eugene Zhuravlev * Date: 10/25/11 */ -public class GroovyBuilder extends Builder { +public class GroovyBuilder extends ModuleLevelBuilder { public static final String BUILDER_NAME = "groovy"; private final boolean myForStubs; private final String myBuilderName; @@ -43,7 +43,7 @@ public class GroovyBuilder extends Builder { return myBuilderName; } - public Builder.ExitCode build(final CompileContext context, ModuleChunk chunk) throws ProjectBuildException { + public ModuleLevelBuilder.ExitCode build(final CompileContext context, ModuleChunk chunk) throws ProjectBuildException { ExitCode exitCode = ExitCode.OK; final Map toCompile = new HashMap(); try { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java index 1be97a81fcf4..2269278c3908 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java @@ -49,7 +49,7 @@ import java.util.concurrent.ExecutorService; * @author Eugene Zhuravlev * Date: 9/21/11 */ -public class JavaBuilder extends Builder{ +public class JavaBuilder extends ModuleLevelBuilder { public static final String BUILDER_NAME = "java"; private static final String JAVA_EXTENSION = ".java"; private static final String FORM_EXTENSION = ".form"; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/resourses/ResourcesBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/resourses/ResourcesBuilder.java index a8b7725d3fd4..7aaeb0068972 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/resourses/ResourcesBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/resourses/ResourcesBuilder.java @@ -19,7 +19,7 @@ import java.util.Locale; * @author Eugene Zhuravlev * Date: 10/6/11 */ -public class ResourcesBuilder extends Builder{ +public class ResourcesBuilder extends ModuleLevelBuilder { private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.resourses.ResourcesBuilder"); public static final String BUILDER_NAME = "resources";