From f9038ef5c76a54018385c4bcf04ced036364ea22 Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 9 Oct 2012 10:03:26 +0400 Subject: [PATCH] external compiler: generalized clearing outputs --- .../jps/builders/java/JavaBuilderUtil.java | 5 +- .../jps/incremental/BuildLoggingManager.java | 12 +++- .../jps/incremental/IncProjectBuilder.java | 69 ++++++------------- .../jps/incremental/ProjectBuilderLogger.java | 11 +++ .../incremental/ProjectBuilderLoggerImpl.java | 38 ++++++++++ .../org/jetbrains/jps/incremental/Utils.java | 5 +- .../jetbrains/ether/IncrementalTestCase.java | 38 +++++++--- .../jps/builders/JpsBuildTestCase.java | 10 ++- .../artifacts/ArtifactBuilderTestCase.java | 3 +- 9 files changed, 122 insertions(+), 69 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLogger.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLoggerImpl.java diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/JavaBuilderUtil.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/JavaBuilderUtil.java index 4f5f2ec25170..e0872828eddf 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/java/JavaBuilderUtil.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/JavaBuilderUtil.java @@ -7,6 +7,7 @@ import com.intellij.openapi.util.io.FileUtil; import gnu.trove.THashSet; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.ProjectPaths; +import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.java.dependencyView.Callbacks; import org.jetbrains.jps.builders.java.dependencyView.Mappings; import org.jetbrains.jps.builders.storage.SourceToOutputMapping; @@ -203,7 +204,7 @@ public class JavaBuilderUtil { } private static Set getRemovedPaths(CompileContext context, ModuleChunk chunk) { - final Map> map = Utils.REMOVED_SOURCES_KEY.get(context); + final Map, Collection> map = Utils.REMOVED_SOURCES_KEY.get(context); if (map == null) { return Collections.emptySet(); } @@ -218,7 +219,7 @@ public class JavaBuilderUtil { } private static void dropRemovedPaths(CompileContext context, ModuleChunk chunk) throws IOException { - final Map> map = Utils.REMOVED_SOURCES_KEY.get(context); + final Map, Collection> map = Utils.REMOVED_SOURCES_KEY.get(context); if (map != null) { for (ModuleBuildTarget target : chunk.getTargets()) { final Collection paths = map.remove(target); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildLoggingManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildLoggingManager.java index 2f4c6177a1e9..b57023012cae 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildLoggingManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildLoggingManager.java @@ -10,13 +10,16 @@ import org.jetbrains.jps.incremental.java.JavaBuilderLoggerImpl; * @author nik */ public class BuildLoggingManager { - public static final BuildLoggingManager DEFAULT = new BuildLoggingManager(new ArtifactBuilderLoggerImpl(), new JavaBuilderLoggerImpl()); + public static final BuildLoggingManager DEFAULT = new BuildLoggingManager(new ArtifactBuilderLoggerImpl(), new JavaBuilderLoggerImpl(), + new ProjectBuilderLoggerImpl()); private ArtifactBuilderLogger myArtifactBuilderLogger; private JavaBuilderLogger myJavaBuilderLogger; + private final ProjectBuilderLogger myProjectLogger; - public BuildLoggingManager(@NotNull ArtifactBuilderLogger artifactBuilderLogger, @NotNull JavaBuilderLogger logger) { + public BuildLoggingManager(@NotNull ArtifactBuilderLogger artifactBuilderLogger, @NotNull JavaBuilderLogger logger, @NotNull ProjectBuilderLogger projectLogger) { myArtifactBuilderLogger = artifactBuilderLogger; myJavaBuilderLogger = logger; + myProjectLogger = projectLogger; } @NotNull @@ -28,4 +31,9 @@ public class BuildLoggingManager { public JavaBuilderLogger getJavaBuilderLogger() { return myJavaBuilderLogger; } + + @NotNull + public ProjectBuilderLogger getProjectBuilderLogger() { + return myProjectLogger; + } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 28d0881a3dbd..189e121d34ac 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -14,7 +14,6 @@ import com.intellij.util.io.PersistentEnumerator; import gnu.trove.THashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jetbrains.jps.util.JpsPathUtil; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.ProjectPaths; import org.jetbrains.jps.api.CanceledStatus; @@ -32,12 +31,12 @@ import org.jetbrains.jps.cmdline.ProjectDescriptor; import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.java.ExternalJavacDescriptor; import org.jetbrains.jps.incremental.java.JavaBuilder; -import org.jetbrains.jps.incremental.java.JavaBuilderLogger; import org.jetbrains.jps.incremental.messages.*; import org.jetbrains.jps.incremental.storage.*; import org.jetbrains.jps.model.java.JpsJavaExtensionService; import org.jetbrains.jps.model.java.compiler.ProcessorConfigProfile; import org.jetbrains.jps.service.SharedThreadPool; +import org.jetbrains.jps.util.JpsPathUtil; import java.io.BufferedWriter; import java.io.File; @@ -616,10 +615,10 @@ public class IncProjectBuilder { try { // restore deleted paths that were not procesesd by 'integrate' - final Map> map = Utils.REMOVED_SOURCES_KEY.get(context); + final Map, Collection> map = Utils.REMOVED_SOURCES_KEY.get(context); if (map != null) { - for (Map.Entry> entry : map.entrySet()) { - final ModuleBuildTarget target = entry.getKey(); + for (Map.Entry, Collection> entry : map.entrySet()) { + final BuildTarget target = entry.getKey(); final Collection paths = entry.getValue(); if (paths != null) { for (String path : paths) { @@ -689,7 +688,7 @@ public class IncProjectBuilder { private void processDeletedPaths(CompileContext context, ModuleChunk chunk) throws ProjectBuildException { try { // cleanup outputs - final Map> removedSources = new HashMap>(); + final Map, Collection> removedSources = new HashMap, Collection>(); for (ModuleBuildTarget target : chunk.getTargets()) { final Collection deletedPaths = myProjectDescriptor.fsState.getAndClearDeletedPaths(target); @@ -705,19 +704,9 @@ public class IncProjectBuilder { final Collection outputs = sourceToOutputStorage.getOutputs(deletedSource); if (outputs != null && !outputs.isEmpty()) { - final JavaBuilderLogger logger = context.getLoggingManager().getJavaBuilderLogger(); + final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); if (logger.isEnabled()) { - final String[] buffer = new String[outputs.size()]; - int i = 0; - for (final String o : outputs) { - buffer[i++] = o; - } - Arrays.sort(buffer); - logger.log("Cleaning output files:"); - for (final String o : buffer) { - logger.log(o); - } - logger.log("End of files"); + logger.logDeletedFiles(outputs); } for (String output : outputs) { @@ -739,9 +728,9 @@ public class IncProjectBuilder { } } if (!removedSources.isEmpty()) { - final Map> existing = Utils.REMOVED_SOURCES_KEY.get(context); + final Map, Collection> existing = Utils.REMOVED_SOURCES_KEY.get(context); if (existing != null) { - for (Map.Entry> entry : existing.entrySet()) { + for (Map.Entry, Collection> entry : existing.entrySet()) { final Collection paths = removedSources.get(entry.getKey()); if (paths != null) { paths.addAll(entry.getValue()); @@ -778,7 +767,7 @@ public class IncProjectBuilder { } }; if (!context.isProjectRebuild()) { - syncOutputFiles(context, dirtyFilesHolder); + deleteOutputsOfDirtyFiles(context, dirtyFilesHolder); } BUILDER_CATEGORY_LOOP: @@ -829,8 +818,7 @@ public class IncProjectBuilder { } } else { - context.getLoggingManager().getJavaBuilderLogger().log( - "Builder " + builder.getDescription() + " requested second chunk rebuild"); + LOG.debug("Builder " + builder.getDescription() + " requested second chunk rebuild"); } } @@ -844,17 +832,18 @@ public class IncProjectBuilder { return doneSomething; } - private static void syncOutputFiles(final CompileContext context, - DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { + private static > + void deleteOutputsOfDirtyFiles(final CompileContext context, DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { final BuildDataManager dataManager = context.getProjectDescriptor().dataManager; try { - final Collection allOutputs = new LinkedList(); + ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); + final Collection outputsToLog = logger.isEnabled() ? new LinkedList() : null; - dirtyFilesHolder.processDirtyFiles(new FileProcessor() { - private final Map storageMap = new HashMap(); + dirtyFilesHolder.processDirtyFiles(new FileProcessor() { + private final Map storageMap = new HashMap(); @Override - public boolean apply(ModuleBuildTarget target, File file, JavaSourceRootDescriptor sourceRoot) throws IOException { + public boolean apply(T target, File file, R sourceRoot) throws IOException { SourceToOutputMapping srcToOut = storageMap.get(target); if (srcToOut == null) { srcToOut = dataManager.getSourceToOutputMap(target); @@ -864,10 +853,9 @@ public class IncProjectBuilder { final Collection outputs = srcToOut.getOutputs(srcPath); if (outputs != null) { - final JavaBuilderLogger logger = context.getLoggingManager().getJavaBuilderLogger(); for (String output : outputs) { - if (logger.isEnabled()) { - allOutputs.add(output); + if (outputsToLog != null) { + outputsToLog.add(output); } new File(output).delete(); } @@ -880,21 +868,8 @@ public class IncProjectBuilder { } }); - final JavaBuilderLogger logger = context.getLoggingManager().getJavaBuilderLogger(); - if (logger.isEnabled()) { - if (context.isMake() && allOutputs.size() > 0) { - logger.log("Cleaning output files:"); - final String[] buffer = new String[allOutputs.size()]; - int i = 0; - for (String output : allOutputs) { - buffer[i++] = output; - } - Arrays.sort(buffer); - for (String output : buffer) { - logger.log(output); - } - logger.log("End of files"); - } + if (outputsToLog != null && context.isMake()) { + logger.logDeletedFiles(outputsToLog); } } catch (Exception e) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLogger.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLogger.java new file mode 100644 index 000000000000..d9883375c79b --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLogger.java @@ -0,0 +1,11 @@ +package org.jetbrains.jps.incremental; + +import java.util.Collection; + +/** + * @author nik + */ +public interface ProjectBuilderLogger { + boolean isEnabled(); + void logDeletedFiles(Collection paths); +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLoggerImpl.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLoggerImpl.java new file mode 100644 index 000000000000..bfb94564f9aa --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ProjectBuilderLoggerImpl.java @@ -0,0 +1,38 @@ +package org.jetbrains.jps.incremental; + +import com.intellij.openapi.diagnostic.Logger; + +import java.util.Arrays; +import java.util.Collection; + +/** + * @author nik + */ +public class ProjectBuilderLoggerImpl implements ProjectBuilderLogger { + private static final Logger LOG = Logger.getInstance(ProjectBuilderLoggerImpl.class); + + @Override + public boolean isEnabled() { + return LOG.isDebugEnabled(); + } + + @Override + public void logDeletedFiles(Collection outputs) { + if (outputs.isEmpty()) return; + final String[] buffer = new String[outputs.size()]; + int i = 0; + for (final String o : outputs) { + buffer[i++] = o; + } + Arrays.sort(buffer); + logLine("Cleaning output files:"); + for (final String o : buffer) { + logLine(o); + } + logLine("End of files"); + } + + protected void logLine(final String message) { + LOG.debug(message); + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/Utils.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/Utils.java index f0bb7ed12482..8706efd46000 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/Utils.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/Utils.java @@ -5,6 +5,7 @@ import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import org.jetbrains.annotations.Nullable; +import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.model.module.JpsModule; import java.io.File; @@ -18,7 +19,7 @@ import java.util.*; * Date: 10/20/11 */ public class Utils { - public static final Key>> REMOVED_SOURCES_KEY = Key.create("_removed_sources_"); + public static final Key, Collection>> REMOVED_SOURCES_KEY = Key.create("_removed_sources_"); public static final Key PROCEED_ON_ERROR_KEY = Key.create("_proceed_on_error_"); public static final Key ERRORS_DETECTED_KEY = Key.create("_errors_detected_"); private static volatile File ourSystemRoot = new File(System.getProperty("user.home"), ".idea-build"); @@ -127,7 +128,7 @@ public class Utils { } public static boolean hasRemovedSources(CompileContext context) { - final Map> removed = REMOVED_SOURCES_KEY.get(context); + final Map, Collection> removed = REMOVED_SOURCES_KEY.get(context); return removed != null && !removed.isEmpty(); } diff --git a/jps/jps-builders/testSrc/org/jetbrains/ether/IncrementalTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/ether/IncrementalTestCase.java index 1fb78cd56d3d..e7258d062508 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/ether/IncrementalTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/ether/IncrementalTestCase.java @@ -19,16 +19,13 @@ import com.intellij.openapi.application.ex.PathManagerEx; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.Processor; +import org.jetbrains.jps.incremental.*; import org.jetbrains.jps.util.JpsPathUtil; import org.jetbrains.jps.builders.BuildResult; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.JpsBuildTestCase; import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType; import org.jetbrains.jps.cmdline.ProjectDescriptor; -import org.jetbrains.jps.incremental.BuildLoggingManager; -import org.jetbrains.jps.incremental.CompileScope; -import org.jetbrains.jps.incremental.CompileScopeImpl; -import org.jetbrains.jps.incremental.Utils; import org.jetbrains.jps.incremental.artifacts.ArtifactBuilderLoggerImpl; import org.jetbrains.jps.incremental.java.JavaBuilderLogger; import org.jetbrains.jps.model.JpsDummyElement; @@ -173,8 +170,11 @@ public abstract class IncrementalTestCase extends JpsBuildTestCase { } protected BuildResult doTestBuild(int makesCount) { - final TestJavaBuilderLogger builderLogger = new TestJavaBuilderLogger(FileUtil.toSystemIndependentName(workDir.getAbsolutePath()) + "/"); - final ProjectDescriptor pd = createProjectDescriptor(new BuildLoggingManager(new ArtifactBuilderLoggerImpl(), builderLogger)); + StringBuilder log = new StringBuilder(); + String rootPath = FileUtil.toSystemIndependentName(workDir.getAbsolutePath()) + "/"; + final TestJavaBuilderLogger builderLogger = new TestJavaBuilderLogger(rootPath, log); + final ProjectDescriptor pd = createProjectDescriptor(new BuildLoggingManager(new ArtifactBuilderLoggerImpl(), builderLogger, + new TestProjectBuilderLogger(rootPath, log))); try { doBuild(pd, createAllModulesScope(true), false, true, false).assertSuccessful(); @@ -206,7 +206,7 @@ public abstract class IncrementalTestCase extends JpsBuildTestCase { logFile = new File(baseDir, "build.log"); } final String expected = StringUtil.convertLineSeparators(FileUtil.loadFile(logFile)); - final String actual = builderLogger.myLog.toString(); + final String actual = log.toString(); assertEquals(expected, actual); @@ -262,9 +262,9 @@ public abstract class IncrementalTestCase extends JpsBuildTestCase { private final String myRoot; private final StringBuilder myLog; - public TestJavaBuilderLogger(String root) { + public TestJavaBuilderLogger(String root, final StringBuilder log) { myRoot = root; - myLog = new StringBuilder(); + myLog = log; } @Override @@ -277,4 +277,24 @@ public abstract class IncrementalTestCase extends JpsBuildTestCase { return true; } } + + private static class TestProjectBuilderLogger extends ProjectBuilderLoggerImpl { + private final String myRoot; + private StringBuilder myLog; + + private TestProjectBuilderLogger(String root, StringBuilder log) { + myRoot = root; + myLog = log; + } + + @Override + public boolean isEnabled() { + return true; + } + + @Override + protected void logLine(String line) { + myLog.append(StringUtil.trimStart(line, myRoot)).append('\n'); + } + } } diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java index cdb04c9cd468..e319664b5db4 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java @@ -6,18 +6,15 @@ import com.intellij.openapi.util.text.StringUtil; import com.intellij.testFramework.UsefulTestCase; import com.intellij.util.io.TestFileSystemBuilder; import org.jetbrains.annotations.Nullable; -import org.jetbrains.jps.builders.impl.BuildDataPathsImpl; -import org.jetbrains.jps.builders.storage.BuildDataPaths; -import org.jetbrains.jps.util.JpsPathUtil; import org.jetbrains.jps.api.CanceledStatus; +import org.jetbrains.jps.builders.impl.BuildDataPathsImpl; import org.jetbrains.jps.builders.impl.BuildRootIndexImpl; import org.jetbrains.jps.builders.impl.BuildTargetIndexImpl; +import org.jetbrains.jps.builders.storage.BuildDataPaths; import org.jetbrains.jps.cmdline.ClasspathBootstrap; import org.jetbrains.jps.cmdline.ProjectDescriptor; import org.jetbrains.jps.incremental.*; -import org.jetbrains.jps.incremental.artifacts.ArtifactBuilderLoggerImpl; import org.jetbrains.jps.incremental.fs.BuildFSState; -import org.jetbrains.jps.incremental.java.JavaBuilderLoggerImpl; import org.jetbrains.jps.incremental.storage.BuildDataManager; import org.jetbrains.jps.incremental.storage.BuildTargetsState; import org.jetbrains.jps.incremental.storage.ProjectTimestamps; @@ -34,6 +31,7 @@ import org.jetbrains.jps.model.library.JpsTypedLibrary; import org.jetbrains.jps.model.library.sdk.JpsSdk; import org.jetbrains.jps.model.module.JpsModule; import org.jetbrains.jps.model.serialization.JpsProjectLoader; +import org.jetbrains.jps.util.JpsPathUtil; import java.io.File; import java.io.IOException; @@ -190,7 +188,7 @@ public abstract class JpsBuildTestCase extends UsefulTestCase { } protected void rebuildAll() { - ProjectDescriptor descriptor = createProjectDescriptor(new BuildLoggingManager(new ArtifactBuilderLoggerImpl(), new JavaBuilderLoggerImpl())); + ProjectDescriptor descriptor = createProjectDescriptor(BuildLoggingManager.DEFAULT); try { CompileScope scope = new CompileScopeImpl(true, BuilderRegistry.getInstance().getTargetTypes(), Collections.>emptySet(), Collections.,Set>emptyMap()); doBuild(descriptor, scope, false, true, false).assertSuccessful(); diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java index 7ba15975a863..442e5d2efd58 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java @@ -19,6 +19,7 @@ import com.intellij.openapi.application.PathManager; import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.io.TestFileSystemBuilder; import com.intellij.util.text.UniqueNameGenerator; +import org.jetbrains.jps.incremental.ProjectBuilderLoggerImpl; import org.jetbrains.jps.util.JpsPathUtil; import org.jetbrains.jps.builders.BuildResult; import org.jetbrains.jps.builders.BuildTarget; @@ -104,7 +105,7 @@ public abstract class ArtifactBuilderTestCase extends JpsBuildTestCase { private BuildResult doBuild(boolean force, JpsArtifact... artifacts) { BuildResult result; - ProjectDescriptor descriptor = createProjectDescriptor(new BuildLoggingManager(myArtifactBuilderLogger, new JavaBuilderLoggerImpl())); + ProjectDescriptor descriptor = createProjectDescriptor(new BuildLoggingManager(myArtifactBuilderLogger, new JavaBuilderLoggerImpl(), new ProjectBuilderLoggerImpl())); try { myArtifactBuilderLogger.clear(); List> targets = new ArrayList>();