From 5b775b56ea473eb074ff431b00107de5e2949c03 Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 2 Oct 2012 14:57:17 +0400 Subject: [PATCH] external compiler: extracted interface to process dirty files --- .../compiler/BaseCompilerTestCase.java | 2 +- .../jps/builders/BuildRootDescriptor.java | 3 ++ .../jps/builders/BuildRootIndex.java | 11 ++++- .../jps/builders/DirtyFilesHolder.java | 12 +++++ .../FileProcessor.java | 6 +-- .../jps/builders/impl/BuildRootIndexImpl.java | 32 +++++++++++-- .../jetbrains/jps/cmdline/BuildRunner.java | 18 ++++---- .../jetbrains/jps/cmdline/BuildSession.java | 40 +++++------------ .../jps/incremental/FSOperations.java | 17 +------ .../jps/incremental/FilesCollector.java | 45 ------------------- .../jps/incremental/IncProjectBuilder.java | 26 ++++++----- .../jps/incremental/ModuleLevelBuilder.java | 6 ++- .../jps/incremental/ResourcePatterns.java | 5 ++- .../instructions/ArtifactRootDescriptor.java | 12 +++++ .../jps/incremental/fs/BuildFSState.java | 41 ++++++++--------- .../jps/incremental/fs/RootDescriptor.java | 14 ++++++ .../jps/incremental/java/JavaBuilder.java | 10 +++-- .../resources/ResourcesBuilder.java | 20 ++++++--- .../AndroidLibraryPackagingBuilder.java | 9 ++-- .../AndroidSourceGeneratingBuilder.java | 16 ++++--- .../jps/incremental/groovy/GroovyBuilder.java | 17 ++++--- 21 files changed, 192 insertions(+), 170 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/builders/DirtyFilesHolder.java rename jps/jps-builders/src/org/jetbrains/jps/{incremental => builders}/FileProcessor.java (50%) delete mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/FilesCollector.java diff --git a/java/compiler/impl/testSrc/com/intellij/compiler/BaseCompilerTestCase.java b/java/compiler/impl/testSrc/com/intellij/compiler/BaseCompilerTestCase.java index f480ceb7aedc..ec393dcc1812 100644 --- a/java/compiler/impl/testSrc/com/intellij/compiler/BaseCompilerTestCase.java +++ b/java/compiler/impl/testSrc/com/intellij/compiler/BaseCompilerTestCase.java @@ -261,7 +261,7 @@ public abstract class BaseCompilerTestCase extends ModuleTestCase { return set; } - protected void changeFile(VirtualFile file) throws Exception { + protected void changeFile(VirtualFile file) { changeFile(file, null); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootDescriptor.java b/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootDescriptor.java index 7043b930a994..b347e5e5b8f7 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootDescriptor.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootDescriptor.java @@ -1,6 +1,7 @@ package org.jetbrains.jps.builders; import java.io.File; +import java.io.FileFilter; /** * @author nik @@ -11,4 +12,6 @@ public abstract class BuildRootDescriptor { public abstract File getRootFile(); public abstract BuildTarget getTarget(); + + public abstract FileFilter createFileFilter(); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootIndex.java b/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootIndex.java index 0d45b2b71e7c..eb53d4408c55 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootIndex.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/BuildRootIndex.java @@ -6,6 +6,7 @@ import org.jetbrains.jps.incremental.CompileContext; import org.jetbrains.jps.incremental.fs.RootDescriptor; import java.io.File; +import java.io.FileFilter; import java.util.Collection; import java.util.List; @@ -21,7 +22,7 @@ public interface BuildRootIndex { List getTempTargetRoots(@NotNull BuildTarget target, @NotNull CompileContext context); @NotNull - List getRootDescriptors(@NotNull File root, @NotNull Collection>> types, + List getRootDescriptors(@NotNull File root, @Nullable Collection>> types, @Nullable CompileContext context); void associateTempRoot(@NotNull CompileContext context, @NotNull BuildTarget target, @NotNull R root); @@ -35,9 +36,15 @@ public interface BuildRootIndex { @NotNull Collection findAllParentDescriptors(@NotNull File file, - @NotNull Collection>> types, + @Nullable Collection>> types, @Nullable CompileContext context); + @NotNull + Collection findAllParentDescriptors(@NotNull File file, @Nullable CompileContext context); + @Nullable RootDescriptor getModuleAndRoot(@Nullable CompileContext context, File file); + + @NotNull + FileFilter getRootFilter(@NotNull BuildRootDescriptor descriptor); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/DirtyFilesHolder.java b/jps/jps-builders/src/org/jetbrains/jps/builders/DirtyFilesHolder.java new file mode 100644 index 000000000000..404fb9ec87c0 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/DirtyFilesHolder.java @@ -0,0 +1,12 @@ +package org.jetbrains.jps.builders; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; + +/** + * @author nik + */ +public interface DirtyFilesHolder> { + void processDirtyFiles(@NotNull FileProcessor processor) throws IOException; +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/FileProcessor.java b/jps/jps-builders/src/org/jetbrains/jps/builders/FileProcessor.java similarity index 50% rename from jps/jps-builders/src/org/jetbrains/jps/incremental/FileProcessor.java rename to jps/jps-builders/src/org/jetbrains/jps/builders/FileProcessor.java index ab4ca9a725ca..c55950042849 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/FileProcessor.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/FileProcessor.java @@ -1,4 +1,4 @@ -package org.jetbrains.jps.incremental; +package org.jetbrains.jps.builders; import java.io.File; import java.io.IOException; @@ -7,9 +7,9 @@ import java.io.IOException; * @author Eugene Zhuravlev * Date: 9/21/11 */ -public interface FileProcessor { +public interface FileProcessor> { /** * @return true if processing should continue, false if should stop */ - boolean apply(ModuleBuildTarget target, File file, String sourceRoot) throws IOException; + boolean apply(T target, File file, R root) throws IOException; } diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildRootIndexImpl.java b/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildRootIndexImpl.java index 73f8474820bc..a6db7f3e56e0 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildRootIndexImpl.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/impl/BuildRootIndexImpl.java @@ -3,6 +3,7 @@ package org.jetbrains.jps.builders.impl; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.SmartList; +import com.intellij.util.containers.ConcurrentHashMap; import gnu.trove.THashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -17,7 +18,9 @@ import org.jetbrains.jps.model.JpsModel; import org.jetbrains.jps.service.JpsServiceManager; import java.io.File; +import java.io.FileFilter; import java.util.*; +import java.util.concurrent.ConcurrentMap; /** * @author nik @@ -27,6 +30,7 @@ public class BuildRootIndexImpl implements BuildRootIndex { private static final Key, List>> TEMP_TARGET_ROOTS_MAP = Key.create("_module_to_root_map"); private HashMap, List> myRootsByTarget; private THashMap> myRootToDescriptor; + private ConcurrentMap myFileFilters; public BuildRootIndexImpl(BuildTargetIndex targetIndex, JpsModel model, @@ -34,6 +38,7 @@ public class BuildRootIndexImpl implements BuildRootIndex { File dataStorageRoot, final IgnoredFileIndex ignoredFileIndex) { myRootsByTarget = new HashMap, List>(); myRootToDescriptor = new THashMap>(FileUtil.FILE_HASHING_STRATEGY); + myFileFilters = new ConcurrentHashMap(); final Iterable rootsProviders = JpsServiceManager.getInstance().getExtensions(AdditionalRootsProviderService.class); for (BuildTargetType targetType : BuilderRegistry.getInstance().getTargetTypes()) { for (BuildTarget target : targetIndex.getAllTargets(targetType)) { @@ -73,13 +78,13 @@ public class BuildRootIndexImpl implements BuildRootIndex { @NotNull @Override public List getRootDescriptors(@NotNull File root, - @NotNull Collection>> types, + @Nullable Collection>> types, @Nullable CompileContext context) { List descriptors = myRootToDescriptor.get(root); List result = new SmartList(); if (descriptors != null) { for (BuildRootDescriptor descriptor : descriptors) { - if (types.contains(descriptor.getTarget().getTargetType())) { + if (types == null || types.contains(descriptor.getTarget().getTargetType())) { //noinspection unchecked result.add((R)descriptor); } @@ -89,7 +94,7 @@ public class BuildRootIndexImpl implements BuildRootIndex { final Map contextMap = ROOT_DESCRIPTOR_MAP.get(context); if (contextMap != null) { BuildRootDescriptor descriptor = contextMap.get(root); - if (descriptor != null && types.contains(descriptor.getTarget().getTargetType())) { + if (descriptor != null && (types == null || types.contains(descriptor.getTarget().getTargetType()))) { //noinspection unchecked result.add((R)descriptor); } @@ -172,10 +177,12 @@ public class BuildRootIndexImpl implements BuildRootIndex { return null; } + + @Override @NotNull public Collection findAllParentDescriptors(@NotNull File file, - @NotNull Collection>> types, + @Nullable Collection>> types, @Nullable CompileContext context) { File current = file; Collection result = null; @@ -195,6 +202,12 @@ public class BuildRootIndexImpl implements BuildRootIndex { return result != null ? result : Collections.emptyList(); } + @NotNull + @Override + public Collection findAllParentDescriptors(@NotNull File file, @Nullable CompileContext context) { + return findAllParentDescriptors(file, null, context); + } + @Override @NotNull public Collection clearTempRoots(@NotNull CompileContext context) { @@ -213,4 +226,15 @@ public class BuildRootIndexImpl implements BuildRootIndex { public RootDescriptor getModuleAndRoot(@Nullable CompileContext context, File file) { return findParentDescriptor(file, JavaModuleBuildTargetType.ALL_TYPES, context); } + + @NotNull + @Override + public FileFilter getRootFilter(@NotNull BuildRootDescriptor descriptor) { + FileFilter filter = myFileFilters.get(descriptor); + if (filter == null) { + filter = descriptor.createFileFilter(); + myFileFilters.put(descriptor, filter); + } + return filter; + } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java index 2190556beefc..9536aed12ac4 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java @@ -7,24 +7,24 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.api.BuildType; import org.jetbrains.jps.api.CanceledStatus; import org.jetbrains.jps.api.GlobalOptions; +import org.jetbrains.jps.builders.BuildRootDescriptor; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.BuildTargetLoader; import org.jetbrains.jps.builders.BuildTargetType; import org.jetbrains.jps.builders.impl.BuildRootIndexImpl; import org.jetbrains.jps.builders.impl.BuildTargetIndexImpl; -import org.jetbrains.jps.indices.impl.IgnoredFileIndexImpl; -import org.jetbrains.jps.indices.impl.ModuleExcludeIndexImpl; -import org.jetbrains.jps.indices.ModuleExcludeIndex; import org.jetbrains.jps.builders.java.dependencyView.Callbacks; import org.jetbrains.jps.incremental.*; import org.jetbrains.jps.incremental.fs.BuildFSState; -import org.jetbrains.jps.incremental.fs.RootDescriptor; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; import org.jetbrains.jps.incremental.storage.BuildDataManager; import org.jetbrains.jps.incremental.storage.BuildTargetsState; import org.jetbrains.jps.incremental.storage.ProjectTimestamps; import org.jetbrains.jps.incremental.storage.Timestamps; +import org.jetbrains.jps.indices.ModuleExcludeIndex; +import org.jetbrains.jps.indices.impl.IgnoredFileIndexImpl; +import org.jetbrains.jps.indices.impl.ModuleExcludeIndexImpl; import org.jetbrains.jps.model.JpsModel; import java.io.File; @@ -171,16 +171,16 @@ public class BuildRunner { files = new HashMap, Set>(); for (String path : paths) { final File file = new File(path); - final RootDescriptor rd = pd.getBuildRootIndex().getModuleAndRoot(null, file); - if (rd != null) { - Set fileSet = files.get(rd.target); + final Collection descriptors = pd.getBuildRootIndex().findAllParentDescriptors(file, null); + for (BuildRootDescriptor descriptor : descriptors) { + Set fileSet = files.get(descriptor.getTarget()); if (fileSet == null) { fileSet = new THashSet(FileUtil.FILE_HASHING_STRATEGY); - files.put(rd.target, fileSet); + files.put(descriptor.getTarget(), fileSet); } fileSet.add(file); if (buildType == BuildType.FORCED_COMPILATION) { - pd.fsState.markDirty(null, file, rd, timestamps); + pd.fsState.markDirty(null, file, descriptor, timestamps); } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java index 3582f36ce1ed..d3f3a434dbef 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java @@ -11,16 +11,14 @@ import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.Channels; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.api.*; +import org.jetbrains.jps.builders.BuildRootDescriptor; import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType; import org.jetbrains.jps.builders.java.dependencyView.Callbacks; import org.jetbrains.jps.incremental.MessageHandler; import org.jetbrains.jps.incremental.ModuleBuildTarget; import org.jetbrains.jps.incremental.Utils; -import org.jetbrains.jps.incremental.artifacts.ArtifactBuildTargetType; -import org.jetbrains.jps.incremental.artifacts.instructions.ArtifactRootDescriptor; import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.fs.FSState; -import org.jetbrains.jps.incremental.fs.RootDescriptor; import org.jetbrains.jps.incremental.messages.*; import org.jetbrains.jps.incremental.storage.Timestamps; import org.jetbrains.jps.model.module.JpsModule; @@ -263,49 +261,33 @@ final class BuildSession implements Runnable, CanceledStatus { for (String deleted : event.getDeletedPathsList()) { final File file = new File(deleted); - final RootDescriptor rd = pd.getBuildRootIndex().getModuleAndRoot(null, file); - if (rd != null) { + Collection descriptor = pd.getBuildRootIndex().findAllParentDescriptors(file, null, null); + if (!descriptor.isEmpty()) { if (Utils.IS_TEST_MODE) { LOG.info("Applying deleted path from fs event: " + file.getPath()); } - pd.fsState.registerDeleted(rd.target, file, timestamps); + for (BuildRootDescriptor rootDescriptor : descriptor) { + pd.fsState.registerDeleted(rootDescriptor.getTarget(), file, timestamps); + } } else if (Utils.IS_TEST_MODE) { LOG.info("Skipping deleted path: " + file.getPath()); } - - Collection descriptor = pd.getBuildRootIndex().findAllParentDescriptors(file, Collections.singletonList(ArtifactBuildTargetType.INSTANCE), null); - if (!descriptor.isEmpty()) { - if (Utils.IS_TEST_MODE) { - LOG.info("Applying deleted path from fs event to artifacts: " + file.getPath()); - } - for (ArtifactRootDescriptor rootDescriptor : descriptor) - pd.fsState.registerDeleted(rootDescriptor.getTarget(), file, timestamps); - } } for (String changed : event.getChangedPathsList()) { final File file = new File(changed); - final RootDescriptor rd = pd.getBuildRootIndex().getModuleAndRoot(null, file); - if (rd != null) { + Collection descriptors = pd.getBuildRootIndex().findAllParentDescriptors(file, null, null); + if (!descriptors.isEmpty()) { if (Utils.IS_TEST_MODE) { LOG.info("Applying dirty path from fs event: " + file.getPath()); } - pd.fsState.markDirty(null, file, rd, timestamps); + for (BuildRootDescriptor descriptor : descriptors) { + pd.fsState.markDirty(null, file, descriptor, timestamps); + } } else if (Utils.IS_TEST_MODE) { LOG.info("Skipping dirty path: " + file.getPath()); } - - Collection descriptors = pd.getBuildRootIndex().findAllParentDescriptors(file, Collections - .singletonList(ArtifactBuildTargetType.INSTANCE), null); - if (!descriptors.isEmpty()) { - if (Utils.IS_TEST_MODE) { - LOG.info("Applying dirty path from fs event to artifacts: " + file.getPath()); - } - for (ArtifactRootDescriptor descriptor : descriptors) { - pd.fsState.markDirty(null, file, descriptor, timestamps); - } - } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java index ed45b36b5a37..4fc9ed4d9f2a 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java @@ -8,16 +8,16 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.JpsPathUtil; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.builders.BuildTarget; +import org.jetbrains.jps.builders.FileProcessor; import org.jetbrains.jps.builders.impl.BuildTargetChunk; -import org.jetbrains.jps.indices.ModuleExcludeIndex; import org.jetbrains.jps.cmdline.ProjectDescriptor; import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.fs.RootDescriptor; import org.jetbrains.jps.incremental.storage.Timestamps; +import org.jetbrains.jps.indices.ModuleExcludeIndex; import org.jetbrains.jps.model.java.JpsJavaClasspathKind; import org.jetbrains.jps.model.java.JpsJavaExtensionService; import org.jetbrains.jps.model.module.JpsModule; -import org.jetbrains.jps.model.module.JpsModuleType; import java.io.File; import java.io.IOException; @@ -114,19 +114,6 @@ public class FSOperations { processFilesToRecompile(context, chunk, Condition.TRUE, processor); } - public static void processFilesToRecompile(final CompileContext context, - final ModuleChunk chunk, - final JpsModuleType moduleType, - final FileProcessor processor) throws IOException { - final Condition moduleFilter = new Condition() { - public boolean value(final JpsModule module) { - return module.getModuleType() == moduleType; - } - }; - - processFilesToRecompile(context, chunk, moduleFilter, processor); - } - public static void processFilesToRecompile(final CompileContext context, final ModuleChunk chunk, final Condition moduleFilter, diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/FilesCollector.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/FilesCollector.java deleted file mode 100644 index f06d32fe727c..000000000000 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/FilesCollector.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.jetbrains.jps.incremental; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; - -/** - * @author Eugene Zhuravlev - * Date: 9/21/11 - */ -public class FilesCollector implements FileProcessor{ - public static FileFilter ALL_FILES = new FileFilter() { - public boolean accept(File pathname) { - return true; - } - }; - private final Collection myContainer; - private final FileFilter myFilter; - - public FilesCollector() { - this(ALL_FILES); - } - - public FilesCollector(FileFilter filter) { - this(new ArrayList(), filter); - } - - public FilesCollector(Collection container, FileFilter filter) { - myFilter = filter; - myContainer = container; - } - - public Collection getFiles() { - return myContainer; - } - - public boolean apply(ModuleBuildTarget target, File file, String sourceRoot) throws IOException { - if (myFilter.accept(file)) { - myContainer.add(file); - } - return true; - } -} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 77a0dc6524ec..d155a0bf1745 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -20,10 +20,7 @@ import org.jetbrains.jps.ProjectPaths; import org.jetbrains.jps.api.CanceledStatus; import org.jetbrains.jps.api.GlobalOptions; import org.jetbrains.jps.api.RequestFuture; -import org.jetbrains.jps.builders.BuildRootDescriptor; -import org.jetbrains.jps.builders.BuildTarget; -import org.jetbrains.jps.builders.BuildTargetIndex; -import org.jetbrains.jps.builders.BuildTargetType; +import org.jetbrains.jps.builders.*; import org.jetbrains.jps.builders.impl.BuildTargetChunk; import org.jetbrains.jps.builders.java.JavaBuilderUtil; import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType; @@ -752,7 +749,7 @@ public class IncProjectBuilder { } // return true if changed something, false otherwise - private boolean runModuleLevelBuilders(final CompileContext context, ModuleChunk chunk) throws ProjectBuildException { + private boolean runModuleLevelBuilders(final CompileContext context, final ModuleChunk chunk) throws ProjectBuildException { boolean doneSomething = false; boolean rebuildFromScratchRequested = false; float stageCount = myTotalModuleLevelBuilderCount; @@ -763,8 +760,14 @@ public class IncProjectBuilder { nextPassRequired = false; myProjectDescriptor.fsState.beforeNextRoundStart(context, chunk); + DirtyFilesHolder dirtyFilesHolder = new DirtyFilesHolder() { + @Override + public void processDirtyFiles(@NotNull FileProcessor processor) throws IOException { + FSOperations.processFilesToRecompile(context, chunk, processor); + } + }; if (!context.isProjectRebuild()) { - syncOutputFiles(context, chunk); + syncOutputFiles(context, dirtyFilesHolder); } BUILDER_CATEGORY_LOOP: @@ -778,7 +781,7 @@ public class IncProjectBuilder { if (context.isMake()) { processDeletedPaths(context, chunk); } - final ModuleLevelBuilder.ExitCode buildResult = builder.build(context, chunk); + final ModuleLevelBuilder.ExitCode buildResult = builder.build(context, chunk, dirtyFilesHolder); doneSomething |= (buildResult != ModuleLevelBuilder.ExitCode.NOTHING_DONE); @@ -830,16 +833,17 @@ public class IncProjectBuilder { return doneSomething; } - private static void syncOutputFiles(final CompileContext context, ModuleChunk chunk) throws ProjectBuildException { + private static void syncOutputFiles(final CompileContext context, + DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { final BuildDataManager dataManager = context.getProjectDescriptor().dataManager; try { final Collection allOutputs = new LinkedList(); - FSOperations.processFilesToRecompile(context, chunk, new FileProcessor() { + dirtyFilesHolder.processDirtyFiles(new FileProcessor() { private final Map storageMap = new HashMap(); @Override - public boolean apply(ModuleBuildTarget target, File file, String sourceRoot) throws IOException { + public boolean apply(ModuleBuildTarget target, File file, RootDescriptor sourceRoot) throws IOException { SourceToOutputMapping srcToOut = storageMap.get(target); if (srcToOut == null) { srcToOut = dataManager.getSourceToOutputMap(target); @@ -940,7 +944,7 @@ public class IncProjectBuilder { } final Timestamps timestamps = pd.timestamps.getStorage(); for (RootDescriptor rd : pd.getBuildRootIndex().getTargetRoots(target, context)) { - marked |= fsState.markAllUpToDate(context.getProjectDescriptor().jpsProject, context.getScope(), rd, timestamps, context.getCompilationStartStamp()); + marked |= fsState.markAllUpToDate(context, rd, timestamps); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java index cacabc1c737a..5eefe58c52d4 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ModuleLevelBuilder.java @@ -1,6 +1,8 @@ package org.jetbrains.jps.incremental; import org.jetbrains.jps.ModuleChunk; +import org.jetbrains.jps.builders.DirtyFilesHolder; +import org.jetbrains.jps.incremental.fs.RootDescriptor; import java.io.File; @@ -21,7 +23,9 @@ public abstract class ModuleLevelBuilder extends Builder { NOTHING_DONE, OK, ABORT, ADDITIONAL_PASS_REQUIRED, CHUNK_REBUILD_REQUIRED } - public abstract ExitCode build(CompileContext context, ModuleChunk chunk) throws ProjectBuildException; + public abstract ExitCode build(CompileContext context, + ModuleChunk chunk, + DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException; public boolean shouldHonorFileEncodingForCompilation(File file) { return false; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ResourcePatterns.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ResourcePatterns.java index 229e1e0b76d7..8cfcc725a764 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/ResourcePatterns.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ResourcePatterns.java @@ -43,12 +43,13 @@ public class ResourcePatterns { } } - public boolean isResourceFile(File file, @NotNull final String srcRoot) { + public boolean isResourceFile(File file, @NotNull final File srcRoot) { final String name = file.getName(); final String relativePathToParent; final String parentPath = file.getParent(); if (parentPath != null) { - relativePathToParent = "/" + FileUtil.getRelativePath(srcRoot, FileUtil.toSystemIndependentName(parentPath), '/', SystemInfo.isFileSystemCaseSensitive); + relativePathToParent = "/" + FileUtil.getRelativePath(FileUtil.toSystemIndependentName(srcRoot.getAbsolutePath()), + FileUtil.toSystemIndependentName(parentPath), '/', SystemInfo.isFileSystemCaseSensitive); } else { relativePathToParent = null; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootDescriptor.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootDescriptor.java index 2109dad4ec51..754f98978bce 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootDescriptor.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootDescriptor.java @@ -8,6 +8,7 @@ import org.jetbrains.jps.incremental.artifacts.ArtifactBuildTarget; import org.jetbrains.jps.incremental.artifacts.ArtifactOutputToSourceMapping; import java.io.File; +import java.io.FileFilter; import java.io.IOException; import java.io.PrintWriter; @@ -15,6 +16,12 @@ import java.io.PrintWriter; * @author nik */ public abstract class ArtifactRootDescriptor extends BuildRootDescriptor { + private static final FileFilter ALL_FILES_FILTER = new FileFilter() { + @Override + public boolean accept(File file) { + return true; + } + }; protected final File myRoot; private final SourceFileFilter myFilter; private final int myRootIndex; @@ -53,6 +60,11 @@ public abstract class ArtifactRootDescriptor extends BuildRootDescriptor { return myTarget; } + @Override + public FileFilter createFileFilter() { + return ALL_FILES_FILTER; + } + @NotNull public final File getRootFile() { return myRoot; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java index f2033f89417b..0244743ed3b2 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java @@ -3,20 +3,22 @@ package org.jetbrains.jps.incremental.fs; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.io.FileSystemUtil; -import com.intellij.openapi.util.io.FileUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.builders.BuildRootDescriptor; +import org.jetbrains.jps.builders.BuildRootIndex; import org.jetbrains.jps.builders.BuildTarget; -import org.jetbrains.jps.incremental.*; +import org.jetbrains.jps.builders.FileProcessor; +import org.jetbrains.jps.incremental.CompileContext; +import org.jetbrains.jps.incremental.CompileScope; +import org.jetbrains.jps.incremental.ModuleBuildTarget; +import org.jetbrains.jps.incremental.Utils; import org.jetbrains.jps.incremental.artifacts.instructions.ArtifactRootDescriptor; import org.jetbrains.jps.incremental.storage.Timestamps; -import org.jetbrains.jps.model.JpsProject; -import org.jetbrains.jps.model.java.JpsJavaExtensionService; -import org.jetbrains.jps.model.java.compiler.JpsCompilerExcludes; import java.io.File; +import java.io.FileFilter; import java.io.IOException; import java.util.Collections; import java.util.Map; @@ -110,23 +112,20 @@ public class BuildFSState extends FSState { setRoundDelta(CURRENT_ROUND_DELTA_KEY, context, new FilesDelta()); } - public boolean processFilesToRecompile(CompileContext context, final ModuleBuildTarget target, final FileProcessor processor) throws IOException { + public > boolean processFilesToRecompile(CompileContext context, final T target, final FileProcessor processor) throws IOException { final Map> data = getSourcesToRecompile(context, target); - JpsProject project = context.getProjectDescriptor().jpsProject; - final JpsCompilerExcludes excludes = JpsJavaExtensionService.getInstance().getOrCreateCompilerConfiguration(project).getCompilerExcludes(); + BuildRootIndex rootIndex = context.getProjectDescriptor().getBuildRootIndex(); final CompileScope scope = context.getScope(); synchronized (data) { for (Map.Entry> entry : data.entrySet()) { - File rootFile = entry.getKey().getRootFile(); - final String rootPath = FileUtil.toSystemIndependentName(rootFile.getPath()); + //noinspection unchecked + R root = (R)entry.getKey(); + FileFilter filter = rootIndex.getRootFilter(root); for (File file : entry.getValue()) { - if (!scope.isAffected(target, file)) { + if (!scope.isAffected(target, file) || !filter.accept(file)) { continue; } - if (excludes.isExcluded(file)) { - continue; - } - if (!processor.apply(target, file, rootPath)) { + if (!processor.apply(target, file, root)) { return false; } } @@ -138,18 +137,16 @@ public class BuildFSState extends FSState { /** * @return true if marked something, false otherwise */ - public boolean markAllUpToDate(JpsProject project, - CompileScope scope, - final RootDescriptor rd, - final Timestamps stamps, - final long compilationStartStamp) throws IOException { + public boolean markAllUpToDate(CompileContext context, final RootDescriptor rd, final Timestamps stamps) throws IOException { boolean marked = false; final FilesDelta delta = getDelta(rd.target); final Set files = delta.clearRecompile(rd); if (files != null) { - final JpsCompilerExcludes excludes = JpsJavaExtensionService.getInstance().getOrCreateCompilerConfiguration(project).getCompilerExcludes(); + FileFilter filter = context.getProjectDescriptor().getBuildRootIndex().getRootFilter(rd); + CompileScope scope = context.getScope(); + final long compilationStartStamp = context.getCompilationStartStamp(); for (File file : files) { - if (!excludes.isExcluded(file)) { + if (filter.accept(file)) { if (scope.isAffected(rd.target, file)) { final long stamp = FileSystemUtil.lastModified(file); if (!rd.isGeneratedSources && stamp > compilationStartStamp) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/RootDescriptor.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/RootDescriptor.java index 9d9cfd7e3ea5..1b150b0ccbb2 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/RootDescriptor.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/RootDescriptor.java @@ -5,8 +5,11 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.BuildRootDescriptor; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.incremental.ModuleBuildTarget; +import org.jetbrains.jps.model.java.JpsJavaExtensionService; +import org.jetbrains.jps.model.java.compiler.JpsCompilerExcludes; import java.io.File; +import java.io.FileFilter; /** * @author Eugene Zhuravlev @@ -50,4 +53,15 @@ public final class RootDescriptor extends BuildRootDescriptor { public BuildTarget getTarget() { return target; } + + @Override + public FileFilter createFileFilter() { + final JpsCompilerExcludes excludes = JpsJavaExtensionService.getInstance().getOrCreateCompilerConfiguration(target.getModule().getProject()).getCompilerExcludes(); + return new FileFilter() { + @Override + public boolean accept(File file) { + return !excludes.isExcluded(file); + } + }; + } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java index ccfc0ba4ea96..007a2b3ff907 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java @@ -32,6 +32,8 @@ import org.jetbrains.jps.ProjectPaths; import org.jetbrains.jps.api.GlobalOptions; import org.jetbrains.jps.api.RequestFuture; import org.jetbrains.jps.builders.BuildRootIndex; +import org.jetbrains.jps.builders.DirtyFilesHolder; +import org.jetbrains.jps.builders.FileProcessor; import org.jetbrains.jps.builders.java.JavaBuilderUtil; import org.jetbrains.jps.builders.java.dependencyView.Callbacks; import org.jetbrains.jps.builders.java.dependencyView.Mappings; @@ -168,13 +170,15 @@ public class JavaBuilder extends ModuleLevelBuilder { return "Java Builder"; } - public ExitCode build(final CompileContext context, final ModuleChunk chunk) throws ProjectBuildException { + public ExitCode build(final CompileContext context, + final ModuleChunk chunk, + DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { try { final Set filesToCompile = new THashSet(FileUtil.FILE_HASHING_STRATEGY); final Set formsToCompile = new THashSet(FileUtil.FILE_HASHING_STRATEGY); - FSOperations.processFilesToRecompile(context, chunk, new FileProcessor() { - public boolean apply(ModuleBuildTarget target, File file, String sourceRoot) throws IOException { + dirtyFilesHolder.processDirtyFiles(new FileProcessor() { + public boolean apply(ModuleBuildTarget target, File file, RootDescriptor sourceRoot) throws IOException { if (JAVA_SOURCES_FILTER.accept(file)) { filesToCompile.add(file); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/resources/ResourcesBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/resources/ResourcesBuilder.java index b26a24a62a37..3e6b4aada61e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/resources/ResourcesBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/resources/ResourcesBuilder.java @@ -6,8 +6,11 @@ import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.io.FileUtil; import org.jetbrains.jps.JpsPathUtil; import org.jetbrains.jps.ModuleChunk; +import org.jetbrains.jps.builders.DirtyFilesHolder; +import org.jetbrains.jps.builders.FileProcessor; import org.jetbrains.jps.builders.storage.SourceToOutputMapping; import org.jetbrains.jps.incremental.*; +import org.jetbrains.jps.incremental.fs.RootDescriptor; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; import org.jetbrains.jps.incremental.messages.ProgressMessage; @@ -44,7 +47,9 @@ public class ResourcesBuilder extends ModuleLevelBuilder { } } - public ExitCode build(final CompileContext context, final ModuleChunk chunk) throws ProjectBuildException { + public ExitCode build(final CompileContext context, + final ModuleChunk chunk, + DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { final ResourcePatterns patterns = ResourcePatterns.KEY.get(context); assert patterns != null; try { @@ -61,9 +66,9 @@ public class ResourcesBuilder extends ModuleLevelBuilder { } }; - FSOperations.processFilesToRecompile(context, chunk, moduleFilter, new FileProcessor() { - public boolean apply(final ModuleBuildTarget target, final File file, final String sourceRoot) throws IOException { - if (patterns.isResourceFile(file, sourceRoot)) { + FSOperations.processFilesToRecompile(context, chunk, moduleFilter, new FileProcessor() { + public boolean apply(final ModuleBuildTarget target, final File file, final RootDescriptor sourceRoot) throws IOException { + if (patterns.isResourceFile(file, sourceRoot.root)) { try { context.processMessage(new ProgressMessage("Copying " + file.getPath())); doneSomething.set(true); @@ -94,14 +99,15 @@ public class ResourcesBuilder extends ModuleLevelBuilder { private static void copyResource(CompileContext context, JpsModule module, File file, - String sourceRoot, + RootDescriptor sourceRoot, final SourceToOutputMapping outputToSourceMapping, final boolean tests) throws IOException { final String outputRootUrl = JpsJavaExtensionService.getInstance().getOutputUrl(module, tests); if (outputRootUrl == null) { return; } - final String relativePath = FileUtil.getRelativePath(sourceRoot, FileUtil.toSystemIndependentName(file.getPath()), '/'); - final String prefix = JpsJavaExtensionService.getInstance().getSourcePrefix(module, JpsPathUtil.pathToUrl(sourceRoot)); + String rootPath = FileUtil.toSystemIndependentName(sourceRoot.root.getAbsolutePath()); + final String relativePath = FileUtil.getRelativePath(rootPath, FileUtil.toSystemIndependentName(file.getPath()), '/'); + final String prefix = JpsJavaExtensionService.getInstance().getSourcePrefix(module, JpsPathUtil.pathToUrl(rootPath)); final StringBuilder targetPath = new StringBuilder(); targetPath.append(JpsPathUtil.urlToPath(outputRootUrl)); diff --git a/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidLibraryPackagingBuilder.java b/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidLibraryPackagingBuilder.java index 0af07a13e330..d27f6f8e213a 100644 --- a/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidLibraryPackagingBuilder.java +++ b/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidLibraryPackagingBuilder.java @@ -7,10 +7,9 @@ import org.jetbrains.annotations.NonNls; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.ProjectPaths; import org.jetbrains.jps.android.model.JpsAndroidModuleExtension; -import org.jetbrains.jps.incremental.BuilderCategory; -import org.jetbrains.jps.incremental.CompileContext; -import org.jetbrains.jps.incremental.ModuleLevelBuilder; -import org.jetbrains.jps.incremental.ProjectBuildException; +import org.jetbrains.jps.builders.DirtyFilesHolder; +import org.jetbrains.jps.incremental.*; +import org.jetbrains.jps.incremental.fs.RootDescriptor; import org.jetbrains.jps.incremental.messages.ProgressMessage; import org.jetbrains.jps.model.module.JpsModule; @@ -29,7 +28,7 @@ public class AndroidLibraryPackagingBuilder extends ModuleLevelBuilder { } @Override - public ExitCode build(CompileContext context, ModuleChunk chunk) throws ProjectBuildException { + public ExitCode build(CompileContext context, ModuleChunk chunk, DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { if (chunk.containsTests() || !AndroidJpsUtil.containsAndroidFacet(chunk) || AndroidJpsUtil.isLightBuild(context)) { return ExitCode.NOTHING_DONE; } diff --git a/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidSourceGeneratingBuilder.java b/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidSourceGeneratingBuilder.java index 29ddd160c3ef..d5b7dd222055 100644 --- a/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidSourceGeneratingBuilder.java +++ b/plugins/android/jps-plugin/src/org/jetbrains/jps/android/AndroidSourceGeneratingBuilder.java @@ -21,6 +21,8 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.android.model.JpsAndroidModuleExtension; +import org.jetbrains.jps.builders.DirtyFilesHolder; +import org.jetbrains.jps.builders.FileProcessor; import org.jetbrains.jps.builders.storage.SourceToOutputMapping; import org.jetbrains.jps.incremental.*; import org.jetbrains.jps.incremental.fs.RootDescriptor; @@ -71,20 +73,24 @@ public class AndroidSourceGeneratingBuilder extends ModuleLevelBuilder { } @Override - public ModuleLevelBuilder.ExitCode build(CompileContext context, ModuleChunk chunk) throws ProjectBuildException { + public ModuleLevelBuilder.ExitCode build(CompileContext context, + ModuleChunk chunk, + DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { if (chunk.containsTests() || !AndroidJpsUtil.containsAndroidFacet(chunk)) { return ExitCode.NOTHING_DONE; } try { - return doBuild(context, chunk); + return doBuild(context, chunk, dirtyFilesHolder); } catch (Exception e) { return AndroidJpsUtil.handleException(context, e, BUILDER_NAME); } } - private static ModuleLevelBuilder.ExitCode doBuild(CompileContext context, ModuleChunk chunk) throws IOException { + private static ModuleLevelBuilder.ExitCode doBuild(CompileContext context, + ModuleChunk chunk, + DirtyFilesHolder dirtyFilesHolder) throws IOException { final Map moduleDataMap = computeModuleDatas(chunk.getModules(), context); if (moduleDataMap == null || moduleDataMap.size() == 0) { return ExitCode.ABORT; @@ -104,9 +110,9 @@ public class AndroidSourceGeneratingBuilder extends ModuleLevelBuilder { final Map idlFilesToCompile = new HashMap(); final Map rsFilesToCompile = new HashMap(); - FSOperations.processFilesToRecompile(context, chunk, new FileProcessor() { + dirtyFilesHolder.processDirtyFiles(new FileProcessor() { @Override - public boolean apply(ModuleBuildTarget target, File file, String sourceRoot) throws IOException { + public boolean apply(ModuleBuildTarget target, File file, RootDescriptor sourceRoot) throws IOException { final JpsAndroidModuleExtension extension = AndroidJpsUtil.getExtension(target.getModule()); if (extension == null) { diff --git a/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java b/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java index 1e208eab906f..5783bdb9d192 100644 --- a/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java +++ b/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java @@ -11,6 +11,8 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.asm4.ClassReader; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.builders.BuildRootIndex; +import org.jetbrains.jps.builders.DirtyFilesHolder; +import org.jetbrains.jps.builders.FileProcessor; import org.jetbrains.jps.builders.java.JavaBuilderUtil; import org.jetbrains.jps.builders.java.dependencyView.Callbacks; import org.jetbrains.jps.builders.java.dependencyView.Mappings; @@ -63,9 +65,11 @@ public class GroovyBuilder extends ModuleLevelBuilder { return myBuilderName; } - public ModuleLevelBuilder.ExitCode build(final CompileContext context, ModuleChunk chunk) throws ProjectBuildException { + public ModuleLevelBuilder.ExitCode build(final CompileContext context, + ModuleChunk chunk, + DirtyFilesHolder dirtyFilesHolder) throws ProjectBuildException { try { - final List toCompile = collectChangedFiles(context, chunk); + final List toCompile = collectChangedFiles(context, dirtyFilesHolder); if (toCompile.isEmpty()) { return ExitCode.NOTHING_DONE; } @@ -233,14 +237,15 @@ public class GroovyBuilder extends ModuleLevelBuilder { return isGroovyFile(file.getAbsolutePath()); } - private static List collectChangedFiles(CompileContext context, ModuleChunk chunk) throws IOException { + private static List collectChangedFiles(CompileContext context, + DirtyFilesHolder dirtyFilesHolder) throws IOException { final ResourcePatterns patterns = ResourcePatterns.KEY.get(context); assert patterns != null; final List toCompile = new ArrayList(); - FSOperations.processFilesToRecompile(context, chunk, new FileProcessor() { - public boolean apply(ModuleBuildTarget target, File file, String sourceRoot) throws IOException { + dirtyFilesHolder.processDirtyFiles(new FileProcessor() { + public boolean apply(ModuleBuildTarget target, File file, RootDescriptor sourceRoot) throws IOException { final String path = file.getPath(); - if (isGroovyFile(path) && !patterns.isResourceFile(file, sourceRoot)) { //todo file type check + if (isGroovyFile(path) && !patterns.isResourceFile(file, sourceRoot.root)) { //todo file type check toCompile.add(file); } return true;