From aa2ad1db58649670811e790fa21bbc6763d7e794 Mon Sep 17 00:00:00 2001 From: Eugene Zhuravlev Date: Mon, 6 Jul 2015 13:24:30 +0200 Subject: [PATCH] FilesDelta data synchronization made more explicit; some files could have been incorrectly marked as up-to-date: now comparing file timestamp against build_target compilation start, instead of global adjustable compilation start stamp. --- .../jetbrains/jps/cmdline/BuildRunner.java | 2 +- .../jetbrains/jps/cmdline/BuildSession.java | 11 +- .../ChainedTargetsBuildListener.java | 2 +- .../jps/incremental/CompileContext.java | 7 +- .../jps/incremental/CompileContextImpl.java | 22 +- .../jps/incremental/FSOperations.java | 4 +- .../jps/incremental/IncProjectBuilder.java | 105 ++++---- .../jps/incremental/fs/BuildFSState.java | 126 +++++----- .../jps/incremental/fs/FilesDelta.java | 226 +++++++++++------- 9 files changed, 289 insertions(+), 216 deletions(-) diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java index 1b893a45a302..fc59a611eacc 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java @@ -210,7 +210,7 @@ public class BuildRunner { } fileSet.add(file); if (targetTypesToForceBuild.contains(descriptor.getTarget().getTargetType())) { - pd.fsState.markDirty(null, file, descriptor, timestamps, false); + pd.fsState.markDirty(null, file, descriptor, timestamps); } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java index c5a19f37d5aa..1b6bf999b702 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java @@ -253,7 +253,7 @@ final class BuildSession implements Runnable, CanceledStatus { else { // apply events to already loaded state try { - applyFSEvent(pd, myInitialFSDelta, false); + applyFSEvent(pd, myInitialFSDelta); } catch (Throwable e) { LOG.error(e); @@ -269,7 +269,7 @@ final class BuildSession implements Runnable, CanceledStatus { try { try { fsState.load(fsStateStream, pd.getModel(), pd.getBuildRootIndex()); - applyFSEvent(pd, myInitialFSDelta, false); + applyFSEvent(pd, myInitialFSDelta); TimingLog.LOG.debug("FS Delta loaded"); } finally { @@ -349,7 +349,7 @@ final class BuildSession implements Runnable, CanceledStatus { @Override public void run() { try { - applyFSEvent(myProjectDescriptor, event, true); + applyFSEvent(myProjectDescriptor, event); myLastEventOrdinal += 1; } catch (IOException e) { @@ -378,8 +378,7 @@ final class BuildSession implements Runnable, CanceledStatus { } } - private static void applyFSEvent(ProjectDescriptor pd, @Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent event, - final boolean saveEventStamp) throws IOException { + private static void applyFSEvent(ProjectDescriptor pd, @Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent event) throws IOException { if (event == null) { return; } @@ -426,7 +425,7 @@ final class BuildSession implements Runnable, CanceledStatus { pd.getFSCache().clear(); cacheCleared = true; } - pd.fsState.markDirty(null, file, descriptor, timestamps, saveEventStamp); + pd.fsState.markDirty(null, file, descriptor, timestamps); } else { if (LOG.isDebugEnabled()) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java index 489dc218297a..304aa5250b6a 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java @@ -47,7 +47,7 @@ class ChainedTargetsBuildListener implements BuildListener { Collection descriptors = rootsIndex.findAllParentDescriptors(file, null, myContext); for (BuildRootDescriptor descriptor : descriptors) { try { - fsState.markDirty(myContext, file, descriptor, myContext.getProjectDescriptor().timestamps.getStorage(), false); + fsState.markDirty(myContext, file, descriptor, myContext.getProjectDescriptor().timestamps.getStorage()); } catch (IOException ignored) { } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java index ab21ab49b475..6f994244c4bd 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java @@ -19,9 +19,12 @@ import com.intellij.openapi.util.UserDataHolder; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.api.CanceledStatus; +import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.logging.BuildLoggingManager; import org.jetbrains.jps.cmdline.ProjectDescriptor; +import java.util.Collection; + /** * @author Eugene Zhuravlev * Date: 7/8/12 @@ -58,9 +61,9 @@ public interface CompileContext extends UserDataHolder, MessageHandler { void setDone(float done); - long getCompilationStartStamp(); + long getCompilationStartStamp(BuildTarget target); - void updateCompilationStartStamp(); + void setCompilationStartStamp(Collection> target, long stamp); void markNonIncremental(ModuleBuildTarget target); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContextImpl.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContextImpl.java index 5d35fa2a4c13..9886590b5d30 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContextImpl.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContextImpl.java @@ -18,9 +18,11 @@ package org.jetbrains.jps.incremental; import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.UserDataHolderBase; import com.intellij.util.EventDispatcher; +import gnu.trove.TObjectLongHashMap; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.api.CanceledStatus; +import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.java.JavaBuilderUtil; import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType; import org.jetbrains.jps.builders.logging.BuildLoggingManager; @@ -42,7 +44,7 @@ public class CompileContextImpl extends UserDataHolderBase implements CompileCon private final MessageHandler myDelegateMessageHandler; private final Set myNonIncrementalModules = new HashSet(); - private volatile long myCompilationStartStamp; + private final TObjectLongHashMap> myCompilationStartStamp = new TObjectLongHashMap>(); private final ProjectDescriptor myProjectDescriptor; private final Map myBuilderParams; private final CanceledStatus myCancelStatus; @@ -57,19 +59,23 @@ public class CompileContextImpl extends UserDataHolderBase implements CompileCon myProjectDescriptor = pd; myBuilderParams = Collections.unmodifiableMap(builderParams); myCancelStatus = cancelStatus; - myCompilationStartStamp = System.currentTimeMillis(); myScope = scope; myDelegateMessageHandler = delegateMessageHandler; } - + // todo: add timestamp-setting code @Override - public long getCompilationStartStamp() { - return myCompilationStartStamp; + public long getCompilationStartStamp(BuildTarget target) { + synchronized (myCompilationStartStamp) { + return myCompilationStartStamp.get(target); + } } - @Override - public void updateCompilationStartStamp() { - myCompilationStartStamp = System.currentTimeMillis(); + public void setCompilationStartStamp(Collection> targets, long stamp) { + synchronized (myCompilationStartStamp) { + for (BuildTarget target : targets) { + myCompilationStartStamp.put(target, stamp); + } + } } @Override diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java index f20963439f49..29d2bdedc237 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java @@ -82,7 +82,7 @@ public class FSOperations { final JavaSourceRootDescriptor rd = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file); if (rd != null) { final ProjectDescriptor pd = context.getProjectDescriptor(); - pd.fsState.markDirty(context, round, file, rd, pd.timestamps.getStorage(), false); + pd.fsState.markDirty(context, round, file, rd, pd.timestamps.getStorage()); } } @@ -245,7 +245,7 @@ public class FSOperations { // if it is full project rebuild, all storages are already completely cleared; // so passing null because there is no need to access the storage to clear non-existing data final Timestamps marker = context.isProjectRebuild() ? null : tsStorage; - context.getProjectDescriptor().fsState.markDirty(context, round, file, rd, marker, false); + context.getProjectDescriptor().fsState.markDirty(context, round, file, rd, marker); } if (currentFiles != null) { currentFiles.add(file); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 1301bf530486..1e074f717461 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -47,6 +47,7 @@ import org.jetbrains.jps.cmdline.BuildRunner; import org.jetbrains.jps.cmdline.ProjectDescriptor; import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.fs.CompilationRound; +import org.jetbrains.jps.incremental.fs.FilesDelta; import org.jetbrains.jps.incremental.messages.*; import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration; import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping; @@ -60,9 +61,7 @@ import org.jetbrains.jps.model.module.JpsModule; import org.jetbrains.jps.service.SharedThreadPool; import org.jetbrains.jps.util.JpsPathUtil; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; @@ -82,7 +81,7 @@ public class IncProjectBuilder { private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.IncProjectBuilder"); private static final String CLASSPATH_INDEX_FILE_NAME = "classpath.index"; - private static final boolean GENERATE_CLASSPATH_INDEX = Boolean.parseBoolean(System.getProperty(GlobalOptions.GENERATE_CLASSPATH_INDEX_OPTION, "false")); + //private static final boolean GENERATE_CLASSPATH_INDEX = Boolean.parseBoolean(System.getProperty(GlobalOptions.GENERATE_CLASSPATH_INDEX_OPTION, "false")); private static final boolean SYNC_DELETE = Boolean.parseBoolean(System.getProperty("jps.sync.delete", SystemInfo.isWindows ? "true" : "false")); private static final GlobalContextKey>> TARGET_WITH_CLEARED_OUTPUT = GlobalContextKey.create("_targets_with_cleared_output_"); public static final int MAX_BUILDER_THREADS; @@ -143,10 +142,10 @@ public class IncProjectBuilder { for (BuildTarget target : myProjectDescriptor.getBuildTargetIndex().getAllTargets()) { if (scope.isAffected(target)) { BuildOperations.ensureFSStateInitialized(context, target); - final Map> toRecompile = fsState.getSourcesToRecompile(context, target); - //noinspection SynchronizationOnLocalVariableOrMethodParameter - synchronized (toRecompile) { - for (Set files : toRecompile.values()) { + final FilesDelta delta = fsState.getEffectiveFilesDelta(context, target); + delta.lockData(); + try { + for (Set files : delta.getSourcesToRecompile().values()) { for (File file : files) { if (scope.isAffected(target, file)) { // this will serve as a marker that compiler has work to do @@ -156,6 +155,9 @@ public class IncProjectBuilder { } } } + finally { + delta.unlockData(); + } } } } @@ -663,7 +665,6 @@ public class IncProjectBuilder { buildChunkIfAffected(context, scope, chunk); } finally { - context.updateCompilationStartStamp(); pd.dataManager.closeSourceToOutputStorages(Collections.singleton(chunk)); pd.dataManager.flush(true); } @@ -810,7 +811,6 @@ public class IncProjectBuilder { } } finally { - myContext.updateCompilationStartStamp(); myProjectDescriptor.dataManager.closeSourceToOutputStorages(Collections.singletonList(task.getChunk())); myProjectDescriptor.dataManager.flush(true); } @@ -929,8 +929,11 @@ public class IncProjectBuilder { } private void buildTargetsChunk(CompileContext context, final BuildTargetChunk chunk) throws ProjectBuildException { + final BuildFSState fsState = myProjectDescriptor.fsState; boolean doneSomething; try { + context.setCompilationStartStamp(chunk.getTargets(), System.currentTimeMillis()); + sendBuildingTargetMessages(chunk.getTargets(), BuildingTargetProgressMessage.Event.STARTED); Utils.ERRORS_DETECTED_KEY.set(context, Boolean.FALSE); @@ -940,11 +943,14 @@ public class IncProjectBuilder { doneSomething = processDeletedPaths(context, chunk.getTargets()); - myProjectDescriptor.fsState.beforeChunkBuildStart(context, chunk); + fsState.beforeChunkBuildStart(context, chunk); doneSomething |= runBuildersForChunk(context, chunk); - onChunkBuildComplete(context, chunk); + fsState.clearContextRoundData(context); + fsState.clearContextChunk(context); + + BuildOperations.markTargetsUpToDate(context, chunk); //if (doneSomething && GENERATE_CLASSPATH_INDEX) { // myAsyncTasks.add(SharedThreadPool.getInstance().executeOnPooledThread(new Runnable() { @@ -983,7 +989,7 @@ public class IncProjectBuilder { final Collection paths = entry.getValue(); if (paths != null) { for (String path : paths) { - myProjectDescriptor.fsState.registerDeleted(target, new File(path), null); + fsState.registerDeleted(target, new File(path), null); } } } @@ -1004,40 +1010,40 @@ public class IncProjectBuilder { myMessageDispatcher.processMessage(new BuildingTargetProgressMessage(targets, event)); } - private static void createClasspathIndex(final BuildTargetChunk chunk) { - final Set outputDirs = new THashSet(FileUtil.FILE_HASHING_STRATEGY); - for (BuildTarget target : chunk.getTargets()) { - if (target instanceof ModuleBuildTarget) { - File outputDir = ((ModuleBuildTarget)target).getOutputDir(); - if (outputDir != null && outputDirs.add(outputDir)) { - try { - BufferedWriter writer = new BufferedWriter(new FileWriter(new File(outputDir, CLASSPATH_INDEX_FILE_NAME))); - try { - writeIndex(writer, outputDir, ""); - } - finally { - writer.close(); - } - } - catch (IOException e) { - // Ignore. Failed to create optional classpath index - } - } - } - } - } + //private static void createClasspathIndex(final BuildTargetChunk chunk) { + // final Set outputDirs = new THashSet(FileUtil.FILE_HASHING_STRATEGY); + // for (BuildTarget target : chunk.getTargets()) { + // if (target instanceof ModuleBuildTarget) { + // File outputDir = ((ModuleBuildTarget)target).getOutputDir(); + // if (outputDir != null && outputDirs.add(outputDir)) { + // try { + // BufferedWriter writer = new BufferedWriter(new FileWriter(new File(outputDir, CLASSPATH_INDEX_FILE_NAME))); + // try { + // writeIndex(writer, outputDir, ""); + // } + // finally { + // writer.close(); + // } + // } + // catch (IOException e) { + // // Ignore. Failed to create optional classpath index + // } + // } + // } + // } + //} - private static void writeIndex(final BufferedWriter writer, final File file, final String path) throws IOException { - writer.write(path); - writer.write('\n'); - final File[] files = file.listFiles(); - if (files != null) { - for (File child : files) { - final String _path = path.isEmpty() ? child.getName() : path + "/" + child.getName(); - writeIndex(writer, child, _path); - } - } - } + //private static void writeIndex(final BufferedWriter writer, final File file, final String path) throws IOException { + // writer.write(path); + // writer.write('\n'); + // final File[] files = file.listFiles(); + // if (files != null) { + // for (File child : files) { + // final String _path = path.isEmpty() ? child.getName() : path + "/" + child.getName(); + // writeIndex(writer, child, _path); + // } + // } + //} private boolean processDeletedPaths(CompileContext context, final Set> targets) throws ProjectBuildException { @@ -1286,15 +1292,6 @@ public class IncProjectBuilder { } } - private static void onChunkBuildComplete(CompileContext context, @NotNull BuildTargetChunk chunk) throws IOException { - final ProjectDescriptor pd = context.getProjectDescriptor(); - final BuildFSState fsState = pd.fsState; - fsState.clearContextRoundData(context); - fsState.clearContextChunk(context); - - BuildOperations.markTargetsUpToDate(context, chunk); - } - private static CompileContext createContextWrapper(final CompileContext delegate) { final ClassLoader loader = delegate.getClass().getClassLoader(); final UserDataHolderBase localDataHolder = new UserDataHolderBase(); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java index 04aa6fc1b300..21b193f13786 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java @@ -18,10 +18,8 @@ package org.jetbrains.jps.incremental.fs; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Key; import com.intellij.openapi.util.io.FileSystemUtil; -import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.containers.MultiMap; import com.intellij.util.io.IOUtil; -import gnu.trove.TObjectLongHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.ModuleChunk; @@ -52,7 +50,6 @@ public class BuildFSState { // alternatively, when false, after first scan will rely on external notifications about changes private final boolean myAlwaysScanFS; private final Set> myInitialScanPerformed = Collections.synchronizedSet(new HashSet>()); - private final TObjectLongHashMap myRegistrationStamps = new TObjectLongHashMap(FileUtil.FILE_HASHING_STRATEGY); private final Map, FilesDelta> myDeltas = Collections.synchronizedMap(new HashMap, FilesDelta>()); public BuildFSState(boolean alwaysScanFS) { @@ -107,10 +104,6 @@ public class BuildFSState { getDelta(rd.getTarget()).clearRecompile(rd); } - public long getEventRegistrationStamp(File file) { - return myRegistrationStamps.get(file); - } - public boolean hasWorkToDo(BuildTarget target) { if (!myInitialScanPerformed.contains(target)) { return true; @@ -166,15 +159,16 @@ public class BuildFSState { return !myAlwaysScanFS && myInitialScanPerformed.contains(target); } - public Map> getSourcesToRecompile(@NotNull CompileContext context, BuildTarget target) { + @NotNull + public FilesDelta getEffectiveFilesDelta(@NotNull CompileContext context, BuildTarget target) { if (target instanceof ModuleBuildTarget) { // multiple compilation rounds are applicable to ModuleBuildTarget only final FilesDelta lastRoundDelta = getRoundDelta(CURRENT_ROUND_DELTA_KEY, context); if (lastRoundDelta != null) { - return lastRoundDelta.getSourcesToRecompile(); + return lastRoundDelta; } } - return getDelta(target).getSourcesToRecompile(); + return getDelta(target); } public boolean isMarkedForRecompilation(@Nullable CompileContext context, CompilationRound round, BuildRootDescriptor rd, File file) { @@ -183,46 +177,45 @@ public class BuildFSState { delta = getDelta(rd.getTarget()); } - final Map> recompile = delta.getSourcesToRecompile(); - //noinspection SynchronizationOnLocalVariableOrMethodParameter - synchronized (recompile) { - final Set files = recompile.get(rd); - return files != null && files.contains(file); - } + return delta.isMarkedRecompile(rd, file); } /** * Note: marked file will well be visible as "dirty" only on the next compilation round! * @throws IOException */ - public final boolean markDirty(@Nullable CompileContext context, File file, final BuildRootDescriptor rd, @Nullable Timestamps tsStorage, boolean saveEventStamp) throws IOException { - return markDirty(context, CompilationRound.NEXT, file, rd, tsStorage, saveEventStamp); + public final boolean markDirty(@Nullable CompileContext context, File file, final BuildRootDescriptor rd, @Nullable Timestamps tsStorage) throws IOException { + return markDirty(context, CompilationRound.NEXT, file, rd, tsStorage); } - public boolean markDirty(@Nullable CompileContext context, CompilationRound round, File file, final BuildRootDescriptor rd, @Nullable Timestamps tsStorage, boolean saveEventStamp) throws IOException { + public boolean markDirty(@Nullable CompileContext context, CompilationRound round, File file, final BuildRootDescriptor rd, @Nullable Timestamps tsStorage) throws IOException { final FilesDelta roundDelta = getRoundDelta(round == CompilationRound.NEXT? NEXT_ROUND_DELTA_KEY : CURRENT_ROUND_DELTA_KEY, context); if (roundDelta != null && isInCurrentContextTargets(context, rd)) { roundDelta.markRecompile(rd, file); } - final boolean marked = getDelta(rd.getTarget()).markRecompile(rd, file); - if (marked) { - if (LOG.isDebugEnabled()) { - LOG.debug(rd.getTarget() + ": MARKED DIRTY: " + file.getPath()); + final FilesDelta filesDelta = getDelta(rd.getTarget()); + filesDelta.lockData(); + try { + final boolean marked = filesDelta.markRecompile(rd, file); + if (marked) { + if (LOG.isDebugEnabled()) { + LOG.debug(rd.getTarget() + ": MARKED DIRTY: " + file.getPath()); + } + if (tsStorage != null) { + tsStorage.removeStamp(file, rd.getTarget()); + } } - if (saveEventStamp) { - myRegistrationStamps.put(file, System.currentTimeMillis()); - } - if (tsStorage != null) { - tsStorage.removeStamp(file, rd.getTarget()); + else { + if (LOG.isDebugEnabled()) { + LOG.debug(rd.getTarget() + ": NOT MARKED DIRTY: " + file.getPath()); + } } + return marked; } - else { - if (LOG.isDebugEnabled()) { - LOG.debug(rd.getTarget() + ": NOT MARKED DIRTY: " + file.getPath()); - } + finally { + filesDelta.unlockData(); } - return marked; } private static boolean isInCurrentContextTargets(CompileContext context, BuildRootDescriptor rd) { @@ -254,7 +247,6 @@ public class BuildFSState { clearContextChunk(null); myInitialScanPerformed.clear(); myDeltas.clear(); - myRegistrationStamps.clear(); } public void clearContextRoundData(@Nullable CompileContext context) { @@ -286,10 +278,11 @@ public class BuildFSState { } public > boolean processFilesToRecompile(CompileContext context, final @NotNull T target, final FileProcessor processor) throws IOException { - final Map> data = getSourcesToRecompile(context, target); final CompileScope scope = context.getScope(); - synchronized (data) { - for (Map.Entry> entry : data.entrySet()) { + final FilesDelta delta = getEffectiveFilesDelta(context, target); + delta.lockData(); + try { + for (Map.Entry> entry : delta.getSourcesToRecompile().entrySet()) { //noinspection unchecked R root = (R)entry.getKey(); if (!target.equals(root.getTarget())) { @@ -305,8 +298,11 @@ public class BuildFSState { } } } + return true; + } + finally { + delta.unlockData(); } - return true; } /** @@ -314,36 +310,44 @@ public class BuildFSState { */ public boolean markAllUpToDate(CompileContext context, final BuildRootDescriptor rd, final Timestamps stamps) throws IOException { boolean marked = false; - final FilesDelta delta = getDelta(rd.getTarget()); - final Set files = delta.clearRecompile(rd); - if (files != null) { - CompileScope scope = context.getScope(); - final long compilationStartStamp = context.getCompilationStartStamp(); - for (File file : files) { - if (scope.isAffected(rd.getTarget(), file)) { - final long currentFileStamp = FileSystemUtil.lastModified(file); - if (!rd.isGenerated() && (currentFileStamp > compilationStartStamp || getEventRegistrationStamp(file) > compilationStartStamp)) { - // if the file was modified after the compilation had started, - // do not save the stamp considering file dirty + final BuildTarget target = rd.getTarget(); + final FilesDelta delta = getDelta(target); + final long targetBuildStartStamp = context.getCompilationStartStamp(target); + // prevent modifications to the data structure from external FS events + delta.lockData(); + try { + final Set files = delta.clearRecompile(rd); + if (files != null) { + CompileScope scope = context.getScope(); + for (File file : files) { + if (scope.isAffected(target, file)) { + final long currentFileStamp = FileSystemUtil.lastModified(file); + if (!rd.isGenerated() && currentFileStamp > targetBuildStartStamp) { + // if the file was modified after the compilation had started, + // do not save the stamp considering file dirty + if (Utils.IS_TEST_MODE) { + LOG.info("Timestamp after compilation started; marking dirty again: " + file.getPath()); + } + delta.markRecompile(rd, file); + } + else { + marked = true; + stamps.saveStamp(file, target, currentFileStamp); + } + } + else { if (Utils.IS_TEST_MODE) { - LOG.info("Timestamp after compilation started; marking dirty again: " + file.getPath()); + LOG.info("Not affected by compile scope; marking dirty again: " + file.getPath()); } delta.markRecompile(rd, file); } - else { - marked = true; - stamps.saveStamp(file, rd.getTarget(), currentFileStamp); - } - } - else { - if (Utils.IS_TEST_MODE) { - LOG.info("Not affected by compile scope; marking dirty again: " + file.getPath()); - } - delta.markRecompile(rd, file); } } + return marked; + } + finally { + delta.unlockData(); } - return marked; } private static void setContextTargets(@Nullable CompileContext context, @Nullable Set> targets) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java index 5c08754bf439..bcd48f67a834 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java @@ -31,32 +31,50 @@ import java.io.DataOutput; import java.io.File; import java.io.IOException; import java.util.*; +import java.util.concurrent.locks.ReentrantLock; /** @noinspection SynchronizationOnLocalVariableOrMethodParameter*/ -final class FilesDelta { +public final class FilesDelta { private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.fs.FilesDelta"); + private final ReentrantLock myDataLock = new ReentrantLock(); - private final Set myDeletedPaths = Collections.synchronizedSet(new THashSet(FileUtil.PATH_HASHING_STRATEGY)); - private final Map> myFilesToRecompile = Collections.synchronizedMap(new HashMap>()); + private final Set myDeletedPaths = new THashSet(FileUtil.PATH_HASHING_STRATEGY); + private final Map> myFilesToRecompile = new HashMap>(); + + public void lockData(){ + myDataLock.lock(); + } + + public void unlockData(){ + myDataLock.unlock(); + } protected void addAll(FilesDelta other) { - myDeletedPaths.addAll(other.myDeletedPaths); - final Map> dataToAdd = other.myFilesToRecompile; - synchronized (dataToAdd) { - for (Map.Entry> entry : dataToAdd.entrySet()) { - _addToRecompiled(entry.getKey(), entry.getValue()); + lockData(); + try { + other.lockData(); + try { + myDeletedPaths.addAll(other.myDeletedPaths); + for (Map.Entry> entry : other.myFilesToRecompile.entrySet()) { + _addToRecompiled(entry.getKey(), entry.getValue()); + } } + finally { + other.unlockData(); + } + } + finally { + unlockData(); } } public void save(DataOutput out) throws IOException { - out.writeInt(myDeletedPaths.size()); - synchronized (myDeletedPaths) { + lockData(); + try { + out.writeInt(myDeletedPaths.size()); for (String path : myDeletedPaths) { IOUtil.writeString(path, out); } - } - synchronized (myFilesToRecompile) { out.writeInt(myFilesToRecompile.size()); for (Map.Entry> entry : myFilesToRecompile.entrySet()) { IOUtil.writeString(entry.getKey().getRootId(), out); @@ -67,39 +85,48 @@ final class FilesDelta { } } } + finally { + unlockData(); + } } public void load(DataInput in, @NotNull BuildTarget target, BuildRootIndex buildRootIndex) throws IOException { - myDeletedPaths.clear(); - int deletedCount = in.readInt(); - while (deletedCount-- > 0) { - myDeletedPaths.add(IOUtil.readString(in)); - } - myFilesToRecompile.clear(); - int recompileCount = in.readInt(); - while (recompileCount-- > 0) { - String rootId = IOUtil.readString(in); - BuildRootDescriptor descriptor = target.findRootDescriptor(rootId, buildRootIndex); - Set files; - if (descriptor != null) { - files = myFilesToRecompile.get(descriptor); - if (files == null) { + lockData(); + try { + myDeletedPaths.clear(); + int deletedCount = in.readInt(); + while (deletedCount-- > 0) { + myDeletedPaths.add(IOUtil.readString(in)); + } + myFilesToRecompile.clear(); + int recompileCount = in.readInt(); + while (recompileCount-- > 0) { + String rootId = IOUtil.readString(in); + BuildRootDescriptor descriptor = target.findRootDescriptor(rootId, buildRootIndex); + Set files; + if (descriptor != null) { + files = myFilesToRecompile.get(descriptor); + if (files == null) { + files = new THashSet(FileUtil.FILE_HASHING_STRATEGY); + myFilesToRecompile.put(descriptor, files); + } + } + else { + LOG.debug("Cannot find root by " + rootId + ", delta will be skipped"); files = new THashSet(FileUtil.FILE_HASHING_STRATEGY); - myFilesToRecompile.put(descriptor, files); + } + int filesCount = in.readInt(); + while (filesCount-- > 0) { + final File file = new File(IOUtil.readString(in)); + if (Utils.IS_TEST_MODE) { + LOG.info("Loaded " + file.getPath()); + } + files.add(file); } } - else { - LOG.debug("Cannot find root by " + rootId + ", delta will be skipped"); - files = new THashSet(FileUtil.FILE_HASHING_STRATEGY); - } - int filesCount = in.readInt(); - while (filesCount-- > 0) { - final File file = new File(IOUtil.readString(in)); - if (Utils.IS_TEST_MODE) { - LOG.info("Loaded " + file.getPath()); - } - files.add(file); - } + } + finally { + unlockData(); } } @@ -119,32 +146,55 @@ final class FilesDelta { } public boolean hasChanges() { - return hasPathsToDelete() || hasSourcesToRecompile(); + lockData(); + try { + if (!myDeletedPaths.isEmpty()) { + return true; + } + if(!myFilesToRecompile.isEmpty()) { + for (Set files : myFilesToRecompile.values()) { + if (!files.isEmpty()) { + return true; + } + } + } + return false; + } + finally { + unlockData(); + } } public boolean markRecompile(BuildRootDescriptor root, File file) { - final boolean added = _addToRecompiled(root, file); - if (added) { - synchronized (myDeletedPaths) { + lockData(); + try { + final boolean added = _addToRecompiled(root, file); + if (added) { if (!myDeletedPaths.isEmpty()) { // optimization myDeletedPaths.remove(FileUtil.toCanonicalPath(file.getPath())); } } + return added; + } + finally { + unlockData(); } - return added; } public boolean markRecompileIfNotDeleted(BuildRootDescriptor root, File file) { - final boolean isMarkedDeleted; - synchronized (myDeletedPaths) { - isMarkedDeleted = !myDeletedPaths.isEmpty() && myDeletedPaths.contains(FileUtil.toCanonicalPath(file.getPath())); + lockData(); + try { + final boolean isMarkedDeleted = !myDeletedPaths.isEmpty() && myDeletedPaths.contains(FileUtil.toCanonicalPath(file.getPath())); + if (!isMarkedDeleted) { + _addToRecompiled(root, file); + return true; + } + return false; } - if (!isMarkedDeleted) { - _addToRecompiled(root, file); - return true; + finally { + unlockData(); } - return false; } private boolean _addToRecompiled(BuildRootDescriptor root, File file) { @@ -155,36 +205,45 @@ final class FilesDelta { } private boolean _addToRecompiled(BuildRootDescriptor root, Collection filesToAdd) { - synchronized (myFilesToRecompile) { - Set files = myFilesToRecompile.get(root); - if (files == null) { - files = new THashSet(FileUtil.FILE_HASHING_STRATEGY); - myFilesToRecompile.put(root, files); - } - return files.addAll(filesToAdd); + Set files = myFilesToRecompile.get(root); + if (files == null) { + files = new THashSet(FileUtil.FILE_HASHING_STRATEGY); + myFilesToRecompile.put(root, files); } + return files.addAll(filesToAdd); } public void addDeleted(File file) { - // ensure the file is no more marked to recompilation - synchronized (myFilesToRecompile) { + final String path = FileUtil.toCanonicalPath(file.getPath()); + lockData(); + try { + // ensure the file is not marked to recompilation anymore for (Set files : myFilesToRecompile.values()) { files.remove(file); } + myDeletedPaths.add(path); + if (Utils.IS_TEST_MODE) { + LOG.info("Marking deleted: " + path); + } } - final String path = FileUtil.toCanonicalPath(file.getPath()); - myDeletedPaths.add(path); - if (Utils.IS_TEST_MODE) { - LOG.info("Marking deleted: " + path); + finally { + unlockData(); } } public void clearDeletedPaths() { - myDeletedPaths.clear(); + lockData(); + try { + myDeletedPaths.clear(); + } + finally { + unlockData(); + } } public Set getAndClearDeletedPaths() { - synchronized (myDeletedPaths) { + lockData(); + try { try { final THashSet _paths = new THashSet(FileUtil.PATH_HASHING_STRATEGY); _paths.addAll(myDeletedPaths); @@ -194,31 +253,36 @@ final class FilesDelta { myDeletedPaths.clear(); } } + finally { + unlockData(); + } } + @NotNull public Map> getSourcesToRecompile() { + LOG.assertTrue(myDataLock.isHeldByCurrentThread(), "FilesDelta data must be locked by querying thread"); return myFilesToRecompile; } - private boolean hasSourcesToRecompile() { - synchronized (myFilesToRecompile) { - if(!myFilesToRecompile.isEmpty()) { - for (Set files : myFilesToRecompile.values()) { - if (!files.isEmpty()) { - return true; - } - } - } + public boolean isMarkedRecompile(BuildRootDescriptor rd, File file) { + lockData(); + try { + final Set files = myFilesToRecompile.get(rd); + return files != null && files.contains(file); + } + finally { + unlockData(); } - return false; - } - - private boolean hasPathsToDelete() { - return !myDeletedPaths.isEmpty(); } @Nullable public Set clearRecompile(BuildRootDescriptor root) { - return myFilesToRecompile.remove(root); + lockData(); + try { + return myFilesToRecompile.remove(root); + } + finally { + unlockData(); + } } }