diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactCompilerPersistentData.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactCompilerPersistentData.java index 1d9597f7306f..636de74b2aac 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactCompilerPersistentData.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactCompilerPersistentData.java @@ -31,7 +31,7 @@ import java.util.Set; */ public class ArtifactCompilerPersistentData { private static final Logger LOG = Logger.getInstance("#com.intellij.compiler.impl.generic.ArtifactCompilerPersistentData"); - private static final int VERSION = 0; + private static final int VERSION = 1; private File myFile; private Map myArtifact2Id = new HashMap(); private TIntHashSet myUsedIds = new TIntHashSet(); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactOutputToSourceMapping.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactOutputToSourceMapping.java new file mode 100644 index 000000000000..1370728b1800 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactOutputToSourceMapping.java @@ -0,0 +1,65 @@ +package org.jetbrains.jps.incremental.artifacts; + +import com.intellij.util.SmartList; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.IOUtil; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.jps.incremental.storage.AbstractStateStorage; + +import java.io.*; +import java.util.List; + +/** + * Stores source paths for each output path. If a source file or an output file is located in a jar file the full path is stored + * using '!/' to separate path to the jar file from path to file inside the jar. + * + * @author nik + */ +public class ArtifactOutputToSourceMapping extends AbstractStateStorage> { + public static DataExternalizer> EXTERNALIZER = new DataExternalizer>() { + private final byte[] myBuffer = IOUtil.allocReadWriteUTFBuffer(); + + @Override + public void save(DataOutput out, List value) throws IOException { + for (SourcePathAndRootIndex pair : value) { + IOUtil.writeUTFFast(myBuffer, out, pair.myPath); + out.writeInt(pair.getRootIndex()); + } + } + + @Override + public List read(DataInput in) throws IOException { + List result = new SmartList(); + final DataInputStream stream = (DataInputStream)in; + while (stream.available() > 0) { + final String path = IOUtil.readUTFFast(myBuffer, stream); + final int index = stream.readInt(); + result.add(new SourcePathAndRootIndex(path, index)); + } + return result; + } + }; + + public ArtifactOutputToSourceMapping(@NonNls File storePath) throws IOException { + super(storePath, new EnumeratorStringDescriptor(), EXTERNALIZER); + } + + public static class SourcePathAndRootIndex { + private final String myPath; + private final int myRootIndex; + + public SourcePathAndRootIndex(String path, int rootIndex) { + myPath = path; + myRootIndex = rootIndex; + } + + public String getPath() { + return myPath; + } + + public int getRootIndex() { + return myRootIndex; + } + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceFilesState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceFilesState.java index 71d7c03bf125..205bf885b3de 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceFilesState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceFilesState.java @@ -2,6 +2,7 @@ package org.jetbrains.jps.incremental.artifacts; import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.ArrayUtil; +import com.intellij.util.containers.IntArrayList; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.Project; import org.jetbrains.jps.ProjectPaths; @@ -13,11 +14,7 @@ import org.jetbrains.jps.incremental.artifacts.instructions.*; import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.*; /** * @author nik @@ -28,12 +25,13 @@ public class ArtifactSourceFilesState { private final int myArtifactId; private final ModuleRootsIndex myRootsIndex; private final ArtifactSourceTimestampStorage myTimestampStorage; - private Set myChangedFiles = new HashSet(); + private Map myChangedFiles = new HashMap(); private Set myDeletedFiles = new HashSet(); private ArtifactInstructionsBuilder myInstructionsBuilder; - private ArtifactSourceToOutputMapping myMapping; - private final AtomicBoolean myInitialized = new AtomicBoolean(); - private final File myMappingsFile; + private ArtifactSourceToOutputMapping mySrcOutMapping; + private ArtifactOutputToSourceMapping myOutSrcMapping; + private final File mySrcOutMappingsFile; + private File myOutSrcMappingsFile; public ArtifactSourceFilesState(Artifact artifact, int artifactId, Project project, ModuleRootsIndex rootsIndex, @@ -44,23 +42,34 @@ public class ArtifactSourceFilesState { myRootsIndex = rootsIndex; myTimestampStorage = timestampStorage; myArtifactId = artifactId; - myMappingsFile = new File(new File(mappingsDir, String.valueOf(artifactId)), "src-out"); + mySrcOutMappingsFile = new File(new File(mappingsDir, String.valueOf(artifactId)), "src-out"); + myOutSrcMappingsFile = new File(new File(mappingsDir, String.valueOf(artifactId)), "out-src"); } - public ArtifactSourceToOutputMapping getOrCreateMapping() throws IOException { - if (myMapping == null) { - myMapping = new ArtifactSourceToOutputMapping(myMappingsFile); + public ArtifactSourceToOutputMapping getOrCreateSrcOutMapping() throws IOException { + if (mySrcOutMapping == null) { + mySrcOutMapping = new ArtifactSourceToOutputMapping(mySrcOutMappingsFile); } - return myMapping; + return mySrcOutMapping; + } + + public ArtifactOutputToSourceMapping getOrCreateOutSrcMapping() throws IOException { + if (myOutSrcMapping == null) { + myOutSrcMapping = new ArtifactOutputToSourceMapping(myOutSrcMappingsFile); + } + return myOutSrcMapping; } public void clean() { - if (myMapping != null) { - myMapping.wipe(); + if (mySrcOutMapping != null) { + mySrcOutMapping.wipe(); + } + if (myOutSrcMapping != null) { + myOutSrcMapping.wipe(); } } - public Set getChangedFiles() { + public Map getChangedFiles() { return myChangedFiles; } @@ -69,25 +78,19 @@ public class ArtifactSourceFilesState { } public void initState() throws IOException { - /* - if (!myInitialized.compareAndSet(false, true)) { - return; - } - */ - final Set currentPaths = new HashSet(); myChangedFiles.clear(); myDeletedFiles.clear(); getOrCreateInstructions().processRoots(new ArtifactRootProcessor() { @Override - public void process(ArtifactSourceRoot root, Collection destinations) throws IOException { + public void process(ArtifactSourceRoot root, int rootIndex, Collection destinations) throws IOException { final File rootFile = root.getRootFile(); if (rootFile.exists()) { - processRecursively(rootFile, root.getFilter(), currentPaths); + processRecursively(rootFile, rootIndex, root.getFilter(), currentPaths); } } }); - final ArtifactSourceToOutputMapping mapping = getOrCreateMapping(); + final ArtifactSourceToOutputMapping mapping = getOrCreateSrcOutMapping(); final Iterator iterator = mapping.getKeysIterator(); while (iterator.hasNext()) { String path = iterator.next(); @@ -97,7 +100,7 @@ public class ArtifactSourceFilesState { } } - private void processRecursively(File file, SourceFileFilter filter, Set currentPaths) throws IOException { + private void processRecursively(File file, int rootIndex, SourceFileFilter filter, Set currentPaths) throws IOException { final String filePath = FileUtil.toSystemIndependentName(FileUtil.toCanonicalPath(file.getPath())); if (!filter.accept(filePath)) return; @@ -105,7 +108,7 @@ public class ArtifactSourceFilesState { final File[] children = file.listFiles(); if (children != null) { for (File child : children) { - processRecursively(child, filter, currentPaths); + processRecursively(child, rootIndex, filter, currentPaths); } } } @@ -123,7 +126,12 @@ public class ArtifactSourceFilesState { } if (!upToDate) { myDeletedFiles.remove(filePath); - myChangedFiles.add(filePath); + IntArrayList list = myChangedFiles.get(filePath); + if (list == null) { + list = new IntArrayList(1); + myChangedFiles.put(filePath, list); + } + list.add(rootIndex); } } } @@ -156,7 +164,7 @@ public class ArtifactSourceFilesState { } } } - for (String filePath : myChangedFiles) { + for (String filePath : myChangedFiles.keySet()) { final ArtifactSourceTimestampStorage.PerArtifactTimestamp[] state = myTimestampStorage.getState(filePath); File file = new File(FileUtil.toSystemDependentName(filePath)); final long timestamp = file.lastModified(); @@ -185,14 +193,20 @@ public class ArtifactSourceFilesState { } public void close() throws IOException { - if (myMapping != null) { - myMapping.close(); + if (mySrcOutMapping != null) { + mySrcOutMapping.close(); + } + if (myOutSrcMapping != null) { + myOutSrcMapping.close(); } } public void flush(boolean memoryCachesOnly) { - if (myMapping != null) { - myMapping.flush(memoryCachesOnly); + if (mySrcOutMapping != null) { + mySrcOutMapping.flush(memoryCachesOnly); + } + if (myOutSrcMapping != null) { + myOutSrcMapping.flush(memoryCachesOnly); } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceToOutputMapping.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceToOutputMapping.java index 8919fa3b79c8..2ebb45278445 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceToOutputMapping.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactSourceToOutputMapping.java @@ -1,47 +1,57 @@ package org.jetbrains.jps.incremental.artifacts; +import com.intellij.util.SmartList; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.EnumeratorStringDescriptor; import com.intellij.util.io.IOUtil; import org.jetbrains.annotations.NonNls; import org.jetbrains.jps.incremental.storage.AbstractStateStorage; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.File; -import java.io.IOException; +import java.io.*; +import java.util.List; /** - * Stores output paths for each source file path. If a source file is located inside a jar the path to the jar file is stored. - * If an output file is located in a jar file the path to the jar file with '!/' suffix is stored. + * Stores output paths for each source file path. If a source file or an output file is located inside a jar the path to the jar file is stored. + * //If an output file is located in a jar file the full path is stored using '!/' to separate path to the jar file from path to file inside the jar. * * @author nik */ -public class ArtifactSourceToOutputMapping extends AbstractStateStorage { - private static DataExternalizer STRING_ARRAY_EXTERNALIZER = new DataExternalizer() { +public class ArtifactSourceToOutputMapping extends AbstractStateStorage> { + private static DataExternalizer> STRING_LIST_EXTERNALIZER = new DataExternalizer>() { private final byte[] myBuffer = IOUtil.allocReadWriteUTFBuffer(); @Override - public void save(DataOutput out, String[] value) throws IOException { - out.writeInt(value.length); + public void save(DataOutput out, List value) throws IOException { for (String path : value) { IOUtil.writeUTFFast(myBuffer, out, path); } } @Override - public String[] read(DataInput in) throws IOException { - final int size = in.readInt(); - String[] result = new String[size]; - for (int i = 0; i < size; i++) { - final String path = IOUtil.readUTFFast(myBuffer, in); - result[i] = path; + public List read(DataInput in) throws IOException { + List result = new SmartList(); + final DataInputStream stream = (DataInputStream)in; + while (stream.available() > 0) { + result.add(IOUtil.readUTFFast(myBuffer, stream)); } return result; } }; public ArtifactSourceToOutputMapping(@NonNls File storePath) throws IOException { - super(storePath, new EnumeratorStringDescriptor(), STRING_ARRAY_EXTERNALIZER); + super(storePath, new EnumeratorStringDescriptor(), STRING_LIST_EXTERNALIZER); + } + + public void removeValue(String sourcePath, String outputPath) throws IOException { + final List outputPaths = getState(sourcePath); + if (outputPaths != null) { + outputPaths.remove(outputPath); + if (outputPaths.isEmpty()) { + remove(sourcePath); + } + else { + update(sourcePath, outputPaths); + } + } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactsBuildData.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactsBuildData.java index 4215e52a266f..fdd726c5d0f4 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactsBuildData.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/ArtifactsBuildData.java @@ -1,5 +1,6 @@ package org.jetbrains.jps.incremental.artifacts; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.io.FileUtil; import org.jetbrains.jps.Project; import org.jetbrains.jps.artifacts.Artifact; @@ -14,6 +15,7 @@ import java.util.Map; * @author nik */ public class ArtifactsBuildData { + private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.artifacts.ArtifactsBuildData"); private Map myArtifactState; private final ArtifactSourceTimestampStorage myTimestampStorage; private ArtifactCompilerPersistentData myPersistentData; @@ -26,6 +28,11 @@ public class ArtifactsBuildData { myArtifactState = new HashMap(); myPersistentData = new ArtifactCompilerPersistentData(artifactsDataDir); myMappingsDir = new File(myArtifactsDataDir, "mappings"); + if (myPersistentData.isVersionChanged()) { + myTimestampStorage.wipe(); + FileUtil.delete(myMappingsDir); + //todo[nik] clear artifacts outputs + } } public ArtifactSourceFilesState getOrCreateState(Artifact artifact, Project project, ModuleRootsIndex index) { @@ -60,6 +67,7 @@ public class ArtifactsBuildData { } public void close() throws IOException { + myPersistentData.save(); myTimestampStorage.close(); for (ArtifactSourceFilesState state : myArtifactState.values()) { state.close(); @@ -67,6 +75,12 @@ public class ArtifactsBuildData { } public void flush(boolean memoryCachesOnly) { + try { + myPersistentData.save(); + } + catch (IOException e) { + LOG.info(e); + } myTimestampStorage.flush(memoryCachesOnly); for (ArtifactSourceFilesState state : myArtifactState.values()) { state.flush(memoryCachesOnly); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/IncArtifactBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/IncArtifactBuilder.java index f0776ee871af..c7b4a6d7563f 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/IncArtifactBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/IncArtifactBuilder.java @@ -2,10 +2,10 @@ package org.jetbrains.jps.incremental.artifacts; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.util.ArrayUtil; -import com.intellij.util.SmartList; -import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.containers.IntArrayList; +import com.intellij.util.containers.MultiMap; import gnu.trove.THashSet; +import gnu.trove.TIntObjectHashMap; import org.jetbrains.jps.artifacts.Artifact; import org.jetbrains.jps.incremental.CompileContext; import org.jetbrains.jps.incremental.ProjectBuildException; @@ -74,54 +74,92 @@ public class IncArtifactBuilder extends ProjectLevelBuilder { context.getProject(), context.getRootsIndex()); state.initState(); final Set deletedFiles = state.getDeletedFiles(); - final Set changedFiles = state.getChangedFiles(); + final Map changedFiles = state.getChangedFiles(); if (deletedFiles.isEmpty() && changedFiles.isEmpty()) { return; } context.processMessage(new ProgressMessage("Building artifact '" + artifact.getName() + "'...")); - final ArtifactSourceToOutputMapping mapping = state.getOrCreateMapping(); - final Set deletedJars = deleteOutdatedFiles(deletedFiles, context, mapping); - final ArtifactInstructionsBuilder instructions = state.getOrCreateInstructions(); - final Set changedJars = new THashSet(); - for (String deletedJar : deletedJars) { - ContainerUtil.addIfNotNull(instructions.getJarInfo(deletedJar), changedJars); + final ArtifactSourceToOutputMapping srcOutMapping = state.getOrCreateSrcOutMapping(); + final ArtifactOutputToSourceMapping outSrcMapping = state.getOrCreateOutSrcMapping(); + + final TIntObjectHashMap> filesToProcess = new TIntObjectHashMap>(); + MultiMap filesToDelete = new MultiMap(); + for (String sourcePath : deletedFiles) { + final List outputPaths = srcOutMapping.getState(sourcePath); + if (outputPaths != null) { + for (String outputPath : outputPaths) { + filesToDelete.putValue(outputPath, sourcePath); + final List sources = outSrcMapping.getState(outputPath); + if (sources != null) { + for (ArtifactOutputToSourceMapping.SourcePathAndRootIndex source : sources) { + addFileToProcess(filesToProcess, source.getRootIndex(), source.getPath()); + } + } + } + } } - Map updatedMappings = new HashMap(); - for (final String filePath : changedFiles) { - final List outputs = new SmartList(); - instructions.processContainingRoots(filePath, new ArtifactRootProcessor() { - @Override - public void process(ArtifactSourceRoot root, Collection destinations) throws IOException { + Set changedOutputPaths = new THashSet(); + for (Map.Entry entry : changedFiles.entrySet()) { + final IntArrayList roots = entry.getValue(); + final String sourcePath = entry.getKey(); + for (int i = 0; i < roots.size(); i++) { + addFileToProcess(filesToProcess, roots.get(i), sourcePath); + } + final List outputPaths = srcOutMapping.getState(sourcePath); + if (outputPaths != null) { + changedOutputPaths.addAll(outputPaths); + for (String outputPath : outputPaths) { + final List sources = outSrcMapping.getState(outputPath); + if (sources != null) { + for (ArtifactOutputToSourceMapping.SourcePathAndRootIndex source : sources) { + addFileToProcess(filesToProcess, source.getRootIndex(), source.getPath()); + } + } + } + } + } + for (String sourcePath : changedFiles.keySet()) { + srcOutMapping.remove(sourcePath); + } + for (String outputPath : changedOutputPaths) { + outSrcMapping.remove(outputPath); + } + + deleteOutdatedFiles(filesToDelete, context, srcOutMapping, outSrcMapping); + + final ArtifactInstructionsBuilder instructions = state.getOrCreateInstructions(); + final Set changedJars = new THashSet(); + instructions.processRoots(new ArtifactRootProcessor() { + @Override + public void process(ArtifactSourceRoot root, int rootIndex, Collection destinations) throws IOException { + final Set sourcePaths = filesToProcess.get(rootIndex); + if (sourcePaths == null) return; + + for (String sourcePath : sourcePaths) { + if (!root.containsFile(sourcePath)) continue;//todo[nik] this seems to be unnecessary + for (DestinationInfo destination : destinations) { if (destination instanceof ExplodedDestinationInfo) { - context.getLoggingManager().getArtifactBuilderLogger().fileCopied(filePath); - root.copyFromRoot(filePath, destination.getOutputPath(), outputs); + root.copyFromRoot(sourcePath, rootIndex, destination.getOutputPath(), context, srcOutMapping, outSrcMapping); } - else { - outputs.add(destination.getOutputFilePath() + JarPathUtil.JAR_SEPARATOR); + else if (outSrcMapping.getState(destination.getOutputFilePath()) == null) { + outSrcMapping.update(destination.getOutputFilePath(), Collections.emptyList()); changedJars.add(((JarDestinationInfo)destination).getJarInfo()); } } } - }); - updatedMappings.put(filePath, ArrayUtil.toStringArray(outputs)); - } + } + }); - JarsBuilder builder = new JarsBuilder(changedJars, null, context); - final boolean processed = builder.buildJars(new THashSet()); + JarsBuilder builder = new JarsBuilder(changedJars, context, srcOutMapping, outSrcMapping, instructions); + final boolean processed = builder.buildJars(); if (!processed) { return; } state.updateTimestamps(); - for (String filePath : deletedFiles) { - mapping.remove(filePath); - } - for (Map.Entry entry : updatedMappings.entrySet()) { - mapping.update(entry.getKey(), entry.getValue()); - } state.markUpToDate(); context.processMessage(UptoDateFilesSavedEvent.INSTANCE); } @@ -130,48 +168,45 @@ public class IncArtifactBuilder extends ProjectLevelBuilder { } } - private static Set deleteOutdatedFiles(Set deletedFiles, CompileContext context, - ArtifactSourceToOutputMapping mapping) throws IOException { - if (deletedFiles.isEmpty()) return Collections.emptySet(); + private static void addFileToProcess(TIntObjectHashMap> filesToProcess, final int rootIndex, final String path) { + Set paths = filesToProcess.get(rootIndex); + if (paths == null) { + paths = new THashSet(); + filesToProcess.put(rootIndex, paths); + } + paths.add(path); + } + + private static void deleteOutdatedFiles(MultiMap filesToDelete, CompileContext context, + ArtifactSourceToOutputMapping srcOutMapping, + ArtifactOutputToSourceMapping outSrcMapping) throws IOException { + if (filesToDelete.isEmpty()) return; context.processMessage(new ProgressMessage("Deleting outdated files...")); - Set pathsToDelete = new THashSet(); - for (String path : deletedFiles) { - final String[] outputPaths = mapping.getState(path); - Collections.addAll(pathsToDelete, outputPaths); - } - int notDeletedFilesCount = 0; - final THashSet notDeletedJars = new THashSet(); - final THashSet deletedJars = new THashSet(); + final THashSet notDeletedPaths = new THashSet(); + final THashSet deletedPaths = new THashSet(); - for (String fullPath : pathsToDelete) { - int end = fullPath.indexOf(JarPathUtil.JAR_SEPARATOR); - boolean isJar = end != -1; - String filePath = isJar ? fullPath.substring(0, end) : fullPath; - boolean deleted = false; - if (isJar) { - if (notDeletedJars.contains(filePath)) { - continue; - } - deleted = deletedJars.contains(filePath); + for (String filePath : filesToDelete.keySet()) { + if (notDeletedPaths.contains(filePath)) { + continue; } - File file = new File(FileUtil.toSystemDependentName(filePath)); + boolean deleted = deletedPaths.contains(filePath); if (!deleted) { - deleted = FileUtil.delete(file); + deleted = FileUtil.delete(new File(FileUtil.toSystemDependentName(filePath))); } if (deleted) { context.getLoggingManager().getArtifactBuilderLogger().fileDeleted(filePath); - if (isJar) { - deletedJars.add(filePath); + outSrcMapping.remove(filePath); + deletedPaths.add(filePath); + for (String sourcePath : filesToDelete.get(filePath)) { + srcOutMapping.removeValue(sourcePath, filePath); } } else { - if (isJar) { - notDeletedJars.add(filePath); - } + notDeletedPaths.add(filePath); if (notDeletedFilesCount++ > 50) { context.processMessage(new CompilerMessage(BUILDER_NAME, BuildMessage.Kind.WARNING, "Deletion of outdated files stopped because too many files cannot be deleted")); break; @@ -179,8 +214,6 @@ public class IncArtifactBuilder extends ProjectLevelBuilder { context.processMessage(new CompilerMessage(BUILDER_NAME, BuildMessage.Kind.WARNING, "Cannot delete file '" + filePath + "'")); } } - - return deletedJars; } @Override diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/DependentJarsEvaluator.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/DependentJarsEvaluator.java index df294e94d3ab..03afd446559d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/DependentJarsEvaluator.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/DependentJarsEvaluator.java @@ -17,6 +17,7 @@ package org.jetbrains.jps.incremental.artifacts.impl; import com.intellij.openapi.util.Pair; +import org.jetbrains.jps.incremental.artifacts.instructions.DestinationInfo; import org.jetbrains.jps.incremental.artifacts.instructions.JarDestinationInfo; import org.jetbrains.jps.incremental.artifacts.instructions.JarInfo; @@ -31,11 +32,14 @@ public class DependentJarsEvaluator { public void addJarWithDependencies(final JarInfo jarInfo) { if (myJars.add(jarInfo)) { - for (JarDestinationInfo destination : jarInfo.getJarDestinations()) { - addJarWithDependencies(destination.getJarInfo()); + final DestinationInfo destination = jarInfo.getDestination(); + if (destination instanceof JarDestinationInfo) { + addJarWithDependencies(((JarDestinationInfo)destination).getJarInfo()); } - for (Pair pair : jarInfo.getPackedJars()) { - addJarWithDependencies(pair.getSecond()); + for (Pair pair : jarInfo.getContent()) { + if (pair.getSecond() instanceof JarInfo) { + addJarWithDependencies((JarInfo)pair.getSecond()); + } } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/JarsBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/JarsBuilder.java index a9e44f468ab9..065c11faf637 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/JarsBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/impl/JarsBuilder.java @@ -32,6 +32,8 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.PathUtil; import org.jetbrains.jps.incremental.CompileContext; import org.jetbrains.jps.incremental.artifacts.ArtifactBuilderLogger; +import org.jetbrains.jps.incremental.artifacts.ArtifactOutputToSourceMapping; +import org.jetbrains.jps.incremental.artifacts.ArtifactSourceToOutputMapping; import org.jetbrains.jps.incremental.artifacts.IncArtifactBuilder; import org.jetbrains.jps.incremental.artifacts.instructions.*; import org.jetbrains.jps.incremental.messages.BuildMessage; @@ -50,21 +52,28 @@ import java.util.zip.ZipOutputStream; public class JarsBuilder { private static final Logger LOG = Logger.getInstance("#com.intellij.compiler.impl.packagingCompiler.JarsBuilder"); private final Set myJarsToBuild; - private final FileFilter myFileFilter; private final CompileContext myContext; private Map myBuiltJars; + private final ArtifactSourceToOutputMapping mySrcOutMapping; + private final ArtifactOutputToSourceMapping myOutSrcMapping; + private final ArtifactInstructionsBuilder myInstructions; - public JarsBuilder(Set jarsToBuild, FileFilter fileFilter, CompileContext context) { + public JarsBuilder(Set jarsToBuild, + CompileContext context, + ArtifactSourceToOutputMapping srcOutMapping, + ArtifactOutputToSourceMapping outSrcMapping, ArtifactInstructionsBuilder instructions) { + mySrcOutMapping = srcOutMapping; + myOutSrcMapping = outSrcMapping; + myInstructions = instructions; DependentJarsEvaluator evaluator = new DependentJarsEvaluator(); for (JarInfo jarInfo : jarsToBuild) { evaluator.addJarWithDependencies(jarInfo); } myJarsToBuild = evaluator.getJars(); - myFileFilter = fileFilter; myContext = context; } - public boolean buildJars(Set writtenPaths) throws IOException { + public boolean buildJars() throws IOException { myContext.processMessage(new ProgressMessage("Building archives...")); final JarInfo[] sortedJars = sortJars(); @@ -79,7 +88,7 @@ public class JarsBuilder { } myContext.processMessage(new ProgressMessage("Copying archives...")); - copyJars(writtenPaths); + copyJars(); } finally { deleteTemporaryJars(); @@ -95,33 +104,18 @@ public class JarsBuilder { } } - private void copyJars(final Set writtenPaths) throws IOException { + private void copyJars() throws IOException { for (Map.Entry entry : myBuiltJars.entrySet()) { File fromFile = entry.getValue(); - boolean first = true; - for (DestinationInfo destination : entry.getKey().getAllDestinations()) { - if (destination instanceof ExplodedDestinationInfo) { - File toFile = new File(FileUtil.toSystemDependentName(destination.getOutputPath())); - - if (first) { - first = false; - renameFile(fromFile, toFile, writtenPaths); - fromFile = toFile; - } - else { - FileUtil.copyContent(fromFile, toFile); - } - - } + final JarInfo jarInfo = entry.getKey(); + DestinationInfo destination = jarInfo.getDestination(); + if (destination instanceof ExplodedDestinationInfo) { + File toFile = new File(FileUtil.toSystemDependentName(destination.getOutputPath())); + FileUtil.rename(fromFile, toFile); } } } - private static void renameFile(final File fromFile, final File toFile, final Set writtenPaths) throws IOException { - FileUtil.rename(fromFile, toFile); - writtenPaths.add(toFile.getPath()); - } - @Nullable private JarInfo[] sortJars() { final DFSTBuilder builder = new DFSTBuilder(GraphGenerator.create(CachingSemiGraph.create(new JarsGraph()))); @@ -139,12 +133,8 @@ public class JarsBuilder { return jars; } - public Set getJarsToBuild() { - return myJarsToBuild; - } - private void buildJar(final JarInfo jar) throws IOException { - if (jar.getPackedJars().isEmpty() && jar.getPackedRoots().isEmpty()) { + if (jar.getContent().isEmpty()) { final String message = "Archive '" + jar.getPresentableDestination() + "' has no files so it won't be created"; myContext.processMessage(new CompilerMessage(IncArtifactBuilder.BUILDER_NAME, BuildMessage.Kind.WARNING, message)); return; @@ -157,27 +147,38 @@ public class JarsBuilder { FileUtil.createParentDirs(jarFile); final JarOutputStream jarOutputStream = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(jarFile))); + final String targetJarPath = jar.getDestination().getOutputFilePath(); try { final THashSet writtenPaths = new THashSet(); - for (Pair pair : jar.getPackedRoots()) { - final ArtifactSourceRoot root = pair.getSecond(); - final ArtifactBuilderLogger logger = myContext.getLoggingManager().getArtifactBuilderLogger(); - if (root instanceof FileBasedArtifactSourceRoot) { - addFileToJar(jarOutputStream, jarFile, root.getRootFile(), root.getFilter(), pair.getFirst(), writtenPaths); + for (Pair pair : jar.getContent()) { + final String relativePath = pair.getFirst(); + if (pair.getSecond() instanceof ArtifactSourceRoot) { + final ArtifactSourceRoot root = (ArtifactSourceRoot)pair.getSecond(); + final int rootIndex = myInstructions.getRootIndex(root); + LOG.assertTrue(rootIndex != -1, root + " not found in instructions"); + final ArtifactBuilderLogger logger = myContext.getLoggingManager().getArtifactBuilderLogger(); + if (root instanceof FileBasedArtifactSourceRoot) { + addFileToJar(jarOutputStream, jarFile, root.getRootFile(), root.getFilter(), relativePath, targetJarPath, writtenPaths, + rootIndex); + } + else { + final String filePath = FileUtil.toSystemIndependentName(root.getRootFile().getAbsolutePath()); + logger.fileCopied(filePath); + mySrcOutMapping.appendData(filePath, Collections.singletonList(targetJarPath)); + myOutSrcMapping.appendData(targetJarPath, Collections + .singletonList(new ArtifactOutputToSourceMapping.SourcePathAndRootIndex(filePath, rootIndex))); + extractFileAndAddToJar(jarOutputStream, (JarBasedArtifactSourceRoot)root, relativePath, writtenPaths); + } } else { - logger.fileCopied(FileUtil.toSystemIndependentName(root.getRootFile().getAbsolutePath())); - extractFileAndAddToJar(jarOutputStream, (JarBasedArtifactSourceRoot)root, pair.getFirst(), writtenPaths); - } - } - - for (Pair nestedJar : jar.getPackedJars()) { - File nestedJarFile = myBuiltJars.get(nestedJar.getSecond()); - if (nestedJarFile != null) { - addFileToJar(jarOutputStream, jarFile, nestedJarFile, SourceFileFilter.ALL, nestedJar.getFirst(), writtenPaths); - } - else { - LOG.debug("nested jar file " + nestedJar.getFirst() + " for " + jar.getPresentableDestination() + " not found"); + JarInfo nestedJar = (JarInfo)pair.getSecond(); + File nestedJarFile = myBuiltJars.get(nestedJar); + if (nestedJarFile != null) { + addFileToJar(jarOutputStream, jarFile, nestedJarFile, SourceFileFilter.ALL, relativePath, targetJarPath, writtenPaths, -1); + } + else { + LOG.debug("nested jar file " + relativePath + " for " + jar.getPresentableDestination() + " not found"); + } } } } @@ -211,21 +212,23 @@ public class JarsBuilder { } private void addFileToJar(final @NotNull JarOutputStream jarOutputStream, final @NotNull File jarFile, @NotNull File file, - SourceFileFilter filter, @NotNull String relativePath, final @NotNull Set writtenPaths) throws IOException { + SourceFileFilter filter, @NotNull String relativePath, String targetJarPath, + final @NotNull Set writtenPaths, final int rootIndex) throws IOException { if (!file.exists() || FileUtil.isAncestor(file, jarFile, false)) { return; } relativePath = addParentDirectories(jarOutputStream, writtenPaths, relativePath); - addFileOrDirRecursively(jarOutputStream, file, filter, relativePath, writtenPaths); + addFileOrDirRecursively(jarOutputStream, file, filter, relativePath, targetJarPath, writtenPaths, rootIndex); } private void addFileOrDirRecursively(@NotNull ZipOutputStream jarOutputStream, @NotNull File file, SourceFileFilter filter, @NotNull String relativePath, - @NotNull Set writtenItemRelativePaths) throws IOException { - if (!filter.accept(FileUtil.toSystemIndependentName(file.getAbsolutePath()))) { + String targetJarPath, @NotNull Set writtenItemRelativePaths, int rootIndex) throws IOException { + final String filePath = FileUtil.toSystemIndependentName(file.getAbsolutePath()); + if (!filter.accept(filePath)) { return; } @@ -237,15 +240,20 @@ public class JarsBuilder { final File[] children = file.listFiles(); if (children != null) { for (File child : children) { - addFileOrDirRecursively(jarOutputStream, child, filter, directoryPath + child.getName(), writtenItemRelativePaths); + addFileOrDirRecursively(jarOutputStream, child, filter, directoryPath + child.getName(), targetJarPath, writtenItemRelativePaths, + rootIndex); } } return; } - final boolean added = ZipUtil.addFileToZip(jarOutputStream, file, relativePath, writtenItemRelativePaths, myFileFilter); - if (added) { - myContext.getLoggingManager().getArtifactBuilderLogger().fileCopied(FileUtil.toSystemIndependentName(file.getAbsolutePath())); + final boolean added = ZipUtil.addFileToZip(jarOutputStream, file, relativePath, writtenItemRelativePaths, null); + if (rootIndex != -1) { + myOutSrcMapping.appendData(targetJarPath, Collections.singletonList(new ArtifactOutputToSourceMapping.SourcePathAndRootIndex(filePath, rootIndex))); + if (added) { + mySrcOutMapping.appendData(filePath, Collections.singletonList(targetJarPath)); + myContext.getLoggingManager().getArtifactBuilderLogger().fileCopied(filePath); + } } } @@ -283,8 +291,9 @@ public class JarsBuilder { public Iterator getIn(final JarInfo n) { Set ins = new HashSet(); - for (JarDestinationInfo destination : n.getJarDestinations()) { - ins.add(destination.getJarInfo()); + final DestinationInfo destination = n.getDestination(); + if (destination instanceof JarDestinationInfo) { + ins.add(((JarDestinationInfo)destination).getJarInfo()); } return ins.iterator(); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilder.java index 4f2a935ecbc0..765d26cdf747 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilder.java @@ -1,5 +1,6 @@ package org.jetbrains.jps.incremental.artifacts.instructions; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.IOException; @@ -10,8 +11,8 @@ import java.io.IOException; public interface ArtifactInstructionsBuilder { void processRoots(ArtifactRootProcessor processor) throws IOException; - void processContainingRoots(String filePath, ArtifactRootProcessor processor) throws IOException; - @Nullable JarInfo getJarInfo(String outputPath); + + int getRootIndex(@NotNull ArtifactSourceRoot root); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilderImpl.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilderImpl.java index d1e3e07a80ae..de7b70f87a15 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilderImpl.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactInstructionsBuilderImpl.java @@ -1,6 +1,7 @@ package org.jetbrains.jps.incremental.artifacts.instructions; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.util.containers.LinkedMultiMap; import com.intellij.util.containers.MultiMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -17,7 +18,6 @@ import java.util.Map; * @author nik */ public class ArtifactInstructionsBuilderImpl implements ArtifactInstructionsBuilder { - private final Map mySourceByOutput; private final Map myJarByPath; private final MultiMap myInstructions; private final ModuleRootsIndex myRootsIndex; @@ -26,9 +26,8 @@ public class ArtifactInstructionsBuilderImpl implements ArtifactInstructionsBuil public ArtifactInstructionsBuilderImpl(ModuleRootsIndex rootsIndex, IgnoredFilePatterns patterns) { myRootsIndex = rootsIndex; myIgnoredFilePatterns = patterns; - mySourceByOutput = new HashMap(); myJarByPath = new HashMap(); - myInstructions = new MultiMap(); + myInstructions = new LinkedMultiMap(); } public IgnoredFilePatterns getIgnoredFilePatterns() { @@ -41,28 +40,14 @@ public class ArtifactInstructionsBuilderImpl implements ArtifactInstructionsBuil return false; } - if (checkOutputPath(destinationInfo.getOutputPath(), root)) { - myInstructions.putValue(root, destinationInfo); - return true; - } - return false; + myInstructions.putValue(root, destinationInfo); + return true; } public ModuleRootsIndex getRootsIndex() { return myRootsIndex; } - public boolean checkOutputPath(final String outputPath, final ArtifactSourceRoot sourceFile) { - //todo[nik] combine intersecting roots - //ArtifactSourceRoot old = mySourceByOutput.get(outputPath); - //if (old == null) { - // mySourceByOutput.put(outputPath, sourceFile); - // return true; - //} - //todo[nik] show warning? - return true; - } - public boolean registerJarFile(@NotNull JarInfo jarInfo, @NotNull String outputPath) { if (myJarByPath.containsKey(outputPath)) { return false; @@ -78,20 +63,23 @@ public class ArtifactInstructionsBuilderImpl implements ArtifactInstructionsBuil } @Override - public void processRoots(ArtifactRootProcessor processor) throws IOException { + public int getRootIndex(@NotNull ArtifactSourceRoot root) { + int i = 0; for (Map.Entry> entry : myInstructions.entrySet()) { - processor.process(entry.getKey(), entry.getValue()); + if (entry.getKey().equals(root)) { + return i; + } + i++; } + return -1; } @Override - public void processContainingRoots(String filePath, ArtifactRootProcessor processor) throws IOException { - //todo[nik] improve? + public void processRoots(ArtifactRootProcessor processor) throws IOException { + int i = 0; for (Map.Entry> entry : myInstructions.entrySet()) { - final ArtifactSourceRoot root = entry.getKey(); - if (root.containsFile(filePath)) { - processor.process(root, entry.getValue()); - } + processor.process(entry.getKey(), i, entry.getValue()); + i++; } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootProcessor.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootProcessor.java index 2d048dfbfeeb..7dc38137b27b 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootProcessor.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactRootProcessor.java @@ -7,5 +7,5 @@ import java.util.Collection; * @author nik */ public interface ArtifactRootProcessor { - void process(ArtifactSourceRoot root, Collection destinations) throws IOException; + void process(ArtifactSourceRoot root, int rootIndex, Collection destinations) throws IOException; } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactSourceRoot.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactSourceRoot.java index 5f57da302b7d..503aae769625 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactSourceRoot.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/ArtifactSourceRoot.java @@ -1,10 +1,12 @@ package org.jetbrains.jps.incremental.artifacts.instructions; import org.jetbrains.annotations.NotNull; +import org.jetbrains.jps.incremental.CompileContext; +import org.jetbrains.jps.incremental.artifacts.ArtifactOutputToSourceMapping; +import org.jetbrains.jps.incremental.artifacts.ArtifactSourceToOutputMapping; import java.io.File; import java.io.IOException; -import java.util.List; /** * @author nik @@ -21,7 +23,10 @@ public abstract class ArtifactSourceRoot { public abstract boolean containsFile(String filePath); - public abstract void copyFromRoot(String filePath, String outputPath, List outputs) throws IOException; + public abstract void copyFromRoot(String filePath, + int rootIndex, String outputPath, + CompileContext context, ArtifactSourceToOutputMapping srcOutMapping, + ArtifactOutputToSourceMapping outSrcMapping) throws IOException; public SourceFileFilter getFilter() { return myFilter; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/CopyToDirectoryInstructionCreator.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/CopyToDirectoryInstructionCreator.java index 5e27d032e174..312d3b1d29c8 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/CopyToDirectoryInstructionCreator.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/CopyToDirectoryInstructionCreator.java @@ -45,12 +45,11 @@ public class CopyToDirectoryInstructionCreator extends ArtifactCompilerInstructi public ArtifactCompilerInstructionCreator archive(@NotNull String archiveFileName) { String jarOutputPath = myOutputPath + "/" + archiveFileName; - final JarInfo jarInfo = new JarInfo(); + final ExplodedDestinationInfo destination = new ExplodedDestinationInfo(jarOutputPath); + final JarInfo jarInfo = new JarInfo(destination); if (!myInstructionsBuilder.registerJarFile(jarInfo, jarOutputPath)) { return new SkipAllInstructionCreator(myInstructionsBuilder); } - final ExplodedDestinationInfo destination = new ExplodedDestinationInfo(jarOutputPath); - jarInfo.addDestination(destination); return new PackIntoArchiveInstructionCreator(myInstructionsBuilder, jarInfo, "", destination); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/DestinationInfo.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/DestinationInfo.java index 831c1fbf271b..56bbe0f8efb7 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/DestinationInfo.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/DestinationInfo.java @@ -25,8 +25,8 @@ public abstract class DestinationInfo { private final String myOutputFilePath; protected DestinationInfo(@NotNull final String outputPath, @NotNull String outputFilePath) { - myOutputFilePath = outputFilePath; myOutputPath = outputPath; + myOutputFilePath = outputFilePath; } @NotNull diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/FileBasedArtifactSourceRoot.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/FileBasedArtifactSourceRoot.java index e2096693bdb0..b96514a4d62c 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/FileBasedArtifactSourceRoot.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/FileBasedArtifactSourceRoot.java @@ -3,10 +3,13 @@ package org.jetbrains.jps.incremental.artifacts.instructions; import com.intellij.openapi.util.io.FileUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.PathUtil; +import org.jetbrains.jps.incremental.CompileContext; +import org.jetbrains.jps.incremental.artifacts.ArtifactOutputToSourceMapping; +import org.jetbrains.jps.incremental.artifacts.ArtifactSourceToOutputMapping; import java.io.File; import java.io.IOException; -import java.util.List; +import java.util.Collections; /** * @author nik @@ -49,8 +52,12 @@ public class FileBasedArtifactSourceRoot extends ArtifactSourceRoot { return 31 * super.hashCode() + myFile.hashCode(); } - public void copyFromRoot(String filePath, String outputPath, List outputs) throws IOException { + public void copyFromRoot(String filePath, + int rootIndex, String outputPath, + CompileContext context, ArtifactSourceToOutputMapping srcOutMapping, + ArtifactOutputToSourceMapping outSrcMapping) throws IOException { final File file = new File(FileUtil.toSystemDependentName(filePath)); + if (!file.exists()) return; String targetPath; if (!file.equals(getRootFile())) { final String relativePath = FileUtil.getRelativePath(FileUtil.toSystemIndependentName(getRootFile().getPath()), filePath, '/'); @@ -59,8 +66,13 @@ public class FileBasedArtifactSourceRoot extends ArtifactSourceRoot { else { targetPath = outputPath; } - final File targetFile = new File(FileUtil.toSystemDependentName(targetPath)); - FileUtil.copyContent(file, targetFile); - outputs.add(targetPath); + + if (outSrcMapping.getState(targetPath) == null) { + context.getLoggingManager().getArtifactBuilderLogger().fileCopied(filePath); + final File targetFile = new File(FileUtil.toSystemDependentName(targetPath)); + FileUtil.copyContent(file, targetFile); + srcOutMapping.appendData(filePath, Collections.singletonList(targetPath)); + } + outSrcMapping.appendData(targetPath, Collections.singletonList(new ArtifactOutputToSourceMapping.SourcePathAndRootIndex(filePath, rootIndex))); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarBasedArtifactSourceRoot.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarBasedArtifactSourceRoot.java index 5505f62c7758..3f8de3c31939 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarBasedArtifactSourceRoot.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarBasedArtifactSourceRoot.java @@ -5,11 +5,14 @@ import com.intellij.openapi.util.text.StringUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.PathUtil; +import org.jetbrains.jps.incremental.CompileContext; +import org.jetbrains.jps.incremental.artifacts.ArtifactOutputToSourceMapping; +import org.jetbrains.jps.incremental.artifacts.ArtifactSourceToOutputMapping; import org.jetbrains.jps.incremental.artifacts.JarPathUtil; import java.io.*; +import java.util.Collections; import java.util.Enumeration; -import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -82,7 +85,11 @@ public class JarBasedArtifactSourceRoot extends ArtifactSourceRoot { } } - public void copyFromRoot(String filePath, final String outputPath, final List outputs) throws IOException { + public void copyFromRoot(final String filePath, + final int rootIndex, final String outputPath, + CompileContext context, final ArtifactSourceToOutputMapping srcOutMapping, + final ArtifactOutputToSourceMapping outSrcMapping) throws IOException { + context.getLoggingManager().getArtifactBuilderLogger().fileCopied(filePath); processEntries(new EntryProcessor() { @Override public void process(@Nullable InputStream inputStream, @NotNull String relativePath) throws IOException { @@ -94,17 +101,21 @@ public class JarBasedArtifactSourceRoot extends ArtifactSourceRoot { outputFile.mkdir(); } else { - final BufferedInputStream from = new BufferedInputStream(inputStream); - final BufferedOutputStream to = new BufferedOutputStream(new FileOutputStream(outputFile)); - try { - FileUtil.copy(from, to); - } - finally { - from.close(); - to.close(); + String fullSourcePath = filePath + JarPathUtil.JAR_SEPARATOR + relativePath; + if (outSrcMapping.getState(fullOutputPath) == null) { + final BufferedInputStream from = new BufferedInputStream(inputStream); + final BufferedOutputStream to = new BufferedOutputStream(new FileOutputStream(outputFile)); + try { + FileUtil.copy(from, to); + } + finally { + from.close(); + to.close(); + } + srcOutMapping.appendData(filePath, Collections.singletonList(fullOutputPath)); } + outSrcMapping.appendData(fullOutputPath, Collections.singletonList(new ArtifactOutputToSourceMapping.SourcePathAndRootIndex(fullSourcePath, rootIndex))); } - outputs.add(fullOutputPath); } }); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarInfo.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarInfo.java index db496203ed8b..27c1d5c45a01 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarInfo.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/JarInfo.java @@ -17,60 +17,40 @@ package org.jetbrains.jps.incremental.artifacts.instructions; import com.intellij.openapi.util.Pair; +import org.jetbrains.annotations.NotNull; import java.util.ArrayList; -import java.util.LinkedHashSet; import java.util.List; /** * @author nik */ public class JarInfo { - private final List> myPackedRoots; - private final LinkedHashSet> myPackedJars; - private final List myDestinations; + private final List> myContent; + private final DestinationInfo myDestination; - public JarInfo() { - myDestinations = new ArrayList(); - myPackedRoots = new ArrayList>(); - myPackedJars = new LinkedHashSet>(); - } - - public void addDestination(DestinationInfo info) { - myDestinations.add(info); - if (info instanceof JarDestinationInfo) { - JarDestinationInfo destinationInfo = (JarDestinationInfo)info; - destinationInfo.getJarInfo().myPackedJars.add(Pair.create(destinationInfo.getPathInJar(), this)); - } + public JarInfo(@NotNull DestinationInfo destination) { + myDestination = destination; + myContent = new ArrayList>(); } public void addContent(String pathInJar, ArtifactSourceRoot sourceFile) { - myPackedRoots.add(Pair.create(pathInJar, sourceFile)); + myContent.add(Pair.create(pathInJar, (Object)sourceFile)); } - public List> getPackedRoots() { - return myPackedRoots; + public void addJar(String pathInJar, JarInfo jarInfo) { + myContent.add(Pair.create(pathInJar, (Object)jarInfo)); } - public LinkedHashSet> getPackedJars() { - return myPackedJars; + public List> getContent() { + return myContent; } - public List getJarDestinations() { - final ArrayList list = new ArrayList(); - for (DestinationInfo destination : myDestinations) { - if (destination instanceof JarDestinationInfo) { - list.add((JarDestinationInfo)destination); - } - } - return list; - } - - public List getAllDestinations() { - return myDestinations; + public DestinationInfo getDestination() { + return myDestination; } public String getPresentableDestination() { - return !myDestinations.isEmpty() ? myDestinations.get(0).getOutputPath() : ""; + return myDestination.getOutputPath(); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/PackIntoArchiveInstructionCreator.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/PackIntoArchiveInstructionCreator.java index feba02779bed..1e80318d1203 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/PackIntoArchiveInstructionCreator.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/artifacts/instructions/PackIntoArchiveInstructionCreator.java @@ -15,6 +15,7 @@ */ package org.jetbrains.jps.incremental.artifacts.instructions; +import com.intellij.openapi.util.Pair; import org.jetbrains.annotations.NotNull; import java.io.File; @@ -59,13 +60,13 @@ public class PackIntoArchiveInstructionCreator extends ArtifactCompilerInstructi } public ArtifactCompilerInstructionCreator archive(@NotNull String archiveFileName) { - final JarInfo jarInfo = new JarInfo(); + final JarDestinationInfo destination = new JarDestinationInfo(childPathInJar(archiveFileName), myJarInfo, myJarDestination); + final JarInfo jarInfo = new JarInfo(destination); final String outputPath = myJarDestination.getOutputPath() + "/" + archiveFileName; if (!myInstructionsBuilder.registerJarFile(jarInfo, outputPath)) { return new SkipAllInstructionCreator(myInstructionsBuilder); } - final JarDestinationInfo destination = new JarDestinationInfo(childPathInJar(archiveFileName), myJarInfo, myJarDestination); - jarInfo.addDestination(destination); + myJarInfo.addJar(destination.getPathInJar(), jarInfo); return new PackIntoArchiveInstructionCreator(myInstructionsBuilder, jarInfo, "", destination); } } diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderOverwriteTest.java b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderOverwriteTest.java new file mode 100644 index 000000000000..1319014452bc --- /dev/null +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderOverwriteTest.java @@ -0,0 +1,163 @@ +package org.jetbrains.jps.incremental.artifacts; + +import org.jetbrains.jps.artifacts.Artifact; + +import static com.intellij.util.io.TestFileSystemBuilder.fs; +import static org.jetbrains.jps.incremental.artifacts.LayoutElementTestUtil.root; + +/** + * @author nik + */ +public class ArtifactBuilderOverwriteTest extends ArtifactBuilderTestCase { + public void testOverwriteArchives() { + final String aFile = createFile("aaa.txt", "a"); + final String bFile = createFile("bbb.txt", "b"); + final Artifact a = addArtifact( + root() + .archive("x.jar").fileCopy(aFile).end() + .archive("x.jar") + .fileCopy(bFile)); + buildAll(); + assertOutput(a, fs() + .archive("x.jar") + .file("aaa.txt", "a") + ); + buildAllAndAssertUpToDate(); + + change(aFile, "a2"); + buildAll(); + assertCopied("aaa.txt"); + assertOutput(a, fs().archive("x.jar").file("aaa.txt", "a2")); + buildAllAndAssertUpToDate(); + + change(bFile, "b2"); + buildAllAndAssertUpToDate(); + + delete(bFile); + buildAllAndAssertUpToDate(); + } + + public void testOverwriteNestedArchive() { + final String cFile = createFile("c.txt", "c"); + final String eFile = createFile("e.txt", "e"); + final Artifact a = addArtifact( + root() + .archive("a.jar").archive("b.jar").fileCopy(cFile).end().end() + .archive("a.jar").archive("d.jar").fileCopy(eFile)); + buildAll(); + assertOutput(a, fs().archive("a.jar").archive("b.jar").file("c.txt", "c")); + buildAllAndAssertUpToDate(); + + change(eFile, "e2"); + buildAllAndAssertUpToDate(); + + change(cFile, "c2"); + buildAll(); + assertCopied("c.txt"); + assertOutput(a, fs().archive("a.jar").archive("b.jar").file("c.txt", "c2")); + buildAllAndAssertUpToDate(); + + delete(eFile); + buildAllAndAssertUpToDate(); + } + + public void testOverwriteFileByArchive() { + final String xFile = createFile("x.txt", "1"); + final String jarFile = createFile("junit.jar", "123"); + Artifact a = addArtifact(root() + .archive("junit.jar").fileCopy(xFile).end() + .fileCopy(jarFile)); + buildAll(); + assertOutput(a, fs().archive("junit.jar").file("x.txt", "1")); + buildAllAndAssertUpToDate(); + + change(xFile, "2"); + buildAll(); + assertCopied("x.txt"); + assertOutput(a, fs().archive("junit.jar").file("x.txt", "2")); + buildAllAndAssertUpToDate(); + + change(jarFile, "321"); + buildAllAndAssertUpToDate(); + + delete(jarFile); + buildAllAndAssertUpToDate(); + } + + public void testOverwriteArchiveByFile() { + final String xFile = createFile("x.txt", "1"); + final String jarFile = createFile("jdom.jar", "123"); + Artifact a = addArtifact(root() + .fileCopy(jarFile) + .archive("jdom.jar").fileCopy(xFile)); + buildAll(); + assertOutput(a, fs().file("jdom.jar", "123")); + buildAllAndAssertUpToDate(); + + change(xFile, "2"); + buildAllAndAssertUpToDate(); + + change(jarFile, "321"); + buildAll(); + assertCopied("jdom.jar"); + assertOutput(a, fs().file("jdom.jar", "321")); + buildAllAndAssertUpToDate(); + + delete(xFile); + buildAllAndAssertUpToDate(); + } + + public void testFileOrder() { + final String firstFile = createFile("d1/xxx.txt", "first"); + final String secondFile = createFile("d2/xxx.txt", "second"); + final String fooFile = createFile("d3/xxx.txt", "foo"); + final Artifact a = addArtifact( + root().dir("ddd") + .fileCopy(firstFile) + .fileCopy(fooFile) + .fileCopy(secondFile).end() + ); + buildAll(); + assertOutput(a, fs().dir("ddd").file("xxx.txt", "first")); + buildAllAndAssertUpToDate(); + + change(firstFile, "first2"); + buildAll(); + assertCopied("d1/xxx.txt"); + assertOutput(a, fs().dir("ddd").file("xxx.txt", "first2")); + buildAllAndAssertUpToDate(); + + change(secondFile); + buildAllAndAssertUpToDate(); + + change(fooFile); + buildAllAndAssertUpToDate(); + + delete(fooFile); + buildAllAndAssertUpToDate(); + + delete(secondFile); + buildAllAndAssertUpToDate(); + } + + public void testDeleteOverwritingFiles() { + final String firstFile = createFile("d1/xxx.txt", "1"); + final String secondFile = createFile("d2/xxx.txt", "2"); + final Artifact a = addArtifact("a", + root().dir("ddd").fileCopy(firstFile).fileCopy(secondFile).fileCopy(createFile("y.txt")) + ); + buildAll(); + assertOutput(a, fs().dir("ddd").file("xxx.txt", "1").file("y.txt")); + + delete(firstFile); + buildAll(); + assertDeletedAndCopied("out/artifacts/a/ddd/xxx.txt", "d2/xxx.txt"); + assertOutput(a, fs().dir("ddd").file("xxx.txt", "2").file("y.txt")); + buildAllAndAssertUpToDate(); + + delete(secondFile); + buildAll(); + assertDeleted("out/artifacts/a/ddd/xxx.txt"); + assertOutput(a, fs().dir("ddd").file("y.txt")); + } +} diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTest.java b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTest.java index 004d58fb685a..e9f8a68ad002 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTest.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTest.java @@ -131,51 +131,6 @@ public class ArtifactBuilderTest extends ArtifactBuilderTestCase { ); } - public void testOverwriteArchives() { - final Artifact included = addArtifact("included", - root().archive("x.jar").fileCopy(createFile("aaa.class"))); - final Artifact a = addArtifact( - root() - .artifact(included) - .archive("x.jar") - .fileCopy(createFile("bbb.class"))); - buildAll(); - assertOutput(a, fs() - .archive("x.jar") - .file("aaa.class") - ); - } - - public void testOverwriteNestedArchive() { - final Artifact included = addArtifact("included", root().archive("a.jar").archive("b.jar").fileCopy(createFile("c.class"))); - final Artifact a = addArtifact( - root() - .artifact(included) - .archive("a.jar").archive("d.jar").fileCopy(createFile("e.class"))); - buildAll(); - assertOutput(a, fs().archive("a.jar").archive("b.jar").file("c.class")); - } - - public void testOverwriteFileByArchive() { - final Library library = addProjectLibrary("lib", getJUnitJarPath()); - Artifact included = addArtifact("included", root().archive("junit.jar").fileCopy(createFile("x.class"))); - Artifact a = addArtifact(root() - .artifact(included) - .lib(library)); - buildAll(); - assertOutput(a, fs().archive("junit.jar").file("x.class")); - } - - //todo[nik] fix ordering - public void _testOverwriteArchiveByFile() { - Artifact included = addArtifact("included", root().archive("jdom.jar").fileCopy(createFile("x.class"))); - Artifact a = addArtifact(root() - .fileCopy(createFile("jdom.jar", "123")) - .artifact(included)); - buildAll(); - assertOutput(a, fs().file("jdom.jar", "123")); - } - public void testCopyLibrary() { final Library library = addProjectLibrary("lib", getJUnitJarPath()); final Artifact a = addArtifact(root().lib(library)); @@ -183,24 +138,6 @@ public class ArtifactBuilderTest extends ArtifactBuilderTestCase { assertOutput(a, fs().file("junit.jar")); } - //todo[nik] fix ordering - public void _testFileOrder() { - final Artifact a1 = addArtifact("included1", - root().dir("ddd").fileCopy(createFile("d1/xxx.txt", "first"))); - final Artifact a2 = addArtifact("included2", - root().dir("ddd").fileCopy(createFile("d2/xxx.txt", "second"))); - final Artifact a = addArtifact( - root() - .artifact(a1) - .dir("ddd") - .fileCopy(createFile("d3/xxx.txt", "foo")) - .end() - .artifact(a2) - ); - buildAll(); - assertOutput(a, fs().dir("ddd").file("xxx.txt", "first")); - } - public void testModuleOutput() { final String file = createFile("src/A.java", "public class A {}"); final Module module = addModule("a", PathUtil.getParentPath(file)); diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java index fb3ac9dc1100..20927bd768bb 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/artifacts/ArtifactBuilderTestCase.java @@ -262,7 +262,8 @@ public abstract class ArtifactBuilderTestCase extends UsefulTestCase { } protected void assertUpToDate() { - assertCopied(); + assertEmpty(myArtifactBuilderLogger.myDeletedFilePaths); + assertEmpty(myArtifactBuilderLogger.myCopiedFilePaths); } private String getProjectRelativePath(String path) {