MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

jps internal refactoring to allow event-driven changes management

Browse Source
This commit is contained in:
Eugene Zhuravlev
2012-01-10 20:16:31 +01:00
parent 761298fc39
commit 5503314ebf
24 changed files with 1089 additions and 630 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/compiler/impl/src/com/intellij/compiler/JpsServerManager.java
+3 -2
View File
@@ -58,7 +58,8 @@ import org.jetbrains.jps.client.Client;
import org.jetbrains.jps.server.ClasspathBootstrap;
import org.jetbrains.jps.server.Server;
import javax.tools.*;
import javax.tools.JavaCompiler;
import javax.tools.ToolProvider;
import java.io.File;
import java.io.IOException;
import java.util.*;
@@ -330,7 +331,7 @@ public class JpsServerManager implements ApplicationComponent{
// debugging
cmdLine.addParameter("-XX:+HeapDumpOnOutOfMemoryError");
//cmdLine.addParameter("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5008");
cmdLine.addParameter("-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5008");
// javac's VM should use the same default locale that IDEA uses in order for javac to print messages in 'correct' language
final String lang = System.getProperty("user.language");
jps/jps-builders/proto/jps_remote_proto.proto
+1
View File
@@ -36,6 +36,7 @@ message Message {
MAKE = 2;
CLEAN = 3;
CANCEL = 4;
FORCED_COMPILATION = 5;
}
required Type command_type = 1;
optional string project_id = 2;
jps/jps-builders/src/org/jetbrains/jps/api/BuildType.java
+1 -1
View File
@@ -5,5 +5,5 @@ package org.jetbrains.jps.api;
* Date: 9/10/11
*/
public enum BuildType {
REBUILD, MAKE, CLEAN
PROJECT_REBUILD, FORCED_COMPILATION, MAKE, CLEAN
}
jps/jps-builders/src/org/jetbrains/jps/api/JpsRemoteProto.java
+2
View File
@@ -778,6 +778,7 @@ public final class JpsRemoteProto {
MAKE(1, 2),
CLEAN(2, 3),
CANCEL(3, 4),
FORCED_COMPILATION(4, 5),
;
@@ -789,6 +790,7 @@ public final class JpsRemoteProto {
case 2: return MAKE;
case 3: return CLEAN;
case 4: return CANCEL;
case 5: return FORCED_COMPILATION;
default: return null;
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/Builder.java
+32 -39
View File
@@ -2,17 +2,13 @@ package org.jetbrains.jps.incremental;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.io.FileUtil;
import org.jetbrains.ether.dependencyView.ClassRepr;
import org.jetbrains.ether.dependencyView.Mappings;
import org.jetbrains.jps.Module;
import org.jetbrains.jps.ModuleChunk;
import org.jetbrains.jps.PathUtil;
import org.jetbrains.jps.incremental.storage.TimestampStorage;
import org.jetbrains.jps.incremental.storage.SourceToOutputMapping;
import java.io.File;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.*;
/**
* @author Eugene Zhuravlev
@@ -37,10 +33,6 @@ public abstract class Builder {
ALL_COMPILED_FILES_KEY.set(context, null);
}
public static boolean isFileDirty(File file, CompileContext context, TimestampStorage tsStorage) throws Exception {
return !context.isMake() || tsStorage.getStamp(file) != file.lastModified();
}
/**
* @param context
* @param delta
@@ -53,14 +45,13 @@ public abstract class Builder {
public final boolean updateMappings(CompileContext context, final Mappings delta, ModuleChunk chunk, Collection<File> filesToCompile, Collection<File> successfullyCompiled) throws Exception {
boolean additionalPassRequired = false;
final TimestampStorage tsStorage = context.getBuildDataManager().getTimestampStorage(getName());
final Set<String> removedPaths = getRemovedPaths(context);
final Mappings globalMappings = context.getMappings();
final Mappings globalMappings = context.getDataManager().getMappings();
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (globalMappings) {
if (context.isMake()) {
if (!context.isProjectRebuild()) {
final Set<File> allCompiledFiles = getAllCompiledFilesContainer(context);
final Set<File> allAffectedFiles = getAllAffectedFilesContainer(context);
@@ -80,54 +71,56 @@ public abstract class Builder {
if (incremental) {
final Set<File> newlyAffectedFiles = new HashSet<File>(allAffectedFiles);
newlyAffectedFiles.removeAll(affectedBeforeDif);
// todo: temp code
for (String removedPath : removedPaths) {
newlyAffectedFiles.remove(new File(removedPath));
}
if (!newlyAffectedFiles.isEmpty()) {
for (File file : newlyAffectedFiles) {
tsStorage.markDirty(file);
context.markDirty(file);
}
additionalPassRequired = chunkContainsAffectedFiles(context, chunk, newlyAffectedFiles);
additionalPassRequired = context.isMake() && chunkContainsAffectedFiles(context, chunk, newlyAffectedFiles);
}
}
else {
additionalPassRequired = true;
context.setDirty(chunk, true);
additionalPassRequired = context.isMake();
context.markDirty(chunk);
}
}
globalMappings.integrate(delta, successfullyCompiled, removedPaths);
for (File file : successfullyCompiled) {
tsStorage.saveStamp(file);
}
}
return additionalPassRequired;
}
// delete all class files that according to mappings correspond to given sources
public static void deleteCorrespondingClasses(CompileContext context, Collection<File> sources) {
if (context.isMake() && !sources.isEmpty()) {
final Mappings mappings = context.getMappings();
for (File file : sources) {
final Set<ClassRepr> classes = mappings.getClasses(FileUtil.toSystemIndependentName(file.getPath()));
if (classes != null) {
for (ClassRepr aClass : classes) {
final String fileName = aClass.getFileName();
if (fileName != null) {
FileUtil.delete(new File(fileName));
}
public static void deleteCorrespondingOutputFiles(CompileContext context, Map<File, Module> sources) throws Exception {
if (!context.isProjectRebuild() && !sources.isEmpty()) {
for (Map.Entry<File, Module> pair : sources.entrySet()) {
final File file = pair.getKey();
final String srcPath = file.getPath();
final String moduleName = pair.getValue().getName().toLowerCase(Locale.US);
final SourceToOutputMapping srcToOut = context.getDataManager().getSourceToOutputMap(moduleName, context.isCompilingTests());
final Collection<String> outputs = srcToOut.getState(srcPath);
if (outputs != null) {
for (String output : outputs) {
FileUtil.delete(new File(output));
}
srcToOut.remove(srcPath);
}
}
}
}
private static boolean chunkContainsAffectedFiles(CompileContext context, ModuleChunk chunk, final Set<File> affected) throws Exception {
final Set<File> chunkRoots = new HashSet<File>();
for (String r : context.isCompilingTests() ? chunk.getTestRoots() : chunk.getSourceRoots()) {
chunkRoots.add(new File(r));
}
for (File file : affected) {
if (PathUtil.isUnder(chunkRoots, file)) {
return true;
final Set<Module> chunkModules = new HashSet<Module>(chunk.getModules());
if (!chunkModules.isEmpty()) {
for (File file : affected) {
final RootDescriptor moduleAndRoot = context.getModuleAndRoot(file);
if (moduleAndRoot != null && chunkModules.contains(moduleAndRoot.module)) {
return true;
}
}
}
return false;
jps/jps-builders/src/org/jetbrains/jps/incremental/CompileContext.java
+179 -70
View File
@@ -2,57 +2,78 @@ package org.jetbrains.jps.incremental;
import com.intellij.openapi.util.UserDataHolderBase;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.containers.SLRUCache;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.ether.dependencyView.Mappings;
import org.jetbrains.jps.*;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.incremental.storage.SourceToOutputMapping;
import org.jetbrains.jps.incremental.storage.TimestampStorage;
import java.io.File;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.*;
/**
* @author Eugene Zhuravlev
* Date: 9/17/11
*/
public class CompileContext extends UserDataHolderBase implements MessageHandler{
private final CompileScope myScope;
private final boolean myIsMake;
private final boolean myIsProjectRebuild;
private final ProjectChunks myProductionChunks;
private final ProjectChunks myTestChunks;
private final FSState myFsState;
private final MessageHandler myDelegateMessageHandler;
private volatile boolean myCompilingTests = false;
private final BuildDataManager myDataManager;
private final Mappings myMappings;
private final Set<Module> myDirtyModules = new HashSet<Module>();
private SLRUCache<Module, FSSnapshot> myFilesCache = new SLRUCache<Module, FSSnapshot>(10, 10) {
@NotNull
public FSSnapshot createValue(Module key) {
return buildSnapshot(key);
}
};
private final Map<File, RootDescriptor> myRootToModuleMap = new HashMap<File, RootDescriptor>();
private final Map<Module, List<RootDescriptor>> myModuleToRootsMap = new HashMap<Module, List<RootDescriptor>>();
private final ProjectPaths myProjectPaths;
private volatile boolean myErrorsFound = false;
private final long myCompilationStartStamp;
private final TimestampStorage myTsStorage;
public CompileContext(CompileScope scope,
String projectName,
public CompileContext(String projectName, CompileScope scope,
boolean isMake,
final Mappings mappings,
boolean isProjectRebuild,
ProjectChunks productionChunks,
ProjectChunks testChunks,
MessageHandler delegateMessageHandler) {
FSState fsState, TimestampStorage tsStorage, MessageHandler delegateMessageHandler) throws ProjectBuildException {
myTsStorage = tsStorage;
myCompilationStartStamp = System.currentTimeMillis();
myScope = scope;
myIsMake = isMake;
myIsProjectRebuild = isProjectRebuild;
myIsMake = isProjectRebuild? false : isMake;
myProductionChunks = productionChunks;
myTestChunks = testChunks;
myFsState = fsState;
myDelegateMessageHandler = delegateMessageHandler;
myDataManager = new BuildDataManager(projectName);
myMappings = mappings;
myProjectPaths = new ProjectPaths(scope.getProject());
final Project project = scope.getProject();
myProjectPaths = new ProjectPaths(project);
for (Module module : project.getModules().values()) {
List<RootDescriptor> moduleRoots = myModuleToRootsMap.get(module);
if (moduleRoots == null) {
moduleRoots = new ArrayList<RootDescriptor>();
myModuleToRootsMap.put(module, moduleRoots);
}
for (String r : module.getSourceRoots()) {
final File root = new File(FileUtil.toCanonicalPath(r));
final RootDescriptor descriptor = new RootDescriptor(module, root, false);
myRootToModuleMap.put(root, descriptor);
moduleRoots.add(descriptor);
}
for (String r : module.getTestRoots()) {
final File root = new File(FileUtil.toCanonicalPath(r));
final RootDescriptor descriptor = new RootDescriptor(module, root, true);
myRootToModuleMap.put(root, descriptor);
moduleRoots.add(descriptor);
}
}
}
public Project getProject() {
@@ -67,28 +88,21 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler
return myIsMake;
}
public boolean isDirty(ModuleChunk chunk) {
for (Module module : chunk.getModules()) {
if (isDirty(module)) {
return true;
}
public boolean isProjectRebuild() {
return myIsProjectRebuild;
}
public void markDirty(final File file) throws Exception {
final RootDescriptor descriptor = getModuleAndRoot(file);
if (descriptor != null) {
myFsState.markDirty(file, descriptor, myTsStorage);
}
return false;
}
public boolean isDirty(Module module) {
return myDirtyModules.contains(module);
}
public void setDirty(ModuleChunk chunk, boolean isDirty) {
public void markDirty(ModuleChunk chunk) throws Exception {
final Set<Module> modules = chunk.getModules();
if (!isDirty) {
myDirtyModules.removeAll(modules);
return;
}
myDirtyModules.addAll(modules);
final Set<Module> dirtyModules = new HashSet<Module>(modules);
// now mark all modules that depend on dirty modules
final ClasspathKind classpathKind = ClasspathKind.compile(isCompilingTests());
final ProjectChunks chunks = isCompilingTests()? myTestChunks : myProductionChunks;
@@ -102,22 +116,22 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler
else {
MODULES_LOOP: for (final Module module : moduleChunk.getModules()) {
for (ClasspathItem dependency : module.getClasspath(classpathKind)) {
if (dependency instanceof Module && isDirty((Module)dependency)) {
myDirtyModules.addAll(moduleChunk.getModules());
if (dependency instanceof Module && dirtyModules.contains((Module)dependency)) {
dirtyModules.addAll(moduleChunk.getModules());
break MODULES_LOOP;
}
}
}
}
}
for (Module module : dirtyModules) {
markDirtyFiles(module, myTsStorage, true, DirtyMarkScope.BOTH, null);
}
}
public Mappings getMappings() {
return myMappings;
}
public Mappings createDelta() {
return myMappings.createDelta();
return myDataManager.getMappings().createDelta();
}
public boolean isCompilingTests() {
@@ -128,30 +142,79 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler
myCompilingTests = compilingTests;
}
void onChunkBuildComplete(@NotNull ModuleChunk chunk) {
for (Module module : chunk.getModules()) {
myFilesCache.remove(module);
void beforeNextCompileRound(@NotNull ModuleChunk chunk) {
myFsState.beforeNextRoundStart();
}
void onChunkBuildComplete(@NotNull ModuleChunk chunk) throws Exception {
myDataManager.getMappings().clearMemoryCaches();
try {
if (!myErrorsFound) {
final boolean compilingTests = isCompilingTests();
for (Module module : chunk.getModules()) {
final List<RootDescriptor> roots = myModuleToRootsMap.get(module);
for (RootDescriptor descriptor : roots) {
if (compilingTests? descriptor.isTestRoot : !descriptor.isTestRoot) {
myFsState.markAllUpToDate(descriptor, myTsStorage, myCompilationStartStamp);
}
}
}
}
}
finally {
myFsState.clearRoundDeltas();
}
myMappings.clearMemoryCaches();
}
public CompileScope getScope() {
return myScope;
}
public BuildDataManager getBuildDataManager() {
public BuildDataManager getDataManager() {
return myDataManager;
}
public void processMessage(BuildMessage msg) {
if (msg.getKind() == BuildMessage.Kind.ERROR) {
myErrorsFound = true;
}
myDelegateMessageHandler.processMessage(msg);
}
public void processFiles(ModuleChunk chunk, FileProcessor processor) throws Exception {
public void processFilesToRecompile(ModuleChunk chunk, FileProcessor processor) throws Exception {
for (Module module : chunk.getModules()) {
final FSSnapshot snapshot = myFilesCache.get(module);
if (!snapshot.processFiles(processor)) {
return;
myFsState.processFilesToRecompile(module, isCompilingTests(), processor);
}
}
final void ensureFSStateInitialized(ModuleChunk chunk) throws Exception {
for (Module module : chunk.getModules()) {
if (isProjectRebuild()) {
markDirtyFiles(module, myTsStorage, true, isCompilingTests() ? DirtyMarkScope.TESTS : DirtyMarkScope.PRODUCTION, null);
}
else {
// in 'make' mode
// todo: consider situation when only several files are forced to be compiled => this is not project rebuild and not make
if (myFsState.markInitialScanPerformed(module, isCompilingTests())) {
initModuleFSState(module);
}
}
}
}
private void initModuleFSState(Module module) throws Exception {
final HashSet<File> currentFiles = new HashSet<File>();
markDirtyFiles(module, myTsStorage, false, isCompilingTests() ? DirtyMarkScope.TESTS : DirtyMarkScope.PRODUCTION, currentFiles);
final String moduleName = module.getName().toLowerCase(Locale.US);
final SourceToOutputMapping sourceToOutputMap = getDataManager().getSourceToOutputMap(moduleName, isCompilingTests());
for (final Iterator<String> it = sourceToOutputMap.getKeysIterator(); it.hasNext();) {
final String path = it.next();
// can check if the file exists
final File file = new File(path);
if (!currentFiles.contains(file)) {
myFsState.registerDeleted(module, path, isCompilingTests(), myTsStorage);
}
}
}
@@ -161,35 +224,81 @@ public class CompileContext extends UserDataHolderBase implements MessageHandler
return removed != null && !removed.isEmpty();
}
/** @noinspection unchecked*/
private FSSnapshot buildSnapshot(Module module) {
@Nullable
public RootDescriptor getModuleAndRoot(File file) {
File current = file;
while (current != null) {
final RootDescriptor descriptor = myRootToModuleMap.get(current);
if (descriptor != null) {
return descriptor;
}
current = FileUtil.getParentFile(current);
}
return null;
}
@NotNull
public List<RootDescriptor> getModuleRoots(Module module) {
final List<RootDescriptor> rootDescriptors = myModuleToRootsMap.get(module);
return rootDescriptors != null? Collections.unmodifiableList(rootDescriptors) : Collections.<RootDescriptor>emptyList();
}
private static enum DirtyMarkScope{
PRODUCTION, TESTS, BOTH
}
private void markDirtyFiles(Module module, final TimestampStorage tsStorage, final boolean forceMarkDirty, @NotNull final DirtyMarkScope scope, @Nullable final Set<File> currentFiles) throws Exception {
final Set<File> excludes = new HashSet<File>();
for (String excludePath : module.getExcludes()) {
excludes.add(new File(excludePath));
}
final FSSnapshot snapshot = new FSSnapshot(module);
final Collection<String> roots = myCompilingTests? module.getTestRoots() : module.getSourceRoots();
for (String srcRoot : roots) {
final String normalizedRoot = FileUtil.toCanonicalPath(srcRoot);
final File rootFile = new File(normalizedRoot);
if (rootFile.exists()) {
final FSSnapshot.Root root = snapshot.addRoot(rootFile, normalizedRoot);
buildStructure(root.getNode(), rootFile, excludes);
for (RootDescriptor rd : getModuleRoots(module)) {
if (scope == DirtyMarkScope.TESTS) {
if (!rd.isTestRoot) {
continue;
}
}
else if (scope == DirtyMarkScope.PRODUCTION) {
if (rd.isTestRoot) {
continue;
}
}
if (!rd.root.exists()) {
continue;
}
if (forceMarkDirty) {
myFsState.clearRecompile(rd);
myFsState.clearDeletedPaths(module, isCompilingTests());
}
traverseRecursively(rd, rd.root, excludes, tsStorage, forceMarkDirty, currentFiles);
}
return snapshot;
}
private static void buildStructure(final FSSnapshot.Node from, final File nodeFile, final Set<File> excluded) {
if (from.isDirectory()) {
if (!PathUtil.isUnder(excluded, nodeFile)) {
final File[] children = nodeFile.listFiles();
private void traverseRecursively(final RootDescriptor rd, final File file, Set<File> excludes, @NotNull final TimestampStorage tsStorage, final boolean forceDirty, @Nullable Set<File> currentFiles) throws Exception {
if (file.isDirectory()) {
if (!PathUtil.isUnder(excludes, file)) {
final File[] children = file.listFiles();
if (children != null) {
for (File child : children) {
buildStructure(from.addChild(child), child, excluded);
traverseRecursively(rd, child, excludes, tsStorage, forceDirty, currentFiles);
}
}
}
}
else {
boolean markDirty = forceDirty;
if (!markDirty) {
markDirty = tsStorage.getStamp(file) != file.lastModified();
}
if (markDirty) {
// if it is full project rebuild, all storages are already completely cleared;
// so passing null because there is no need to access the storage to clear non-existing data
final TimestampStorage _tsStorage = isProjectRebuild() ? null : tsStorage;
myFsState.markDirty(file, rd, _tsStorage);
}
if (currentFiles != null) {
currentFiles.add(file);
}
}
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/FSSnapshot.java
-96
View File
@@ -1,96 +0,0 @@
package org.jetbrains.jps.incremental;
import org.jetbrains.jps.Module;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* @author Eugene Zhuravlev
* Date: 10/8/11
*/
public class FSSnapshot {
private final List<Root> myRoots = new ArrayList<Root>();
private final Module myModule;
public FSSnapshot(Module module) {
myModule = module;
}
public Root addRoot(File file, String path) {
final Root root = new Root(new Node(file), path);
myRoots.add(root);
return root;
}
public boolean processFiles(FileProcessor processor) throws Exception {
for (Root root : myRoots) {
if (!processRecursively(root.node, root.path, processor, root.path)) {
return false;
}
}
return true;
}
private boolean processRecursively(Node from, String currentPath, FileProcessor processor, String srcRoot) throws Exception {
if (from.isDirectory()) {
for (Node child : from.children) {
if (!processRecursively(child, currentPath + "/" + child.getName(), processor, srcRoot)) {
return false;
}
}
return true;
}
return processor.apply(myModule, new File(currentPath), srcRoot);
}
static class Root {
private final Node node;
private final String path;
private Root(Node node, String path) {
this.node = node;
this.path = path;
}
public Node getNode() {
return node;
}
public String getPath() {
return path;
}
}
static class Node {
private final String myName;
private final boolean myIsDirectory;
private final List<Node> children = new ArrayList<Node>();
Node(File file) {
myName = file.getName();
myIsDirectory = file.isDirectory();
}
public Node addChild(File file) {
final Node node = new Node(file);
children.add(node);
return node;
}
public List<Node> getChildren() {
return Collections.unmodifiableList(children);
}
public boolean isDirectory() {
return myIsDirectory;
}
public String getName() {
return myName;
}
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/FSState.java
+174
View File
@@ -0,0 +1,174 @@
package org.jetbrains.jps.incremental;
import com.intellij.openapi.util.io.FileUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.Module;
import org.jetbrains.jps.incremental.storage.TimestampStorage;
import java.io.File;
import java.util.*;
/**
* @author Eugene Zhuravlev
* Date: 12/16/11
*/
public class FSState {
private final Map<Module, FilesDelta> myDeltas = Collections.synchronizedMap(new HashMap<Module, FilesDelta>());
private final Set<Module> myInitialTestsScanPerformed = Collections.synchronizedSet(new HashSet<Module>());
private final Set<Module> myInitialProductionScanPerformed = Collections.synchronizedSet(new HashSet<Module>());
private volatile FilesDelta myCurrentRoundDelta;
private volatile FilesDelta myLastRoundDelta;
public FSState() {
}
public boolean markInitialScanPerformed(Module module, boolean forTests) {
final Set<Module> map = forTests ? myInitialTestsScanPerformed : myInitialProductionScanPerformed;
// todo: temp comment
//return map.add(module);
return true;
}
public void beforeNextRoundStart() {
myLastRoundDelta = myCurrentRoundDelta;
myCurrentRoundDelta = new FilesDelta();
}
public void clearRoundDeltas() {
myCurrentRoundDelta = null;
myLastRoundDelta = null;
}
public void clearRecompile(RootDescriptor rd) {
getDelta(rd.module).clearRecompile(rd.root, rd.isTestRoot);
}
public void markDirty(final File file, final RootDescriptor rd, final @Nullable TimestampStorage tsStorage) throws Exception {
final FilesDelta roundDelta = myCurrentRoundDelta;
if (roundDelta != null) {
roundDelta.markRecompile(rd.root, rd.isTestRoot, file);
}
final FilesDelta mainDelta = getDelta(rd.module);
final boolean marked = mainDelta.markRecompile(rd.root, rd.isTestRoot, file);
if (marked && tsStorage != null) {
tsStorage.markDirty(file);
}
}
public void markAllUpToDate(final RootDescriptor rd, final TimestampStorage tsStorage, final long compilationStartStamp) throws Exception {
final FilesDelta delta = getDelta(rd.module);
final Set<File> files = delta.clearRecompile(rd.root, rd.isTestRoot);
if (files != null) {
for (File file : files) {
final long stamp = file.lastModified();
if (stamp > compilationStartStamp) {
// if the file was modified after the compilation had started,
// do not save the stamp considering file dirty
delta.markRecompile(rd.root, rd.isTestRoot, file);
}
else {
tsStorage.saveStamp(file, stamp);
}
}
}
}
public boolean processFilesToRecompile(final Module module, final boolean forTests, final FileProcessor processor) throws Exception {
final FilesDelta lastRoundDelta = myLastRoundDelta;
final FilesDelta delta = lastRoundDelta != null? lastRoundDelta : getDelta(module);
final Map<File, Set<File>> data = delta.getSourcesToRecompile(forTests);
synchronized (data) {
for (Map.Entry<File, Set<File>> entry : data.entrySet()) {
final String root = FileUtil.toSystemIndependentName(entry.getKey().getPath());
for (File file : entry.getValue()) {
if (!processor.apply(module, file, root)) {
return false;
}
}
}
}
return true;
}
public void registerDeleted(final Module module, final String path, final boolean isTest, @Nullable TimestampStorage tsStorage) throws Exception {
getDelta(module).addDeleted(path, isTest);
if (tsStorage != null) {
tsStorage.remove(new File(path));
}
}
public Collection<String> getDeletedPaths(final Module module, final boolean isTest) {
final FilesDelta delta = myDeltas.get(module);
if (delta == null) {
return Collections.emptyList();
}
return delta.getDeletedPaths(isTest);
}
public void clearDeletedPaths(final Module module, final boolean isTest) {
final FilesDelta delta = myDeltas.get(module);
if (delta != null) {
delta.clearDeletedPaths(isTest);
}
}
@NotNull
private FilesDelta getDelta(Module module) {
synchronized (myDeltas) {
FilesDelta delta = myDeltas.get(module);
if (delta == null) {
delta = new FilesDelta();
myDeltas.put(module, delta);
}
return delta;
}
}
private static final class FilesDelta {
private final Set<String> myDeletedProduction = Collections.synchronizedSet(new HashSet<String>());
private final Set<String> myDeletedTests = Collections.synchronizedSet(new HashSet<String>());
private final Map<File, Set<File>> mySourcesToRecompile = Collections.synchronizedMap(new HashMap<File, Set<File>>()); // srcRoot -> set of sources
private final Map<File, Set<File>> myTestsToRecompile = Collections.synchronizedMap(new HashMap<File, Set<File>>()); // srcRoot -> set of sources
public boolean markRecompile(File root, boolean isTestRoot, File file) {
final Map<File, Set<File>> toRecompile = isTestRoot ? myTestsToRecompile : mySourcesToRecompile;
Set<File> files;
synchronized (toRecompile) {
files = toRecompile.get(root);
if (files == null) {
files = new HashSet<File>();
toRecompile.put(root, files);
}
}
return files.add(file);
}
public void addDeleted(String path, boolean isTest) {
final Set<String> map = isTest? myDeletedTests : myDeletedProduction;
map.add(path);
}
public void clearDeletedPaths(boolean isTest) {
final Set<String> map = isTest? myDeletedTests : myDeletedProduction;
map.clear();
}
public Map<File, Set<File>> getSourcesToRecompile(boolean forTests) {
return forTests? myTestsToRecompile : mySourcesToRecompile;
}
public Set<String> getDeletedPaths(boolean isTest) {
final Set<String> set = isTest ? myDeletedTests : myDeletedProduction;
synchronized (set) {
return new HashSet<String>(set);
}
}
@Nullable
public Set<File> clearRecompile(File root, boolean isTestRoot) {
return isTestRoot? myTestsToRecompile.remove(root) : mySourcesToRecompile.remove(root);
}
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java
+131 -130
View File
@@ -2,12 +2,14 @@ package org.jetbrains.jps.incremental;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.io.PersistentEnumerator;
import org.jetbrains.ether.dependencyView.Mappings;
import org.jetbrains.jps.*;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.messages.ProgressMessage;
import org.jetbrains.jps.incremental.storage.OutputToSourceMapping;
import org.jetbrains.jps.incremental.storage.SourceToFormMapping;
import org.jetbrains.jps.incremental.storage.SourceToOutputMapping;
import org.jetbrains.jps.incremental.storage.TimestampStorage;
import org.jetbrains.jps.server.ProjectDescriptor;
import java.io.File;
import java.io.IOException;
@@ -21,64 +23,65 @@ import java.util.*;
public class IncProjectBuilder {
public static final String JPS_SERVER_NAME = "JPS BUILD";
private final String myProjectName;
private final ProjectDescriptor myProjectDescriptor;
private final BuilderRegistry myBuilderRegistry;
private ProjectChunks myProductionChunks;
private ProjectChunks myTestChunks;
private final List<MessageHandler> myMessageHandlers = new ArrayList<MessageHandler>();
private final Mappings myMappings;
private final MessageHandler myMessageDispatcher = new MessageHandler() {
public void processMessage(BuildMessage msg) {
for (MessageHandler h : myMessageHandlers) {
h.processMessage(msg);
}
}
};
public IncProjectBuilder(String projectName,
Project project,
final Mappings mappings,
BuilderRegistry builderRegistry) {
myProjectName = projectName;
public IncProjectBuilder(ProjectDescriptor pd, BuilderRegistry builderRegistry) {
myProjectDescriptor = pd;
myBuilderRegistry = builderRegistry;
myProductionChunks = new ProjectChunks(project, ClasspathKind.PRODUCTION_COMPILE);
myTestChunks = new ProjectChunks(project, ClasspathKind.TEST_COMPILE);
myMappings = mappings;
myProductionChunks = new ProjectChunks(pd.project, ClasspathKind.PRODUCTION_COMPILE);
myTestChunks = new ProjectChunks(pd.project, ClasspathKind.TEST_COMPILE);
}
public void addMessageHandler(MessageHandler handler) {
myMessageHandlers.add(handler);
}
// todo: pass dirty and removed sources from outside
public void build(CompileScope scope, final boolean isMake) {
final CompileContext context = createContext(scope, isMake);
public void build(CompileScope scope, final boolean isMake, final boolean isProjectRebuild) {
CompileContext context = null;
try {
try {
context = createContext(scope, isMake, isProjectRebuild);
runBuild(context);
}
catch (ProjectBuildException e) {
if (e.getCause() instanceof PersistentEnumerator.CorruptedException) {
// force rebuild
context.processMessage(new CompilerMessage(
myMessageDispatcher.processMessage(new CompilerMessage(
JPS_SERVER_NAME, BuildMessage.Kind.INFO,
"Internal caches are corrupted or have outdated format, forcing project rebuild: " + e.getMessage())
);
runBuild(createContext(new CompileScope(scope.getProject()), false));
context = createContext(new CompileScope(scope.getProject()), false, true);
runBuild(context);
}
else {
throw e;
}
}
}
catch (ProjectBuildException e) {
final Throwable cause = e.getCause();
if (cause == null) {
context.processMessage(new ProgressMessage(e.getMessage()));
myMessageDispatcher.processMessage(new ProgressMessage(e.getMessage()));
}
else {
context.processMessage(new CompilerMessage(JPS_SERVER_NAME, cause));
myMessageDispatcher.processMessage(new CompilerMessage(JPS_SERVER_NAME, cause));
}
}
finally {
context.getBuildDataManager().close();
if (context != null) {
context.getDataManager().close();
}
cleanupJavacNameTable();
}
}
@@ -95,7 +98,7 @@ public class IncProjectBuilder {
}
private void runBuild(CompileContext context) throws ProjectBuildException {
if (!context.isMake()) {
if (context.isProjectRebuild()) {
cleanOutputRoots(context);
}
@@ -114,85 +117,70 @@ public class IncProjectBuilder {
runTasks(context, myBuilderRegistry.getAfterTasks());
}
private CompileContext createContext(CompileScope scope, boolean isMake) {
private CompileContext createContext(CompileScope scope, boolean isMake, final boolean isProjectRebuild) throws ProjectBuildException {
final String projectName = myProjectDescriptor.projectName;
final TimestampStorage tsStorage = myProjectDescriptor.timestamps.getStorage();
final FSState fsState = myProjectDescriptor.fsState;
return new CompileContext(
scope, myProjectName, isMake, myMappings, myProductionChunks, myTestChunks, new MessageHandler() {
public void processMessage(BuildMessage msg) {
for (MessageHandler h : myMessageHandlers) {
h.processMessage(msg);
}
}
});
projectName, scope, isMake, isProjectRebuild, myProductionChunks, myTestChunks, fsState, tsStorage, myMessageDispatcher
);
}
private static void cleanOutputRoots(CompileContext context) throws ProjectBuildException {
final CompileScope scope = context.getScope();
final Collection<Module> allProjectModules = scope.getProject().getModules().values();
final Collection<Module> modulesToClean = new HashSet<Module>(scope.getAffectedModules());
// whole project is affected
try {
context.getDataManager().clean();
}
catch (IOException e) {
throw new ProjectBuildException("Error cleaning compiler storages", e);
}
final Set<Module> allModules = new HashSet<Module>(allProjectModules);
allModules.removeAll(modulesToClean);
if (allModules.isEmpty()) {
// whole project is affected
context.getBuildDataManager().clean();
try {
context.getMappings().clean();
final Collection<Module> modulesToClean = context.getProject().getModules().values();
final Set<File> toDelete = new HashSet<File>();
final Set<File> allSourceRoots = new HashSet<File>();
for (Module module : modulesToClean) {
final File out = context.getProjectPaths().getModuleOutputDir(module, false);
if (out != null) {
toDelete.add(out);
}
catch (IOException e) {
throw new ProjectBuildException(e);
final File testOut = context.getProjectPaths().getModuleOutputDir(module, true);
if (testOut != null) {
toDelete.add(testOut);
}
final List<RootDescriptor> moduleRoots = context.getModuleRoots(module);
for (RootDescriptor d : moduleRoots) {
allSourceRoots.add(d.root);
}
}
if (!modulesToClean.isEmpty()) {
final Set<File> toDelete = new HashSet<File>();
final Set<File> allSourceRoots = new HashSet<File>();
for (Module module : modulesToClean) {
final File out = context.getProjectPaths().getModuleOutputDir(module, false);
if (out != null) {
toDelete.add(out);
}
final File testOut = context.getProjectPaths().getModuleOutputDir(module, true);
if (testOut != null) {
toDelete.add(testOut);
}
// check that output and source roots are not overlapping
for (File outputRoot : toDelete) {
boolean okToDelete = true;
if (PathUtil.isUnder(allSourceRoots, outputRoot)) {
okToDelete = false;
}
for (Module module : allProjectModules) {
for (Object root : module.getSourceRoots()) {
allSourceRoots.add(new File((String)root));
}
for (Object root : module.getTestRoots()) {
allSourceRoots.add(new File((String)root));
}
}
// check that output and source roots are not overlapping
for (File outputRoot : toDelete) {
boolean okToDelete = true;
if (PathUtil.isUnder(allSourceRoots, outputRoot)) {
okToDelete = false;
}
else {
final Set<File> _outRoot = Collections.singleton(outputRoot);
for (File srcRoot : allSourceRoots) {
if (PathUtil.isUnder(_outRoot, srcRoot)) {
okToDelete = false;
break;
}
else {
final Set<File> _outRoot = Collections.singleton(outputRoot);
for (File srcRoot : allSourceRoots) {
if (PathUtil.isUnder(_outRoot, srcRoot)) {
okToDelete = false;
break;
}
}
if (okToDelete) {
context.processMessage(new ProgressMessage("Cleaning " + outputRoot.getPath()));
// do not delete output root itself to avoid lots of unnecessary "roots_changed" events in IDEA
final File[] children = outputRoot.listFiles();
if (children != null) {
for (File child : children) {
FileUtil.delete(child);
}
}
if (okToDelete) {
context.processMessage(new ProgressMessage("Cleaning " + outputRoot.getPath()));
// do not delete output root itself to avoid lots of unnecessary "roots_changed" events in IDEA
final File[] children = outputRoot.listFiles();
if (children != null) {
for (File child : children) {
FileUtil.delete(child);
}
}
else {
context.processMessage(new CompilerMessage(JPS_SERVER_NAME, BuildMessage.Kind.WARNING, "Output path " + outputRoot.getPath() + " intersects with a source root. The output cannot be cleaned."));
}
}
else {
context.processMessage(new CompilerMessage(JPS_SERVER_NAME, BuildMessage.Kind.WARNING, "Output path " + outputRoot.getPath() + " intersects with a source root. The output cannot be cleaned."));
}
}
}
@@ -214,25 +202,47 @@ public class IncProjectBuilder {
private void buildChunk(CompileContext context, ModuleChunk chunk) throws ProjectBuildException{
try {
// TODO: check how the output-source storage is filled and!
context.ensureFSStateInitialized(chunk);
if (context.isMake()) {
// cleanup outputs
final Set<String> allChunkRemovedSources = new HashSet<String>();
final SourceToFormMapping sourceToFormMap = context.getDataManager().getSourceToFormMap();
// todo: use removed paths passed from IDEA?
final OutputToSourceMapping storage = context.getBuildDataManager().getOutputToSourceStorage();
final HashSet<String> allChunkRemovedSources = new HashSet<String>();
for (Module module : chunk.getModules()) {
final File moduleOutput = context.getProjectPaths().getModuleOutputDir(module, context.isCompilingTests());
if (moduleOutput != null && moduleOutput.exists()) {
deleteOutputsOfRemovedSources(moduleOutput, storage, allChunkRemovedSources);
final Collection<String> deletedPaths = myProjectDescriptor.fsState.getDeletedPaths(module, context.isCompilingTests());
allChunkRemovedSources.addAll(deletedPaths);
final String moduleName = module.getName().toLowerCase(Locale.US);
final SourceToOutputMapping sourceToOutputStorage = context.getDataManager().getSourceToOutputMap(moduleName, context.isCompilingTests());
// actually delete outputs associated with removed paths
for (String deletedSource : deletedPaths) {
// deleting outputs corresponding to non-existing source
final Collection<String> outputs = sourceToOutputStorage.getState(deletedSource);
if (outputs != null) {
for (String output : outputs) {
FileUtil.delete(new File(output));
}
sourceToOutputStorage.remove(deletedSource);
}
// check if deleted source was associated with a form
final String formPath = sourceToFormMap.getState(deletedSource);
if (formPath != null) {
final File formFile = new File(formPath);
if (formFile.exists()) {
context.markDirty(formFile);
}
sourceToFormMap.remove(deletedSource);
}
}
}
Paths.CHUNK_REMOVED_SOURCES_KEY.set(context, allChunkRemovedSources);
for (Module module : chunk.getModules()) {
myProjectDescriptor.fsState.clearDeletedPaths(module, context.isCompilingTests());
}
}
for (BuilderCategory category : BuilderCategory.values()) {
runBuilders(context, chunk, myBuilderRegistry.getBuilders(category));
runBuilders(context, chunk, category);
}
}
catch (ProjectBuildException e) {
@@ -242,20 +252,36 @@ public class IncProjectBuilder {
throw new ProjectBuildException(e);
}
finally {
for (BuilderCategory category : BuilderCategory.values()) {
for (Builder builder : myBuilderRegistry.getBuilders(category)) {
builder.cleanupResources(context, chunk);
try {
for (BuilderCategory category : BuilderCategory.values()) {
for (Builder builder : myBuilderRegistry.getBuilders(category)) {
builder.cleanupResources(context, chunk);
}
}
}
finally {
try {
context.onChunkBuildComplete(chunk);
}
catch (Exception e) {
throw new ProjectBuildException(e);
}
finally {
Paths.CHUNK_REMOVED_SOURCES_KEY.set(context, null);
}
}
context.onChunkBuildComplete(chunk);
Paths.CHUNK_REMOVED_SOURCES_KEY.set(context, null);
}
}
private static void runBuilders(CompileContext context, ModuleChunk chunk, List<Builder> builders) throws ProjectBuildException {
private void runBuilders(CompileContext context, ModuleChunk chunk, BuilderCategory category) throws ProjectBuildException {
final List<Builder> builders = myBuilderRegistry.getBuilders(category);
if (builders.isEmpty()) {
return;
}
boolean nextPassRequired;
do {
nextPassRequired = false;
context.beforeNextCompileRound(chunk);
for (Builder builder : builders) {
final Builder.ExitCode buildResult = builder.build(context, chunk);
if (buildResult == Builder.ExitCode.ABORT) {
@@ -268,29 +294,4 @@ public class IncProjectBuilder {
}
while (nextPassRequired);
}
private static void deleteOutputsOfRemovedSources(File file, final OutputToSourceMapping outputToSourceStorage, Set<String> removedSources) throws Exception {
if (file.isDirectory()) {
final File[] files = file.listFiles();
if (files != null) {
for (File child : files) {
deleteOutputsOfRemovedSources(child, outputToSourceStorage, removedSources);
}
}
}
else {
final String outPath = file.getPath();
final String srcPath = outputToSourceStorage.getState(outPath);
if (srcPath != null) { // if we know about the association
final File srcFile = new File(srcPath);
// todo: optimize
if (!srcFile.exists()) {
removedSources.add(srcPath);
FileUtil.delete(file);
outputToSourceStorage.remove(outPath);
}
}
}
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/Paths.java
-12
View File
@@ -37,18 +37,6 @@ public class Paths {
return new File(getInstance().mySystemRoot, projectName.toLowerCase(Locale.US));
}
public static File getBuilderDataRoot(final String projectName, String builderName) {
return new File(getDataStorageRoot(projectName), builderName.toLowerCase(Locale.US));
}
public static File getMappingsStorageFile(final String projectName) {
return new File(getDataStorageRoot(projectName), "mappings/data");
}
public static File getMappingsStorageRoot(final String projectName) {
return new File(getDataStorageRoot(projectName), "mappings");
}
public static URI toURI(String localPath) {
try {
String p = FileUtil.toSystemIndependentName(localPath);
jps/jps-builders/src/org/jetbrains/jps/incremental/RootDescriptor.java
+24
View File
@@ -0,0 +1,24 @@
package org.jetbrains.jps.incremental;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.Module;
import java.io.File;
/**
* @author Eugene Zhuravlev
* Date: 1/3/12
*/
public final class RootDescriptor {
@NotNull
public final Module module;
@NotNull
public final File root;
public final boolean isTestRoot;
RootDescriptor(@NotNull Module module, @NotNull File root, boolean isTestRoot) {
this.module = module;
this.root = root;
this.isTestRoot = isTestRoot;
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/groovy/GroovyBuilder.java
+35 -33
View File
@@ -15,8 +15,7 @@ import org.jetbrains.jps.incremental.java.JavaBuilder;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.messages.ProgressMessage;
import org.jetbrains.jps.incremental.storage.OutputToSourceMapping;
import org.jetbrains.jps.incremental.storage.TimestampStorage;
import org.jetbrains.jps.incremental.storage.SourceToOutputMapping;
import org.jetbrains.jps.server.ClasspathBootstrap;
import org.objectweb.asm.ClassReader;
@@ -43,16 +42,14 @@ public class GroovyBuilder extends Builder {
public Builder.ExitCode build(final CompileContext context, ModuleChunk chunk) throws ProjectBuildException {
ExitCode exitCode = ExitCode.OK;
final List<File> toCompile = new ArrayList<File>();
final Map<File, Module> toCompile = new HashMap<File, Module>();
try {
final TimestampStorage tsStorage = context.getBuildDataManager().getTimestampStorage(getName());
// todo: use dirty files passed from outside
context.processFiles(chunk, new FileProcessor() {
context.processFilesToRecompile(chunk, new FileProcessor() {
@Override
public boolean apply(Module module, File file, String sourceRoot) throws Exception {
final String path = file.getPath();
if (isGroovyFile(path) && isFileDirty(file, context, tsStorage)) { //todo file type check
toCompile.add(file);
if (isGroovyFile(path)) { //todo file type check
toCompile.put(file, module);
}
return true;
}
@@ -80,22 +77,22 @@ public class GroovyBuilder extends Builder {
final File dir = myForStubs ? FileUtil.createTempDirectory(/*new File("/tmp/stubs/"), */"groovyStubs", null) : moduleOutputDir;
assert dir != null;
Set<String> toCompilePaths = new LinkedHashSet<String>();
for (File file : toCompile) {
toCompilePaths.add(file.getPath());
final Set<String> toCompilePaths = new LinkedHashSet<String>();
for (File file : toCompile.keySet()) {
toCompilePaths.add(FileUtil.toSystemIndependentName(file.getPath()));
}
String moduleOutputPath = FileUtil.toCanonicalPath(moduleOutputDir.getPath());
if (!moduleOutputPath.endsWith("/")) {
moduleOutputPath += "/";
}
Map<String, String> class2Src = buildClassToSourceMap(context, toCompilePaths, moduleOutputPath);
Map<String, String> class2Src = buildClassToSourceMap(chunk, context, toCompilePaths, moduleOutputPath);
String encoding = "UTF-8"; //todo encoding
List<String> patchers = Collections.emptyList(); //todo patchers
GroovycOSProcessHandler.fillFileWithGroovycParameters(tempFile, FileUtil.toCanonicalPath(dir.getPath()), toCompilePaths,
moduleOutputPath, class2Src,
encoding, patchers);
GroovycOSProcessHandler.fillFileWithGroovycParameters(
tempFile, FileUtil.toCanonicalPath(dir.getPath()), toCompilePaths, moduleOutputPath, class2Src, encoding, patchers
);
if (myForStubs) {
JavaBuilder.addTempSourcePathRoot(context, dir);
@@ -112,7 +109,7 @@ public class GroovyBuilder extends Builder {
Arrays.<String>asList(myForStubs ? "stubs" : "groovyc", tempFile.getPath())
);
deleteCorrespondingClasses(context, toCompile);
deleteCorrespondingOutputFiles(context, toCompile);
List<GroovycOSProcessHandler.OutputItem> successfullyCompiled = Collections.emptyList();
try {
@@ -146,28 +143,26 @@ public class GroovyBuilder extends Builder {
}
}
finally {
if (myForStubs) {
for (GroovycOSProcessHandler.OutputItem item : successfullyCompiled) {
tsStorage.saveStamp(new File(item.sourcePath));
}
}
else {
if (!myForStubs) {
final Mappings delta = context.createDelta();
final List<File> successfullyCompiledFiles = new ArrayList<File>();
if (!successfullyCompiled.isEmpty()) {
final Callbacks.Backend callback = delta.getCallback();
final OutputToSourceMapping storage = context.getBuildDataManager().getOutputToSourceStorage();
for (GroovycOSProcessHandler.OutputItem item : successfullyCompiled) {
final String sourcePath = FileUtil.toSystemIndependentName(item.sourcePath);
final String outputPath = FileUtil.toSystemIndependentName(item.outputPath);
storage.update(outputPath, sourcePath);
final RootDescriptor moduleAndRoot = context.getModuleAndRoot(new File(sourcePath));
if (moduleAndRoot != null) {
final String moduleName = moduleAndRoot.module.getName().toLowerCase(Locale.US);
context.getDataManager().getSourceToOutputMap(moduleName, moduleAndRoot.isTestRoot).appendData(sourcePath, outputPath);
}
callback.associate(outputPath, Callbacks.getDefaultLookup(sourcePath), new ClassReader(FileUtil.loadFileBytes(new File(outputPath))));
successfullyCompiledFiles.add(new File(sourcePath));
}
}
final boolean needSecondPass = updateMappings(context, delta, chunk, toCompile, successfullyCompiledFiles);
final boolean needSecondPass = updateMappings(context, delta, chunk, toCompile.keySet(), successfullyCompiledFiles);
if (needSecondPass) {
exitCode = ExitCode.ADDITIONAL_PASS_REQUIRED;
}
@@ -185,15 +180,22 @@ public class GroovyBuilder extends Builder {
return path.endsWith(".groovy") || path.endsWith(".gpp");
}
private static Map<String, String> buildClassToSourceMap(CompileContext context, Set<String> toCompilePaths, String moduleOutputPath)
throws Exception {
Map<String, String> class2Src = new HashMap<String, String>();
for (String out : context.getBuildDataManager().getOutputToSourceStorage().getKeys()) {
if (out.endsWith(".class") && out.startsWith(moduleOutputPath)) {
String src = context.getBuildDataManager().getOutputToSourceStorage().getState(out);
private static Map<String, String> buildClassToSourceMap(ModuleChunk chunk, CompileContext context, Set<String> toCompilePaths, String moduleOutputPath) throws Exception {
final Map<String, String> class2Src = new HashMap<String, String>();
for (Module module : chunk.getModules()) {
final String moduleName = module.getName().toLowerCase(Locale.US);
final SourceToOutputMapping srcToOut = context.getDataManager().getSourceToOutputMap(moduleName, context.isCompilingTests());
for (String src : srcToOut.getKeys()) {
if (!toCompilePaths.contains(src) && isGroovyFile(src)) {
String className = out.substring(moduleOutputPath.length(), out.length() - ".class".length()).replace('/', '.');
class2Src.put(className, src);
final Collection<String> outs = srcToOut.getState(src);
if (outs != null) {
for (String out : outs) {
if (out.endsWith(".class") && out.startsWith(moduleOutputPath)) {
final String className = out.substring(moduleOutputPath.length(), out.length() - ".class".length()).replace('/', '.');
class2Src.put(className, src);
}
}
}
}
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java
+50 -64
View File
@@ -16,14 +16,12 @@ import org.jetbrains.ether.dependencyView.Mappings;
import org.jetbrains.jps.Module;
import org.jetbrains.jps.ModuleChunk;
import org.jetbrains.jps.ProjectPaths;
import org.jetbrains.jps.incremental.Builder;
import org.jetbrains.jps.incremental.CompileContext;
import org.jetbrains.jps.incremental.FileProcessor;
import org.jetbrains.jps.incremental.ProjectBuildException;
import org.jetbrains.jps.incremental.*;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.messages.ProgressMessage;
import org.jetbrains.jps.incremental.storage.TimestampStorage;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.incremental.storage.SourceToFormMapping;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.Opcodes;
@@ -74,16 +72,21 @@ public class JavaBuilder extends Builder{
if (srcFile != null && content != null) {
final String outputPath = FileUtil.toSystemIndependentName(out.getFile().getPath());
final String sourcePath = FileUtil.toSystemIndependentName(srcFile.getPath());
try {
context.getBuildDataManager().getOutputToSourceStorage().update(outputPath, sourcePath);
}
catch (Exception e) {
context.processMessage(new CompilerMessage(BUILDER_NAME, e));
final RootDescriptor moduleAndRoot = context.getModuleAndRoot(srcFile);
final BuildDataManager dataManager = context.getDataManager();
if (moduleAndRoot != null) {
try {
final String moduleName = moduleAndRoot.module.getName().toLowerCase(Locale.US);
dataManager.getSourceToOutputMap(moduleName, context.isCompilingTests()).appendData(sourcePath, outputPath);
}
catch (Exception e) {
context.processMessage(new CompilerMessage(BUILDER_NAME, e));
}
}
final ClassReader reader = new ClassReader(content.getBuffer(), content.getOffset(), content.getLength());
// todo: the callback is not thread-safe?
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (context.getMappings()) {
synchronized (dataManager.getMappings()) {
// todo: parse class data out of synchronized block (move it from the 'associate' implementation)
callback.associate(outputPath, Callbacks.getDefaultLookup(sourcePath), reader);
}
@@ -115,64 +118,53 @@ public class JavaBuilder extends Builder{
public ExitCode build(final CompileContext context, final ModuleChunk chunk) throws ProjectBuildException {
try {
final TimestampStorage tsStorage = context.getBuildDataManager().getTimestampStorage(BUILDER_NAME);
final Set<File> filesToCompile = new LinkedHashSet<File>();
final Map<File, Module> filesToCompile = new HashMap<File, Module>();
final List<File> formsToCompile = new ArrayList<File>();
final List<File> upToDateForms = new ArrayList<File>();
// todo: read srcRoots from JPS model
final Set<String> srcRoots = new HashSet<String>();
final boolean wholeModuleRebuildRequired = context.isDirty(chunk);
// todo: only process all sources if wholeModuleRebuild == true
context.processFiles(chunk, new FileProcessor() {
context.processFilesToRecompile(chunk, new FileProcessor() {
public boolean apply(Module module, File file, String sourceRoot) throws Exception {
if (JAVA_SOURCES_FILTER.accept(file)) {
srcRoots.add(sourceRoot);
if (wholeModuleRebuildRequired || isFileDirty(file, context, tsStorage)) {
filesToCompile.add(file);
}
filesToCompile.put(file, module);
}
else if (FORM_SOURCES_FILTER.accept(file)){
if (wholeModuleRebuildRequired || isFileDirty(file, context, tsStorage)) {
formsToCompile.add(file);
}
else {
upToDateForms.add(file);
}
else if (FORM_SOURCES_FILTER.accept(file)) {
formsToCompile.add(file);
}
return true;
}
});
// todo: change logic below so that complete forms list is not required
// force compilation of bound source file if the form is dirty
for (File form : formsToCompile) {
for (String root : srcRoots) {
final File boundSource = getBoundSource(root, form);
if (boundSource != null) {
// force compilation of classes that modified forms are bound to
filesToCompile.add(boundSource);
break;
final RootDescriptor descriptor = context.getModuleAndRoot(form);
if (descriptor != null) {
for (RootDescriptor rd : context.getModuleRoots(descriptor.module)) {
final File boundSource = getBoundSource(rd.root, form);
if (boundSource != null) {
filesToCompile.put(boundSource, rd.module);
break;
}
}
}
}
// form should be considered dirty if the class it is bound to is also dirty!
for (File form : upToDateForms) {
for (String root : srcRoots) {
final File boundSource = getBoundSource(root, form);
if (boundSource != null && filesToCompile.contains(boundSource)) {
formsToCompile.add(form);
break;
// form should be considered dirty if the class it is bound to is dirty
final SourceToFormMapping sourceToFormMap = context.getDataManager().getSourceToFormMap();
for (File srcFile : filesToCompile.keySet()) {
final String srcPath = srcFile.getPath();
final String formPath = sourceToFormMap.getState(srcPath);
if (formPath != null) {
final File formFile = new File(formPath);
if (formFile.exists()) {
context.markDirty(formFile);
formsToCompile.add(formFile);
}
sourceToFormMap.remove(srcPath);
}
}
upToDateForms.clear();
deleteCorrespondingClasses(context, filesToCompile);
deleteCorrespondingOutputFiles(context, filesToCompile);
return compile(context, chunk, filesToCompile, formsToCompile);
return compile(context, chunk, filesToCompile.keySet(), formsToCompile);
}
catch (ProjectBuildException e) {
throw e;
@@ -190,7 +182,7 @@ public class JavaBuilder extends Builder{
}
@Nullable
private static File getBoundSource(String srcRoot, File formFile) throws IOException {
private static File getBoundSource(File srcRoot, File formFile) throws IOException {
final String boundClassName = FormsParsing.readBoundClassName(formFile);
if (boundClassName == null) {
return null;
@@ -226,15 +218,14 @@ public class JavaBuilder extends Builder{
// todo: consider corresponding setting in CompilerWorkspaceConfiguration
final boolean addNotNullAssertions = true;
final Collection<File> classpath = paths.getCompilationClasspath(chunk, context.isCompilingTests(), !context.isMake());
final Collection<File> platformCp = paths.getPlatformCompilationClasspath(chunk, context.isCompilingTests(), !context.isMake());
final Collection<File> classpath = paths.getCompilationClasspath(chunk, context.isCompilingTests(), context.isProjectRebuild());
final Collection<File> platformCp = paths.getPlatformCompilationClasspath(chunk, context.isCompilingTests(), context.isProjectRebuild());
final Map<File, Set<File>> outs = buildOutputDirectoriesMap(context, chunk);
final List<String> options = getCompilationOptions(context, chunk);
// begin compilation round
final DiagnosticSink diagnosticSink = new DiagnosticSink(context);
final OutputFilesSink outputSink = new OutputFilesSink(context);
Collection<File> successfulForms = Collections.emptyList();
try {
if (hasSourcesToCompile) {
final Set<File> sourcePath = TEMPORARY_SOURCE_ROOTS_KEY.get(context,Collections.<File>emptySet());
@@ -249,7 +240,7 @@ public class JavaBuilder extends Builder{
if (!forms.isEmpty()) {
try {
context.processMessage(new ProgressMessage("Instrumenting forms [" + chunk.getName() + "]"));
successfulForms = instrumentForms(context, chunk, chunkSourcePath, compiledClassesLoader, forms, outputSink);
instrumentForms(context, chunk, chunkSourcePath, compiledClassesLoader, forms, outputSink);
}
finally {
context.processMessage(new ProgressMessage("Finished instrumenting forms [" + chunk.getName() + "]"));
@@ -275,8 +266,6 @@ public class JavaBuilder extends Builder{
}
}
finally {
context.setDirty(chunk, false); // no matter what result was, we should clear this flag
outputSink.writePendingData();
final Set<File> successfullyCompiled = outputSink.getSuccessfullyCompiled();
@@ -285,11 +274,6 @@ public class JavaBuilder extends Builder{
if (updateMappings(context, delta, chunk, files, successfullyCompiled)) {
exitCode = ExitCode.ADDITIONAL_PASS_REQUIRED;
}
final TimestampStorage tsStorage = context.getBuildDataManager().getTimestampStorage(BUILDER_NAME);
for (File file : successfulForms) {
tsStorage.saveStamp(file);
}
}
if (exitCode != ExitCode.ADDITIONAL_PASS_REQUIRED) {
@@ -385,7 +369,7 @@ public class JavaBuilder extends Builder{
}
}
private static Collection<File> instrumentForms(
private static void instrumentForms(
CompileContext context,
ModuleChunk chunk,
final Map<File, String> chunkSourcePath,
@@ -394,7 +378,7 @@ public class JavaBuilder extends Builder{
OutputFilesSink outputSink) throws ProjectBuildException {
final Map<String, File> class2form = new HashMap<String, File>();
final List<File> successfullForms = new ArrayList<File>();
final SourceToFormMapping sourceToFormMap = context.getDataManager().getSourceToFormMap();
final Map<String, OutputFileObject> compiledClassNames = new HashMap<String, OutputFileObject>();
for (OutputFileObject fileObject : outputSink.getFileObjects()) {
@@ -472,7 +456,10 @@ public class JavaBuilder extends Builder{
context.processMessage(new CompilerMessage(BUILDER_NAME, BuildMessage.Kind.ERROR, message.toString()));
}
else {
successfullForms.add(formFile);
final File sourceFile = outputClassFile.getSourceFile();
if (sourceFile != null) {
sourceToFormMap.update(sourceFile.getPath(), formFile.getPath());
}
}
}
catch (Exception e) {
@@ -485,7 +472,6 @@ public class JavaBuilder extends Builder{
}
}
}
return successfullForms;
}
private static OutputFileObject findClassFile(Map<String, OutputFileObject> outputs, String classToBind) {
jps/jps-builders/src/org/jetbrains/jps/incremental/resourses/ResourcesBuilder.java
+14 -18
View File
@@ -9,11 +9,11 @@ import org.jetbrains.jps.incremental.*;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.messages.ProgressMessage;
import org.jetbrains.jps.incremental.storage.OutputToSourceMapping;
import org.jetbrains.jps.incremental.storage.TimestampStorage;
import org.jetbrains.jps.incremental.storage.SourceToOutputMapping;
import java.io.File;
import java.io.IOException;
import java.util.Locale;
/**
* @author Eugene Zhuravlev
@@ -47,25 +47,21 @@ public class ResourcesBuilder extends Builder{
ResourcePatterns.KEY.set(context, patterns = new ResourcePatterns(context.getProject()));
}
try {
final TimestampStorage tsStorage = context.getBuildDataManager().getTimestampStorage(BUILDER_NAME);
final OutputToSourceMapping outputToSourceMapping = context.getBuildDataManager().getOutputToSourceStorage();
final ResourcePatterns finalPatterns = patterns;
// todo: process all files in case of rebuild or wholeModuleDirty
// todo: otherwise avoid traverwing the whole module and use dirty file list taken from params
context.processFiles(chunk, new FileProcessor() {
context.processFilesToRecompile(chunk, new FileProcessor() {
public boolean apply(final Module module, final File file, final String sourceRoot) throws Exception {
if (finalPatterns.isResourceFile(file, sourceRoot)) {
if (isFileDirty(file, context, tsStorage)) {
try {
context.processMessage(new ProgressMessage("Copying " + file.getPath()));
copyResource(context, module, file, sourceRoot, outputToSourceMapping);
}
catch (IOException e) {
LOG.info(e);
context.processMessage(new CompilerMessage("Resource Compiler", BuildMessage.Kind.ERROR, e.getMessage(), FileUtil.toSystemIndependentName(file.getPath())));
return false;
}
tsStorage.saveStamp(file);
try {
context.processMessage(new ProgressMessage("Copying " + file.getPath()));
final String moduleName = module.getName().toLowerCase(Locale.US);
copyResource(context, module, file, sourceRoot, context.getDataManager().getSourceToOutputMap(moduleName, context.isCompilingTests()));
}
catch (IOException e) {
LOG.info(e);
context.processMessage(new CompilerMessage("Resource Compiler", BuildMessage.Kind.ERROR, e.getMessage(), FileUtil.toSystemIndependentName(file.getPath())));
return false;
}
}
return true;
@@ -83,7 +79,7 @@ public class ResourcesBuilder extends Builder{
Module module,
File file,
String sourceRoot,
final OutputToSourceMapping outputToSourceMapping) throws Exception {
final SourceToOutputMapping outputToSourceMapping) throws Exception {
final String outputRoot = context.isCompilingTests() ? module.getTestOutputPath() : module.getOutputPath();
final String relativePath = FileUtil.getRelativePath(sourceRoot, FileUtil.toSystemIndependentName(file.getPath()), '/');
final String prefix = module.getSourceRootPrefixes().get(sourceRoot);
@@ -98,7 +94,7 @@ public class ResourcesBuilder extends Builder{
final String outputPath = targetPath.toString();
FileUtil.copyContent(file, new File(outputPath));
try {
outputToSourceMapping.update(outputPath, file.getPath());
outputToSourceMapping.update(file.getPath(), outputPath);
}
catch (Exception e) {
context.processMessage(new CompilerMessage(BUILDER_NAME, e));
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java
+11
View File
@@ -6,6 +6,7 @@ import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import org.jetbrains.annotations.NonNls;
import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
@@ -66,6 +67,16 @@ public abstract class AbstractStateStorage<Key, T> {
}
}
public void appendData(final Key key, final T data) throws Exception {
synchronized (myDataLock) {
myMap.appendData(key, new PersistentHashMap.ValueDataAppender() {
public void append(DataOutput out) throws IOException {
myStateExternalizer.save(out, data);
}
});
}
}
public void remove(Key key) throws Exception {
synchronized (myDataLock) {
myMap.remove(key);
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java
+136 -83
View File
@@ -2,10 +2,10 @@ package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import org.jetbrains.jps.incremental.Builder;
import org.jetbrains.jps.incremental.BuilderCategory;
import org.jetbrains.jps.incremental.BuilderRegistry;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.ether.dependencyView.Mappings;
import org.jetbrains.jps.incremental.Paths;
import org.jetbrains.jps.incremental.ProjectBuildException;
import java.io.File;
import java.io.IOException;
@@ -18,122 +18,175 @@ import java.util.Map;
*/
public class BuildDataManager {
private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.storage.BuildDataManager");
private static final String TIMESTAMP_STORAGE = "stamps";
private static final String OUTPUTS_STORAGE = "out-src";
private static final String SRC_TO_OUTPUTS_STORAGE = "src-out";
private static final String SRC_TO_FORM_STORAGE = "src-form";
private static final String MAPPINGS_STORAGE = "mappings";
private final String myProjectName;
private final Map<String, TimestampStorage> myBuilderToStampStorageMap = new HashMap<String, TimestampStorage>();
private final OutputToSourceMapping myOutputToSourceMap;
private final Object mySourceToOutputLock = new Object();
private final Map<String, SourceToOutputMapping> myProductionSourceToOutputs = new HashMap<String, SourceToOutputMapping>();
private final Map<String, SourceToOutputMapping> myTestSourceToOutputs = new HashMap<String, SourceToOutputMapping>();
public BuildDataManager(String projectName) {
private final SourceToFormMapping mySrcToFormMap;
private final Mappings myMappings;
public BuildDataManager(String projectName) throws ProjectBuildException {
myProjectName = projectName;
myOutputToSourceMap = createOutputToSourceMap();
}
private OutputToSourceMapping createOutputToSourceMap() {
final File root = getOutputToSourceStorageRoot();
final File dataFile = new File(root, "data");
try {
return new OutputToSourceMapping(dataFile);
mySrcToFormMap = createStorage(getSourceToFormsRoot(), new StorageFactory<SourceToFormMapping>() {
public SourceToFormMapping create(File dataFile) throws Exception {
return new SourceToFormMapping(dataFile);
}
});
final File mappingsRoot = getMappingsRoot();
myMappings = createStorage(mappingsRoot, new StorageFactory<Mappings>() {
public Mappings create(File dataFile) throws Exception {
return new Mappings(mappingsRoot);
}
});
}
catch (Exception e) {
FileUtil.delete(root);
try {
return new OutputToSourceMapping(dataFile);
clean();
}
catch (Exception e1) {
throw new RuntimeException(e1);
catch (IOException ignored) {
LOG.info(ignored);
}
throw new ProjectBuildException(e);
}
}
public TimestampStorage getTimestampStorage(String builderName) throws Exception {
synchronized (myBuilderToStampStorageMap) {
TimestampStorage storage = myBuilderToStampStorageMap.get(builderName);
if (storage == null) {
storage = new TimestampStorage(new File(getTimestampsStorageRoot(builderName), "data"));
myBuilderToStampStorageMap.put(builderName, storage);
public SourceToOutputMapping getSourceToOutputMap(String moduleName, boolean testSources) throws Exception {
final Map<String, SourceToOutputMapping> storageMap = testSources? myTestSourceToOutputs : myProductionSourceToOutputs;
SourceToOutputMapping mapping;
synchronized (mySourceToOutputLock) {
mapping = storageMap.get(moduleName);
if (mapping == null) {
mapping = createStorage(getSourceToOutputRoot(moduleName, testSources), new StorageFactory<SourceToOutputMapping>() {
public SourceToOutputMapping create(File dataFile) throws Exception {
return new SourceToOutputMapping(dataFile);
}
});
storageMap.put(moduleName, mapping);
}
return storage;
}
return mapping;
}
public OutputToSourceMapping getOutputToSourceStorage() {
return myOutputToSourceMap;
public SourceToFormMapping getSourceToFormMap() {
return mySrcToFormMap;
}
public void clean() {
synchronized (myBuilderToStampStorageMap) {
public Mappings getMappings() {
return myMappings;
}
public void clean() throws IOException {
try {
synchronized (mySourceToOutputLock) {
closeOutputToSourceStorages();
FileUtil.delete(getSourceToOutputsRoot());
}
}
finally {
try {
final BuilderRegistry registry = BuilderRegistry.getInstance();
for (BuilderCategory category : BuilderCategory.values()) {
for (Builder builder : registry.getBuilders(category)) {
cleanTimestampStorage(builder.getName());
wipeStorage(getSourceToFormsRoot(), mySrcToFormMap);
}
finally {
final Mappings mappings = myMappings;
if (mappings != null) {
synchronized (mappings) {
mappings.clean();
}
}
}
finally {
myOutputToSourceMap.wipe();
}
}
}
private void cleanTimestampStorage(String builderName) {
final TimestampStorage storage = myBuilderToStampStorageMap.remove(builderName);
if (storage != null) {
try {
storage.close();
}
catch (IOException e) {
LOG.info(e);
}
}
FileUtil.delete(Paths.getBuilderDataRoot(myProjectName, builderName));
}
public void close() {
try {
closeTimestampStorages();
synchronized (mySourceToOutputLock) {
closeOutputToSourceStorages();
}
}
finally {
synchronized (myOutputToSourceMap) {
try {
closeStorage(getSourceToFormsRoot(), mySrcToFormMap);
}
finally {
final Mappings mappings = myMappings;
if (mappings != null) {
synchronized (mappings) {
mappings.close();
}
}
}
}
}
private void closeOutputToSourceStorages() {
for (Map.Entry<String, SourceToOutputMapping> entry : myProductionSourceToOutputs.entrySet()) {
closeStorage(getSourceToOutputRoot(entry.getKey(), false), entry.getValue());
}
for (Map.Entry<String, SourceToOutputMapping> entry : myTestSourceToOutputs.entrySet()) {
closeStorage(getSourceToOutputRoot(entry.getKey(), true), entry.getValue());
}
}
public File getSourceToFormsRoot() {
return new File(Paths.getDataStorageRoot(myProjectName), SRC_TO_FORM_STORAGE);
}
public File getSourceToOutputRoot(String moduleName, boolean forTests) {
return new File(getSourceToOutputsRoot(), (forTests? "tests" : "production") + "/" + moduleName);
}
private File getSourceToOutputsRoot() {
return new File(Paths.getDataStorageRoot(myProjectName), SRC_TO_OUTPUTS_STORAGE);
}
public File getMappingsRoot() {
return new File(Paths.getDataStorageRoot(myProjectName), MAPPINGS_STORAGE);
}
private interface StorageFactory<T> {
T create(File dataFile) throws Exception;
}
private static <T> T createStorage(File root, StorageFactory<T> factory) throws Exception {
final File dataFile = new File(root, "data");
try {
return factory.create(dataFile);
}
catch (Exception e) {
FileUtil.delete(root);
return factory.create(dataFile);
}
}
private static void wipeStorage(File root, @Nullable AbstractStateStorage<?, ?> storage) {
if (storage != null) {
synchronized (storage) {
storage.wipe();
}
}
else {
FileUtil.delete(root);
}
}
private static void closeStorage(File root, @Nullable AbstractStateStorage<?, ?> storage) {
if (storage != null) {
synchronized (storage) {
try {
myOutputToSourceMap.close();
storage.close();
}
catch (IOException e) {
LOG.error(e);
FileUtil.delete(getOutputToSourceStorageRoot());
FileUtil.delete(root);
}
}
}
}
private void closeTimestampStorages() {
synchronized (myBuilderToStampStorageMap) {
try {
for (Map.Entry<String, TimestampStorage> entry : myBuilderToStampStorageMap.entrySet()) {
final TimestampStorage storage = entry.getValue();
try {
storage.close();
}
catch (IOException e) {
LOG.error(e);
final String builderName = entry.getKey();
FileUtil.delete(getTimestampsStorageRoot(builderName));
}
}
}
finally {
myBuilderToStampStorageMap.clear();
}
}
}
public File getOutputToSourceStorageRoot() {
return new File(Paths.getDataStorageRoot(myProjectName), OUTPUTS_STORAGE);
}
public File getTimestampsStorageRoot(String builderName) {
return new File(Paths.getBuilderDataRoot(myProjectName, builderName), TIMESTAMP_STORAGE);
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToSourceMapping.java
-34
View File
@@ -1,34 +0,0 @@
package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.io.EnumeratorStringDescriptor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
/**
* @author Eugene Zhuravlev
* Date: 10/7/11
*/
public class OutputToSourceMapping extends AbstractStateStorage<String, String> {
public OutputToSourceMapping(File storePath) throws Exception {
super(storePath, new EnumeratorStringDescriptor(), new EnumeratorStringDescriptor());
}
@Override
public void update(@NotNull String outputPath, @Nullable String sourcePath) throws Exception {
super.update(FileUtil.toSystemIndependentName(outputPath), sourcePath != null? FileUtil.toSystemIndependentName(sourcePath) : null);
}
@Override
public void remove(@NotNull String outputPath) throws Exception {
super.remove(FileUtil.toSystemIndependentName(outputPath));
}
@Override
public String getState(@NotNull String outputPath) throws Exception {
return super.getState(FileUtil.toSystemIndependentName(outputPath));
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectTimestamps.java
+82
View File
@@ -0,0 +1,82 @@
package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import org.jetbrains.jps.incremental.Paths;
import org.jetbrains.jps.incremental.ProjectBuildException;
import java.io.File;
import java.io.IOException;
/**
* @author Eugene Zhuravlev
* Date: 10/7/11
*/
public class ProjectTimestamps {
private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.storage.ProjectTimestamps");
private static final String TIMESTAMP_STORAGE = "timestamps";
private final String myProjectName;
private final TimestampStorage myTimestamps;
public ProjectTimestamps(String projectName) throws ProjectBuildException {
myProjectName = projectName;
try {
File root = getTimestampsRoot();
TimestampStorage result;
final File dataFile = new File(root, "data");
try {
result = new TimestampStorage(dataFile);
}
catch (Exception e) {
FileUtil.delete(root);
result = new TimestampStorage(dataFile);
}
myTimestamps = result;
}
catch (Exception e) {
try {
clean();
}
catch (IOException ignored) {
LOG.info(ignored);
}
throw new ProjectBuildException(e);
}
}
public TimestampStorage getStorage() {
return myTimestamps;
}
public void clean() throws IOException {
final TimestampStorage timestamps = myTimestamps;
if (timestamps != null) {
synchronized (timestamps) {
timestamps.wipe();
}
}
else {
FileUtil.delete(getTimestampsRoot());
}
}
public void close() {
final TimestampStorage timestamps = myTimestamps;
if (timestamps != null) {
synchronized (timestamps) {
try {
timestamps.close();
}
catch (IOException e) {
LOG.error(e);
FileUtil.delete(getTimestampsRoot());
}
}
}
}
public File getTimestampsRoot() {
return new File(Paths.getDataStorageRoot(myProjectName), TIMESTAMP_STORAGE);
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToFormMapping.java
+33
View File
@@ -0,0 +1,33 @@
package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.io.EnumeratorStringDescriptor;
import java.io.File;
/**
* @author Eugene Zhuravlev
* Date: 12/29/11
*/
public class SourceToFormMapping extends AbstractStateStorage<String, String>{
public SourceToFormMapping(File storePath) throws Exception {
super(storePath, new EnumeratorStringDescriptor(), new EnumeratorStringDescriptor());
}
public void update(String srcPath, String formPath) throws Exception {
super.update(FileUtil.toSystemIndependentName(srcPath), FileUtil.toSystemIndependentName(formPath));
}
public final void appendData(String srcPath, String formPath) throws Exception {
update(srcPath, formPath);
}
public void remove(String srcPath) throws Exception {
super.remove(FileUtil.toSystemIndependentName(srcPath));
}
public String getState(String srcPath) throws Exception {
return super.getState(FileUtil.toSystemIndependentName(srcPath));
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMapping.java
+80
View File
@@ -0,0 +1,80 @@
package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorStringDescriptor;
import com.intellij.util.io.IOUtil;
import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
*
*
* @author Eugene Zhuravlev
* Date: 10/7/11
*/
public final class SourceToOutputMapping extends AbstractStateStorage<String, Collection<String>> {
public SourceToOutputMapping(File storePath) throws Exception {
super(storePath, new EnumeratorStringDescriptor(), new StringCollectionExternalizer());
}
@Override
public void update(@NotNull String srcPath, @NotNull Collection<String> outputs) throws Exception {
super.update(FileUtil.toSystemIndependentName(srcPath), normalizePaths(outputs));
}
public void update(@NotNull String srcPath, @NotNull String outputPath) throws Exception {
super.update(FileUtil.toSystemIndependentName(srcPath), Collections.singleton(FileUtil.toSystemIndependentName(outputPath)));
}
public void appendData(String srcPath, String outputPath) throws Exception {
super.appendData(FileUtil.toSystemIndependentName(srcPath), Collections.singleton(FileUtil.toSystemIndependentName(outputPath)));
}
public void appendData(String srcPath, Collection<String> data) throws Exception {
super.appendData(FileUtil.toSystemIndependentName(srcPath), normalizePaths(data));
}
@Override
public void remove(@NotNull String srcPath) throws Exception {
super.remove(FileUtil.toSystemIndependentName(srcPath));
}
@Override
public Collection<String> getState(@NotNull String srcPath) throws Exception {
return super.getState(FileUtil.toSystemIndependentName(srcPath));
}
protected static Collection<String> normalizePaths(Collection<String> outputs) {
Collection<String> normalized = new ArrayList<String>(outputs.size());
for (String out : outputs) {
normalized.add(FileUtil.toSystemIndependentName(out));
}
return normalized;
}
protected static class StringCollectionExternalizer implements DataExternalizer<Collection<String>> {
public void save(DataOutput out, Collection<String> value) throws IOException {
for (String str : value) {
IOUtil.writeString(str, out);
}
}
public Collection<String> read(DataInput in) throws IOException {
final List<String> result = new ArrayList<String>();
final DataInputStream stream = (DataInputStream)in;
while (stream.available() > 0) {
final String str = IOUtil.readString(stream);
result.add(str);
}
return result;
}
}
}
jps/jps-builders/src/org/jetbrains/jps/server/ProjectDescriptor.java
+27
View File
@@ -0,0 +1,27 @@
package org.jetbrains.jps.server;
import org.jetbrains.jps.Project;
import org.jetbrains.jps.incremental.FSState;
import org.jetbrains.jps.incremental.storage.ProjectTimestamps;
/**
* @author Eugene Zhuravlev
* Date: 1/8/12
*/
public class ProjectDescriptor {
public final String projectName;
public final Project project;
public final FSState fsState;
public final ProjectTimestamps timestamps;
ProjectDescriptor(String projectName, Project project, FSState fsState, ProjectTimestamps timestamps) {
this.projectName = projectName;
this.project = project;
this.fsState = fsState;
this.timestamps = timestamps;
}
public void close() {
timestamps.close();
}
}
jps/jps-builders/src/org/jetbrains/jps/server/ServerMessageHandler.java
+4 -2
View File
@@ -103,6 +103,7 @@ class ServerMessageHandler extends SimpleChannelHandler {
// todo
case CLEAN:
case MAKE:
case FORCED_COMPILATION:
case REBUILD: {
final CompilationTask task = new CompilationTask(sessionId, channelContext, projectId, compileRequest.getModuleNameList());
if (myBuildsInProgress.putIfAbsent(projectId, task) == null) {
@@ -235,8 +236,9 @@ class ServerMessageHandler extends SimpleChannelHandler {
switch (compileType) {
case CLEAN: return BuildType.CLEAN;
case MAKE: return BuildType.MAKE;
case REBUILD: return BuildType.REBUILD;
case REBUILD: return BuildType.PROJECT_REBUILD;
case FORCED_COMPILATION: return BuildType.FORCED_COMPILATION;
}
return null;
return BuildType.MAKE; // use make by default
}
}
jps/jps-builders/src/org/jetbrains/jps/server/ServerState.java
+57 -33
View File
@@ -3,7 +3,6 @@ package org.jetbrains.jps.server;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import org.codehaus.groovy.runtime.MethodClosure;
import org.jetbrains.ether.dependencyView.Mappings;
import org.jetbrains.jps.JavaSdk;
import org.jetbrains.jps.Library;
import org.jetbrains.jps.Module;
@@ -14,12 +13,9 @@ import org.jetbrains.jps.api.GlobalLibrary;
import org.jetbrains.jps.api.SdkLibrary;
import org.jetbrains.jps.idea.IdeaProjectLoader;
import org.jetbrains.jps.incremental.*;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.storage.ProjectTimestamps;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.*;
@@ -32,7 +28,7 @@ class ServerState {
private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.server.ServerState");
public static final String IDEA_PROJECT_DIRNAME = ".idea";
private final Map<String, Project> myProjects = new HashMap<String, Project>();
private final Map<String, ProjectDescriptor> myProjects = new HashMap<String, ProjectDescriptor>();
private final Object myConfigurationLock = new Object();
private final Map<String, String> myPathVariables = new HashMap<String, String>();
@@ -40,6 +36,9 @@ class ServerState {
public void setGlobals(List<GlobalLibrary> libs, Map<String, String> pathVars) {
synchronized (myConfigurationLock) {
for (ProjectDescriptor descriptor : myProjects.values()) {
descriptor.close();
}
myProjects.clear(); // projects should be reloaded against the latest data
myGlobalLibraries.clear();
myGlobalLibraries.addAll(libs);
@@ -48,9 +47,44 @@ class ServerState {
}
}
public void notifyFileChanged(String projectPath, File file, RootDescriptor rd) {
try {
final ProjectDescriptor d;
synchronized (myConfigurationLock) {
d = myProjects.get(projectPath);
}
if (d != null) {
d.fsState.markDirty(file, rd, d.timestamps.getStorage());
}
}
catch (Exception e) {
LOG.error(e); // todo
}
}
public void notifyFileDeleted(String projectPath, Module module, String filePath, final boolean isTest) {
try {
final ProjectDescriptor d;
synchronized (myConfigurationLock) {
d = myProjects.get(projectPath);
}
if (d != null) {
d.fsState.registerDeleted(module, FileUtil.toCanonicalPath(filePath), isTest, d.timestamps.getStorage());
}
}
catch (Exception e) {
LOG.error(e); // todo
}
}
public void clearProjectCache(Collection<String> projectPaths) {
synchronized (myConfigurationLock) {
myProjects.keySet().removeAll(projectPaths);
for (String projectPath : projectPaths) {
final ProjectDescriptor descriptor = myProjects.remove(projectPath);
if (descriptor != null) {
descriptor.close();
}
}
}
}
@@ -58,29 +92,18 @@ class ServerState {
final String projectName = getProjectName(projectPath);
BuildType buildType = params.buildType;
Project project;
ProjectDescriptor descriptor;
synchronized (myConfigurationLock) {
project = myProjects.get(projectPath);
if (project == null) {
project = loadProject(projectPath, params);
myProjects.put(projectPath, project);
descriptor = myProjects.get(projectPath);
if (descriptor == null) {
final Project project = loadProject(projectPath, params);
final FSState fsState = new FSState();
descriptor = new ProjectDescriptor(projectName, project, fsState, new ProjectTimestamps(projectName));
myProjects.put(projectPath, descriptor);
}
}
Mappings mappings = null;
final File mappingsRoot = Paths.getMappingsStorageRoot(projectName);
try {
mappings = new Mappings(mappingsRoot);
}
catch (FileNotFoundException e) {
mappings = new Mappings(mappingsRoot);
}
catch (IOException e) {
FileUtil.delete(mappingsRoot);
msgHandler.processMessage(new CompilerMessage(IncProjectBuilder.JPS_SERVER_NAME, BuildMessage.Kind.WARNING, "Problems reading dependency information, rebuild required: " + e.getMessage()));
mappings = new Mappings(mappingsRoot);
buildType = BuildType.REBUILD;
}
final Project project = descriptor.project;
try {
final List<Module> toCompile = new ArrayList<Module>();
@@ -97,17 +120,21 @@ class ServerState {
final CompileScope compileScope = new CompileScope(project, toCompile);
final IncProjectBuilder builder = new IncProjectBuilder(projectName, project, mappings, BuilderRegistry.getInstance());
final IncProjectBuilder builder = new IncProjectBuilder(descriptor, BuilderRegistry.getInstance());
if (msgHandler != null) {
builder.addMessageHandler(msgHandler);
}
switch (buildType) {
case REBUILD:
builder.build(compileScope, false);
case PROJECT_REBUILD:
builder.build(compileScope, false, true);
break;
case FORCED_COMPILATION:
builder.build(compileScope, false, false);
break;
case MAKE:
builder.build(compileScope, true);
builder.build(compileScope, true, false);
break;
case CLEAN:
@@ -117,9 +144,6 @@ class ServerState {
}
}
finally {
if (mappings != null) {
mappings.close();
}
clearZipIndexCache();
}
}
jps/model/src/org/jetbrains/ether/dependencyView/Mappings.java
+13 -13
View File
@@ -20,12 +20,12 @@ import java.util.*;
* To change this template use File | Settings | File Templates.
*/
public class Mappings {
private final static String myClassToSubclassesName = "classToSubclasses.tab";
private final static String myClassToClassName = "classToClass.tab";
private final static String mySourceToClassName = "sourceToClass.tab";
private final static String mySourceToAnnotationsName = "sourceToAnnotations.tab";
private final static String mySourceToUsagesName = "sourceToUsages.tab";
private final static String myClassToSourceName = "classToSource.tab";
private final static String CLASS_TO_SUBCLASSES = "classToSubclasses.tab";
private final static String CLASS_TO_CLASS = "classToClass.tab";
private final static String SOURCE_TO_CLASS = "sourceToClass.tab";
private final static String SOURCE_TO_ANNOTATIONS = "sourceToAnnotations.tab";
private final static String SOURCE_TO_USAGES = "sourceToUsages.tab";
private final static String CLASS_TO_SOURCE = "classToSource.tab";
private final boolean myIsTansient;
@@ -91,32 +91,32 @@ public class Mappings {
}
myClassToSubclasses =
new PersistentMultiMaplet<DependencyContext.S, DependencyContext.S>(DependencyContext.getTableFile(rootDir, myClassToSubclassesName),
new PersistentMultiMaplet<DependencyContext.S, DependencyContext.S>(DependencyContext.getTableFile(rootDir, CLASS_TO_SUBCLASSES),
DependencyContext.descriptorS, DependencyContext.descriptorS,
ourStringSetConstructor);
myClassToClassDependency =
new PersistentMultiMaplet<DependencyContext.S, DependencyContext.S>(DependencyContext.getTableFile(rootDir, myClassToClassName),
new PersistentMultiMaplet<DependencyContext.S, DependencyContext.S>(DependencyContext.getTableFile(rootDir, CLASS_TO_CLASS),
DependencyContext.descriptorS, DependencyContext.descriptorS,
ourStringSetConstructor);
mySourceFileToClasses =
new PersistentMultiMaplet<DependencyContext.S, ClassRepr>(DependencyContext.getTableFile(rootDir, mySourceToClassName),
new PersistentMultiMaplet<DependencyContext.S, ClassRepr>(DependencyContext.getTableFile(rootDir, SOURCE_TO_CLASS),
DependencyContext.descriptorS, ClassRepr.externalizer(myContext),
ourClassSetConstructor);
mySourceFileToAnnotationUsages =
new PersistentMultiMaplet<DependencyContext.S, UsageRepr.Usage>(DependencyContext.getTableFile(rootDir, mySourceToAnnotationsName),
new PersistentMultiMaplet<DependencyContext.S, UsageRepr.Usage>(DependencyContext.getTableFile(rootDir, SOURCE_TO_ANNOTATIONS),
DependencyContext.descriptorS, UsageRepr.externalizer(myContext),
ourUsageSetConstructor);
mySourceFileToUsages =
new PersistentMaplet<DependencyContext.S, UsageRepr.Cluster>(DependencyContext.getTableFile(rootDir, mySourceToUsagesName),
new PersistentMaplet<DependencyContext.S, UsageRepr.Cluster>(DependencyContext.getTableFile(rootDir, SOURCE_TO_USAGES),
DependencyContext.descriptorS,
UsageRepr.Cluster.clusterExternalizer(myContext));
myClassToSourceFile =
new PersistentMaplet<DependencyContext.S, DependencyContext.S>(DependencyContext.getTableFile(rootDir, myClassToSourceName),
new PersistentMaplet<DependencyContext.S, DependencyContext.S>(DependencyContext.getTableFile(rootDir, CLASS_TO_SOURCE),
DependencyContext.descriptorS, DependencyContext.descriptorS);
}
@@ -165,7 +165,7 @@ public class Mappings {
if (myRootDir != null) {
close();
FileUtil.delete(myRootDir);
createImplementation(myRootDir, false);
createImplementation(myRootDir, myIsTansient);
}
}