MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

serverless out-of-process make

Browse Source
This commit is contained in:
Eugene Zhuravlev
2012-04-28 17:46:13 +02:00
parent df67832271
commit b781d72c92
12 changed files with 1003 additions and 1759 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java
+81 -87
View File
@@ -31,18 +31,18 @@ import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.components.ApplicationComponent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.fileTypes.FileTypeManager;
import com.intellij.openapi.module.Module;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ProjectManager;
import com.intellij.openapi.project.ProjectManagerAdapter;
import com.intellij.openapi.projectRoots.*;
import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl;
import com.intellij.openapi.roots.*;
import com.intellij.openapi.roots.ModuleRootEvent;
import com.intellij.openapi.roots.ModuleRootListener;
import com.intellij.openapi.roots.OrderRootType;
import com.intellij.openapi.roots.libraries.Library;
import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar;
import com.intellij.openapi.util.JDOMUtil;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.ShutDownTracker;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.registry.Registry;
@@ -69,8 +69,6 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.api.*;
import org.jetbrains.jps.cmdline.BuildMain;
import org.jetbrains.jps.incremental.fs.FSState;
import org.jetbrains.jps.incremental.fs.RootDescriptor;
import org.jetbrains.jps.server.ClasspathBootstrap;
import org.jetbrains.jps.server.Server;
@@ -100,6 +98,7 @@ public class BuildManager implements ApplicationComponent{
private final ProjectManager myProjectManager;
private final Map<RequestFuture, Project> myAutomakeFutures = new HashMap<RequestFuture, Project>();
private final Map<String, RequestFuture> myBuildsInProgress = Collections.synchronizedMap(new HashMap<String, RequestFuture>());
private final CompileServerClasspathManager myClasspathManager = new CompileServerClasspathManager();
private final Executor myPooledThreadExecutor = new Executor() {
@Override
@@ -107,6 +106,8 @@ public class BuildManager implements ApplicationComponent{
ApplicationManager.getApplication().executeOnPooledThread(command);
}
};
private final SequentialTaskExecutor myEventsProcessor = new SequentialTaskExecutor(myPooledThreadExecutor);
private final Map<String, ProjectData> myProjectDataMap = Collections.synchronizedMap(new HashMap<String, ProjectData>());
private final ChannelGroup myAllOpenChannels = new DefaultChannelGroup("build-manager");
@@ -198,80 +199,39 @@ public class BuildManager implements ApplicationComponent{
}
public void notifyFilesChanged(final Collection<String> paths) {
// todo: temporarily commented
//doNotify(paths, false);
doNotify(paths, false);
}
public void notifyFilesDeleted(Collection<String> paths) {
// todo: temporarily commented
//doNotify(paths, true);
doNotify(paths, true);
}
private void doNotify(final Collection<String> paths, final boolean notifyDeletion) {
final Project[] openProjects = myProjectManager.getOpenProjects();
myPooledThreadExecutor.execute(new Runnable() {
// ensure events processed in the order they arrived
myEventsProcessor.submit(new Runnable() {
@Override
public void run() {
List<Pair<ProjectFileIndex, ProjectData>> activeProjects = new ArrayList<Pair<ProjectFileIndex, ProjectData>>();
for (Project project : openProjects) {
if (project.isDisposed() || project.isDefault()) {
continue;
}
final String projectPath = getProjectPath(project);
final ProjectData data = myProjectDataMap.get(projectPath);
if (data != null) {
activeProjects.add(Pair.create(ProjectRootManager.getInstance(project).getFileIndex(), data));
}
}
final LocalFileSystem lfs = LocalFileSystem.getInstance();
for (final Pair<ProjectFileIndex, ProjectData> pair : activeProjects) {
final ProjectFileIndex fileIndex = pair.first;
final FSState fsState = pair.second.fsState;
pair.second.requestQueue.submit(new Runnable() {
@Override
public void run() {
ApplicationManager.getApplication().runReadAction(new Runnable() {
@Override
public void run() {
for (String filePath : paths) {
// todo: estimate performance, probably add a way to obtain all data from file index in a single call
final VirtualFile vFile = lfs.findFileByPath(filePath);
if (vFile == null) {
continue;
}
final Module module = fileIndex.getModuleForFile(vFile);
if (module == null) {
continue;
}
final String moduleName = module.getName();
if (!fsState.isInitialized(moduleName)) { // todo: proper sync!
continue;
}
final VirtualFile srcRoot = fileIndex.getSourceRootForFile(vFile);
if (srcRoot == null) {
return;
}
final boolean inTestSources = fileIndex.isInTestSourceContent(vFile);
if (!inTestSources && !fileIndex.isInSourceContent(vFile)) {
continue;
}
try {
final File file = new File(vFile.getPath());
if (notifyDeletion) {
fsState.registerDeleted(moduleName, file, inTestSources, null);
}
else {
fsState.markDirty(file, new RootDescriptor(moduleName, new File(srcRoot.getPath()), inTestSources, false), null);
}
}
catch (IOException e) {
LOG.error(e);
}
}
}
});
synchronized (myProjectDataMap) {
for (Map.Entry<String, ProjectData> entry : myProjectDataMap.entrySet()) {
final ProjectData data = entry.getValue();
if (notifyDeletion) {
data.addDeleted(paths);
}
});
else {
data.addChanged(paths);
}
final RequestFuture future = myBuildsInProgress.get(entry.getKey());
if (future != null && !future.isCancelled() && !future.isDone()) {
final UUID sessionId = future.getRequestID();
final Channel channel = myMessageDispatcher.getConnectedChannel(sessionId);
if (channel != null) {
final CmdlineRemoteProto.Message.ControllerMessage message =
CmdlineRemoteProto.Message.ControllerMessage.newBuilder().setType(
CmdlineRemoteProto.Message.ControllerMessage.Type.FS_EVENT).setFsEvent(data.createNextEvent()).build();
Channels.write(channel, CmdlineProtoUtil.toMessage(sessionId, message));
}
}
}
}
}
});
@@ -346,8 +306,6 @@ public class BuildManager implements ApplicationComponent{
}
}
// todo: avoid FS scan on every start
@Nullable
public RequestFuture scheduleBuild(
final Project project, final boolean isRebuild,
@@ -366,26 +324,32 @@ public class BuildManager implements ApplicationComponent{
myGlobals = globals;
}
final SequentialTaskExecutor sequential;
final FSState fsState;
CmdlineRemoteProto.Message.ControllerMessage.FSEvent currentFSChanges = null;
final SequentialTaskExecutor projectTaskQueue;
synchronized (myProjectDataMap) {
ProjectData data = myProjectDataMap.get(projectPath);
if (data == null) {
data = new ProjectData(new SequentialTaskExecutor(myPooledThreadExecutor), new FSState());
data = new ProjectData(new SequentialTaskExecutor(myPooledThreadExecutor));
myProjectDataMap.put(projectPath, data);
}
sequential = data.requestQueue;
fsState = data.fsState;
handler = new FSStateMessageHandler(data.fsState, handler);
else {
if (!isRebuild) {
currentFSChanges = data.createNextEvent();
}
else {
data.clearFSChanges();
}
}
projectTaskQueue = data.taskQueue;
}
if (isRebuild) {
params = CmdlineProtoUtil.createRebuildRequest(projectPath, userData, globals, fsState);
params = CmdlineProtoUtil.createRebuildRequest(projectPath, userData, globals);
}
else {
params = isMake ?
CmdlineProtoUtil.createMakeRequest(projectPath, modules, artifacts, userData, globals, fsState) :
CmdlineProtoUtil.createForceCompileRequest(projectPath, modules, artifacts, paths, userData, globals, fsState);
CmdlineProtoUtil.createMakeRequest(projectPath, modules, artifacts, userData, globals, currentFSChanges) :
CmdlineProtoUtil.createForceCompileRequest(projectPath, modules, artifacts, paths, userData, globals, currentFSChanges);
}
myMessageDispatcher.registerBuildMessageHandler(sessionId, handler, params);
@@ -410,7 +374,7 @@ public class BuildManager implements ApplicationComponent{
}
});
sequential.submit(new Runnable() {
projectTaskQueue.submit(new Runnable() {
@Override
public void run() {
try {
@@ -418,6 +382,7 @@ public class BuildManager implements ApplicationComponent{
future.cancel(false);
return;
}
myBuildsInProgress.put(projectPath, future);
final Process process = launchBuildProcess(myListenPort, sessionId);
final OSProcessHandler processHandler = new OSProcessHandler(process, null) {
@Override
@@ -452,6 +417,7 @@ public class BuildManager implements ApplicationComponent{
future.getMessageHandler().sessionTerminated();
}
finally {
myBuildsInProgress.remove(projectPath);
future.setDone();
}
}
@@ -794,12 +760,40 @@ public class BuildManager implements ApplicationComponent{
}
private static class ProjectData {
final SequentialTaskExecutor requestQueue;
final FSState fsState;
final SequentialTaskExecutor taskQueue;
private final Set<String> myChanged = new HashSet<String>();
private final Set<String> myDeleted = new HashSet<String>();
private long myNextEventOrdinal = 0L;
private ProjectData(SequentialTaskExecutor requestQueue, FSState fsState) {
this.requestQueue = requestQueue;
this.fsState = fsState;
private ProjectData(SequentialTaskExecutor taskQueue) {
this.taskQueue = taskQueue;
}
public void addChanged(Collection<String> paths) {
myDeleted.removeAll(paths);
myChanged.addAll(paths);
}
public void addDeleted(Collection<String> paths) {
myChanged.removeAll(paths);
myDeleted.addAll(paths);
}
public CmdlineRemoteProto.Message.ControllerMessage.FSEvent createNextEvent() {
final CmdlineRemoteProto.Message.ControllerMessage.FSEvent.Builder builder =
CmdlineRemoteProto.Message.ControllerMessage.FSEvent.newBuilder();
builder.setOrdinal(++myNextEventOrdinal);
builder.addAllChangedPaths(myChanged);
myChanged.clear();
builder.addAllDeletedPaths(myDeleted);
myDeleted.clear();
return builder.build();
}
public void clearFSChanges() {
myNextEventOrdinal = 0L;
myChanged.clear();
myDeleted.clear();
}
}
java/compiler/impl/src/com/intellij/compiler/server/BuildMessageDispatcher.java
+16 -6
View File
@@ -52,16 +52,26 @@ class BuildMessageDispatcher extends SimpleChannelHandler {
public void cancelSession(UUID sessionId) {
if (myCanceledSessions.add(sessionId)) {
final SessionData data = myMessageHandlers.get(sessionId);
if (data != null) {
final Channel channel = data.channel;
if (channel != null) {
Channels.write(channel, CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createCancelCommand()));
}
final Channel channel = getConnectedChannel(sessionId);
if (channel != null) {
Channels.write(channel, CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createCancelCommand()));
}
}
}
@Nullable
public Channel getConnectedChannel(final UUID sessionId) {
final SessionData data = myMessageHandlers.get(sessionId);
if (data != null) {
final Channel channel = data.channel;
if (channel != null && channel.isConnected()) {
return channel;
}
}
return null;
}
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
final CmdlineRemoteProto.Message message = (CmdlineRemoteProto.Message)e.getMessage();
java/compiler/impl/src/com/intellij/compiler/server/DefaultMessageHandler.java
-6
View File
@@ -31,15 +31,9 @@ public abstract class DefaultMessageHandler implements BuilderMessageHandler {
case COMPILE_MESSAGE:
handleCompileMessage(msg.getCompileMessage());
break;
case FS_STATE:
handleFSStateMessage(msg.getFsstateMessage());
break;
}
}
protected void handleFSStateMessage(CmdlineRemoteProto.Message.FSStateMessage message) {
}
protected abstract void handleCompileMessage(CmdlineRemoteProto.Message.BuilderMessage.CompileMessage message);
protected abstract void handleBuildEvent(CmdlineRemoteProto.Message.BuilderMessage.BuildEvent event);
java/compiler/impl/src/com/intellij/compiler/server/FSStateMessageHandler.java
-83
View File
@@ -1,83 +0,0 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.compiler.server;
import org.jetbrains.jps.api.CmdlineRemoteProto;
import org.jetbrains.jps.incremental.fs.FSState;
import java.io.File;
import java.util.*;
/**
* @author Eugene Zhuravlev
* Date: 4/18/12
*/
public class FSStateMessageHandler extends DefaultMessageHandler {
private final FSState myFsState;
private final DefaultMessageHandler myDelegate;
protected FSStateMessageHandler(FSState fsState, DefaultMessageHandler delegate) {
myFsState = fsState;
myDelegate = delegate;
}
@Override
public void handleFailure(CmdlineRemoteProto.Message.Failure failure) {
myDelegate.handleFailure(failure);
}
@Override
public void sessionTerminated() {
myDelegate.sessionTerminated();
}
protected void handleFSStateMessage(CmdlineRemoteProto.Message.FSStateMessage message) {
try {
myDelegate.handleFSStateMessage(message);
}
finally {
final String moduleName = message.getModuleName();
final List<String> deleteProduction = message.getDeletedProductionList();
final List<String> deletedTests = message.getDeletedTestsList();
final Map<File,Set<File>> recompileProduction = new HashMap<File, Set<File>>();
final Map<File, Set<File>> recompileTests = new HashMap<File, Set<File>>();
for (CmdlineRemoteProto.Message.FSStateMessage.RootDelta delta : message.getRecompileDeltaList()) {
final File root = new File(delta.getRoot());
final Map<File, Set<File>> map = delta.getTestSources()? recompileTests : recompileProduction;
Set<File> files = map.get(root);
if (files == null) {
files = new HashSet<File>();
map.put(root, files);
}
for (String path : delta.getPathList()) {
files.add(new File(path));
}
}
myFsState.init(moduleName, deleteProduction, deletedTests, recompileProduction, recompileTests);
}
}
protected void handleCompileMessage(CmdlineRemoteProto.Message.BuilderMessage.CompileMessage message) {
myDelegate.handleCompileMessage(message);
}
protected void handleBuildEvent(CmdlineRemoteProto.Message.BuilderMessage.BuildEvent event) {
myDelegate.handleBuildEvent(event);
}
}
jps/jps-builders/proto/cmdline_remote_proto.proto
+8 -17
View File
@@ -15,19 +15,6 @@ message Message {
required string value = 2;
}
message FSStateMessage {
message RootDelta {
required bool test_sources = 1;
required string root = 2;
repeated string path = 3;
}
required string module_name = 1;
repeated RootDelta recompile_delta = 2;
repeated string deleted_production = 3;
repeated string deleted_tests = 4;
}
enum Type {
CONTROLLER_MESSAGE = 1;
BUILDER_MESSAGE = 2;
@@ -44,6 +31,13 @@ message Message {
enum Type {
BUILD_PARAMETERS = 1;
CANCEL_BUILD_COMMAND = 2;
FS_EVENT = 3;
}
message FSEvent {
required uint64 ordinal = 1;
repeated string changed_paths = 2;
repeated string deleted_paths = 3;
}
message GlobalSettings {
@@ -78,11 +72,11 @@ message Message {
repeated string file_path = 5;
repeated string artifact_name = 6;
repeated KeyValuePair builder_parameter = 7;
repeated FSStateMessage fs_state = 8;
}
required Type type = 1;
optional ParametersMessage params_message = 2;
optional FSEvent fs_event = 3;
}
message BuilderMessage {
@@ -90,8 +84,6 @@ message Message {
PARAM_REQUEST = 1;
BUILD_EVENT = 2;
COMPILE_MESSAGE = 3;
FS_STATE = 4;
FS_EVENT = 5;
}
message BuildEvent {
@@ -139,7 +131,6 @@ message Message {
required Type type = 1;
optional BuildEvent build_event = 2;
optional CompileMessage compile_message = 3;
optional FSStateMessage fsstate_message = 4;
}
required UUID session_id = 1;
jps/jps-builders/src/org/jetbrains/jps/api/CmdlineProtoUtil.java
+19 -49
View File
@@ -1,15 +1,15 @@
package org.jetbrains.jps.api;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.incremental.fs.FSState;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.UUID;
/**
* @author Eugene Zhuravlev
@@ -17,21 +17,15 @@ import java.util.*;
*/
public class CmdlineProtoUtil {
public static CmdlineRemoteProto.Message.BuilderMessage createFSStateBuilderMessage(String moduleName, FSState fsState) {
final CmdlineRemoteProto.Message.FSStateMessage fsStateMessage = createFSStateMessage(moduleName, fsState);
return CmdlineRemoteProto.Message.BuilderMessage.newBuilder().setType(CmdlineRemoteProto.Message.BuilderMessage.Type.FS_STATE).setFsstateMessage(
fsStateMessage).build();
}
public static CmdlineRemoteProto.Message.ControllerMessage createMakeRequest(String project,
Collection<String> modules,
Collection<String> artifacts,
final Map<String, String> userData,
final CmdlineRemoteProto.Message.ControllerMessage.GlobalSettings globals,
final FSState fsState) {
final CmdlineRemoteProto.Message.ControllerMessage.FSEvent event) {
return createBuildParametersMessage(CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.Type.MAKE, project, modules, artifacts,
userData, Collections.<String>emptyList(),
globals, fsState);
globals, event);
}
public static CmdlineRemoteProto.Message.ControllerMessage createForceCompileRequest(String project,
@@ -40,31 +34,29 @@ public class CmdlineProtoUtil {
Collection<String> paths,
final Map<String, String> userData,
final CmdlineRemoteProto.Message.ControllerMessage.GlobalSettings globals,
final FSState fsState) {
final CmdlineRemoteProto.Message.ControllerMessage.FSEvent event) {
return createBuildParametersMessage(CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.Type.FORCED_COMPILATION, project, modules,
artifacts,
userData, paths, globals, fsState);
userData, paths, globals, event);
}
public static CmdlineRemoteProto.Message.ControllerMessage createRebuildRequest(String project,
final Map<String, String> userData,
final CmdlineRemoteProto.Message.ControllerMessage.GlobalSettings globals,
final FSState fsState) {
final CmdlineRemoteProto.Message.ControllerMessage.GlobalSettings globals) {
return createBuildParametersMessage(CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.Type.REBUILD, project,
Collections.<String>emptyList(),
Collections.<String>emptyList(), userData, Collections.<String>emptyList(),
globals, fsState);
globals, null);
}
public static CmdlineRemoteProto.Message.ControllerMessage createCleanRequest(String project,
Collection<String> modules,
Collection<String> artifacts,
final Map<String, String> userData,
final CmdlineRemoteProto.Message.ControllerMessage.GlobalSettings globals,
final FSState fsState) {
final CmdlineRemoteProto.Message.ControllerMessage.GlobalSettings globals) {
return createBuildParametersMessage(
CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.Type.CLEAN, project, modules, artifacts, userData, Collections.<String>emptyList(), globals, fsState
);
CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.Type.CLEAN, project, modules, artifacts, userData, Collections.<String>emptyList(), globals,
null);
}
public static CmdlineRemoteProto.Message.ControllerMessage createBuildParametersMessage(CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.Type buildType,
@@ -74,7 +66,7 @@ public class CmdlineProtoUtil {
Map<String, String> userData,
Collection<String> paths,
final CmdlineRemoteProto.Message.ControllerMessage.GlobalSettings globals,
FSState fsState) {
CmdlineRemoteProto.Message.ControllerMessage.FSEvent initialEvent) {
final CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.Builder
builder = CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.newBuilder();
builder.setGlobalSettings(globals);
@@ -98,12 +90,12 @@ public class CmdlineProtoUtil {
if (!paths.isEmpty()) {
builder.addAllFilePath(paths);
}
for (String moduleName : fsState.getInitializedModules()) {
builder.addFsState(createFSStateMessage(moduleName, fsState));
final CmdlineRemoteProto.Message.ControllerMessage.Builder controlMessageBuilder =
CmdlineRemoteProto.Message.ControllerMessage.newBuilder();
if (initialEvent != null) {
controlMessageBuilder.setFsEvent(initialEvent);
}
return CmdlineRemoteProto.Message.ControllerMessage.newBuilder().setType(CmdlineRemoteProto.Message.ControllerMessage.Type.BUILD_PARAMETERS).setParamsMessage(builder.build()).build();
return controlMessageBuilder.setType(CmdlineRemoteProto.Message.ControllerMessage.Type.BUILD_PARAMETERS).setParamsMessage(builder.build()).build();
}
public static CmdlineRemoteProto.Message.KeyValuePair createPair(String key, String value) {
@@ -226,26 +218,4 @@ public class CmdlineProtoUtil {
return uuidBuilder.setMostSigBits(sessionId.getMostSignificantBits()).setLeastSigBits(sessionId.getLeastSignificantBits()).build();
}
private static CmdlineRemoteProto.Message.FSStateMessage createFSStateMessage(String moduleName, FSState fsState) {
final CmdlineRemoteProto.Message.FSStateMessage.Builder stateBuilder = CmdlineRemoteProto.Message.FSStateMessage.newBuilder();
stateBuilder.setModuleName(moduleName);
stateBuilder.addAllDeletedProduction(fsState.getDeletedPaths(moduleName, false));
stateBuilder.addAllDeletedTests(fsState.getDeletedPaths(moduleName, true));
fillRecompilePaths(moduleName, fsState, false, stateBuilder);
fillRecompilePaths(moduleName, fsState, true, stateBuilder);
return stateBuilder.build();
}
private static void fillRecompilePaths(String moduleName, FSState fsState, final boolean forTests, CmdlineRemoteProto.Message.FSStateMessage.Builder stateBuilder) {
for (Map.Entry<File, Set<File>> entry : fsState.getSourcesToRecompile(moduleName, forTests).entrySet()) {
final CmdlineRemoteProto.Message.FSStateMessage.RootDelta.Builder deltaBuilder = CmdlineRemoteProto.Message.FSStateMessage.RootDelta.newBuilder();
deltaBuilder.setTestSources(forTests);
deltaBuilder.setRoot(FileUtil.toSystemIndependentName(entry.getKey().getPath()));
for (File file : entry.getValue()) {
deltaBuilder.addPath(FileUtil.toSystemIndependentName(file.getPath()));
}
stateBuilder.addRecompileDelta(deltaBuilder.build());
}
}
}
jps/jps-builders/src/org/jetbrains/jps/api/CmdlineRemoteProto.java
+611 -1470
View File
File diff suppressed because it is too large Load Diff
jps/jps-builders/src/org/jetbrains/jps/api/SequentialTaskExecutor.java
+1 -1
View File
@@ -47,7 +47,7 @@ public class SequentialTaskExecutor {
return futureTask;
}
private void processQueue() {
protected void processQueue() {
if (!myInProgress.getAndSet(true)) {
myExecutor.execute(USER_TASK_RUNNER);
}
jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildMain.java
+14 -3
View File
@@ -89,9 +89,10 @@ public class BuildMain {
final CmdlineRemoteProto.Message.ControllerMessage controllerMessage = message.getControllerMessage();
switch (controllerMessage.getType()) {
case BUILD_PARAMETERS:
case BUILD_PARAMETERS: {
if (mySession == null) {
final BuildSession session = new BuildSession(mySessionId, ctx.getChannel(), controllerMessage.getParamsMessage());
final CmdlineRemoteProto.Message.ControllerMessage.FSEvent delta = controllerMessage.hasFsEvent()? controllerMessage.getFsEvent() : null;
final BuildSession session = new BuildSession(mySessionId, ctx.getChannel(), controllerMessage.getParamsMessage(), delta);
mySession = session;
SharedThreadPool.INSTANCE.submit(session);
}
@@ -99,8 +100,17 @@ public class BuildMain {
LOG.info("Cannot start another build session because one is already running");
}
return;
}
case CANCEL_BUILD_COMMAND:
case FS_EVENT: {
final BuildSession session = mySession;
if (session != null) {
session.processFSEvent(controllerMessage.getFsEvent());
}
return;
}
case CANCEL_BUILD_COMMAND: {
final BuildSession session = mySession;
if (session != null) {
session.cancel();
@@ -110,6 +120,7 @@ public class BuildMain {
closeChannel(ctx.getChannel());
}
return;
}
}
}
jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java
+173 -37
View File
@@ -3,8 +3,10 @@ package org.jetbrains.jps.cmdline;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.io.DataOutputStream;
import groovy.util.Node;
import groovy.util.XmlParser;
import org.codehaus.groovy.runtime.MethodClosure;
@@ -12,6 +14,7 @@ import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.Channels;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.Library;
import org.jetbrains.jps.Module;
import org.jetbrains.jps.Project;
@@ -29,10 +32,9 @@ import org.jetbrains.jps.incremental.storage.ProjectTimestamps;
import org.jetbrains.jps.incremental.storage.Timestamps;
import org.jetbrains.jps.server.ProjectDescriptor;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.io.*;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* @author Eugene Zhuravlev
@@ -41,6 +43,7 @@ import java.util.*;
final class BuildSession implements Runnable, CanceledStatus {
private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.cmdline.BuildSession");
public static final String IDEA_PROJECT_DIRNAME = ".idea";
private static final String FS_STATE_FILE = "fs_state.dat";
private final UUID mySessionId;
private final Channel myChannel;
private volatile boolean myCanceled = false;
@@ -56,11 +59,17 @@ final class BuildSession implements Runnable, CanceledStatus {
private final List<String> myFilePaths;
private final Map<String, String> myBuilderParams;
private String myProjectPath;
private List<CmdlineRemoteProto.Message.FSStateMessage> myModuleFSStates;
@Nullable
private CmdlineRemoteProto.Message.ControllerMessage.FSEvent myInitialFSDelta;
// state
private EventsProcessor myEventsProcessor = new EventsProcessor();
private volatile long myLastEventOrdinal;
private volatile ProjectDescriptor myProjectDescriptor;
// todo pass FSState in order not to scan FS from scratch
BuildSession(UUID sessionId, Channel channel, CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage params) {
BuildSession(UUID sessionId,
Channel channel,
CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage params,
@Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent delta) {
mySessionId = sessionId;
myChannel = channel;
@@ -91,7 +100,7 @@ final class BuildSession implements Runnable, CanceledStatus {
for (CmdlineRemoteProto.Message.KeyValuePair pair : params.getBuilderParameterList()) {
myBuilderParams.put(pair.getKey(), pair.getValue());
}
myModuleFSStates = params.getFsStateList();
myInitialFSDelta = delta;
}
public void run() {
@@ -150,29 +159,10 @@ final class BuildSession implements Runnable, CanceledStatus {
boolean forceCleanCaches = false;
ProjectDescriptor pd;
final Project project = loadProject(projectPath);
final BuildFSState fsState = new BuildFSState(false);
for (CmdlineRemoteProto.Message.FSStateMessage state : myModuleFSStates) {
final Map<File, Set<File>> recompileProduction = new HashMap<File, Set<File>>();
final Map<File, Set<File>> recompileTests = new HashMap<File, Set<File>>();
for (CmdlineRemoteProto.Message.FSStateMessage.RootDelta delta : state.getRecompileDeltaList()) {
final Map<File, Set<File>> map = delta.getTestSources()? recompileTests : recompileProduction;
final File root = new File(delta.getRoot());
Set<File> files = map.get(root);
if (files == null) {
files = new HashSet<File>();
map.put(root, files);
}
for (String path : delta.getPathList()) {
files.add(new File(path));
}
}
fsState.init(state.getModuleName(), state.getDeletedProductionList(), state.getDeletedTestsList(), recompileProduction, recompileTests);
}
final File dataStorageRoot = Utils.getDataStorageRoot(project);
ProjectTimestamps projectTimestamps = null;
BuildDataManager dataManager = null;
final File dataStorageRoot = Utils.getDataStorageRoot(project);
try {
projectTimestamps = new ProjectTimestamps(dataStorageRoot);
dataManager = new BuildDataManager(dataStorageRoot, true);
@@ -192,14 +182,21 @@ final class BuildSession implements Runnable, CanceledStatus {
}
forceCleanCaches = true;
FileUtil.delete(dataStorageRoot);
// todo: create optionally
//projectTimestamps = new ProjectTimestamps(dataStorageRoot);
projectTimestamps = new ProjectTimestamps(dataStorageRoot);
dataManager = new BuildDataManager(dataStorageRoot, true);
// second attempt succeded
msgHandler.processMessage(new CompilerMessage("build", BuildMessage.Kind.INFO, "Project rebuild forced: " + e.getMessage()));
}
final BuildFSState fsState = new BuildFSState(false);
pd = new ProjectDescriptor(project, fsState, projectTimestamps, dataManager, BuildLoggingManager.DEFAULT);
myProjectDescriptor = pd;
loadFsState(myProjectDescriptor, dataStorageRoot, myInitialFSDelta);
// free memory
myInitialFSDelta = null;
// ensure events from controller are processed after FSState initialization
myEventsProcessor.startProcessing();
try {
for (int attempt = 0; attempt < 2; attempt++) {
@@ -244,19 +241,138 @@ final class BuildSession implements Runnable, CanceledStatus {
}
}
}
// save initialized FS state for future use
for (Module module : pd.project.getModules().values()) {
if (fsState.isInitialized(module.getName())) {
Channels.write(myChannel, CmdlineProtoUtil.toMessage(mySessionId, CmdlineProtoUtil.createFSStateBuilderMessage(module.getName(),
fsState)));
}
}
}
finally {
saveFsState(dataStorageRoot, pd.fsState, myLastEventOrdinal);
pd.release();
}
}
public void processFSEvent(final CmdlineRemoteProto.Message.ControllerMessage.FSEvent event) {
myEventsProcessor.submit(new Runnable() {
@Override
public void run() {
try {
applyFSEvent(myProjectDescriptor, event);
}
catch (IOException e) {
LOG.error(e);
}
}
});
}
private void applyFSEvent(ProjectDescriptor pd, @Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent event) throws IOException {
if (event == null) {
return;
}
final Timestamps timestamps = pd.timestamps.getStorage();
for (String deleted : event.getDeletedPathsList()) {
final File file = new File(deleted);
final RootDescriptor rd = pd.rootsIndex.getModuleAndRoot(file);
if (rd != null) {
pd.fsState.registerDeleted(rd.module, file, rd.isTestRoot, timestamps);
}
}
for (String changed : event.getChangedPathsList()) {
final File file = new File(changed);
final RootDescriptor rd = pd.rootsIndex.getModuleAndRoot(file);
if (rd != null) {
pd.fsState.markDirty(file, rd, timestamps);
}
}
myLastEventOrdinal += 1;
}
private static void saveFsState(File dataStorageRoot, BuildFSState state, long lastEventOrdinal) {
final File file = new File(dataStorageRoot, FS_STATE_FILE);
if (lastEventOrdinal < 0L) {
FileUtil.delete(file);
return;
}
BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream();
try {
final DataOutputStream out = new DataOutputStream(bytes);
out.writeLong(lastEventOrdinal);
try {
state.save(out);
}
finally {
out.close();
}
}
catch (IOException e) {
LOG.error(e);
return;
}
FileOutputStream fos = null;
try {
fos = new FileOutputStream(file);
}
catch (FileNotFoundException e) {
FileUtil.createIfDoesntExist(file);
}
try {
if (fos == null) {
fos = new FileOutputStream(file);
}
try {
fos.write(bytes.getInternalBuffer(), 0, bytes.size());
}
finally {
fos.close();
}
}
catch (IOException e) {
LOG.error(e);
FileUtil.delete(file);
}
}
private void loadFsState(final ProjectDescriptor pd, File dataStorageRoot, CmdlineRemoteProto.Message.ControllerMessage.FSEvent initialEvent) {
final File file = new File(dataStorageRoot, FS_STATE_FILE);
try {
final InputStream fs = new FileInputStream(file);
byte[] bytes;
try {
bytes = FileUtil.loadBytes(fs, (int)file.length());
}
finally {
fs.close();
}
final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
try {
final long savedOrdinal = in.readLong();
if (initialEvent != null && (savedOrdinal + 1L == initialEvent.getOrdinal())) {
pd.fsState.load(in);
myLastEventOrdinal = savedOrdinal;
applyFSEvent(pd, initialEvent);
}
else {
// either the first start or some events were lost, forcing scan
pd.fsState.clearAll();
myLastEventOrdinal = initialEvent != null? initialEvent.getOrdinal() : 0L;
}
}
finally {
in.close();
}
}
catch (FileNotFoundException ignored) {
}
catch (IOException e) {
pd.fsState.clearAll();
myLastEventOrdinal = initialEvent != null? initialEvent.getOrdinal() : 0L;
LOG.error(e);
}
}
private void finishBuild(Throwable error, boolean hadBuildErrors, boolean markedUptodateFiles) {
CmdlineRemoteProto.Message lastMessage = null;
try {
@@ -464,4 +580,24 @@ final class BuildSession implements Runnable, CanceledStatus {
return BuildType.MAKE; // use make by default
}
private static class EventsProcessor extends SequentialTaskExecutor {
private final AtomicBoolean myProcessingEnabled = new AtomicBoolean(false);
EventsProcessor() {
super(SharedThreadPool.INSTANCE);
}
public void startProcessing() {
if (!myProcessingEnabled.getAndSet(true)) {
super.processQueue();
}
}
@Override
protected void processQueue() {
if (myProcessingEnabled.get()) {
super.processQueue();
}
}
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FSState.java
+33
View File
@@ -1,9 +1,12 @@
package org.jetbrains.jps.incremental.fs;
import com.intellij.util.io.IOUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.incremental.storage.Timestamps;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.util.*;
@@ -17,6 +20,36 @@ public class FSState {
protected final Set<String> myInitialTestsScanPerformed = Collections.synchronizedSet(new HashSet<String>());
protected final Set<String> myInitialProductionScanPerformed = Collections.synchronizedSet(new HashSet<String>());
public void save(DataOutput out) throws IOException {
out.writeInt(myInitialTestsScanPerformed.size());
for (String moduleName : myInitialTestsScanPerformed) {
IOUtil.writeString(moduleName, out);
getDelta(moduleName).save(out, true);
}
out.writeInt(myInitialProductionScanPerformed.size());
for (String moduleName : myInitialProductionScanPerformed) {
IOUtil.writeString(moduleName, out);
getDelta(moduleName).save(out, false);
}
}
public void load(DataInput in) throws IOException {
int testsDeltaCount = in.readInt();
while (testsDeltaCount-- > 0) {
final String moduleName = IOUtil.readString(in);
getDelta(moduleName).load(in, true);
myInitialTestsScanPerformed.add(moduleName);
}
int productionDeltaCount = in.readInt();
while (productionDeltaCount-- > 0) {
final String moduleName = IOUtil.readString(in);
getDelta(moduleName).load(in, false);
myInitialProductionScanPerformed.add(moduleName);
}
}
public void clearAll() {
myDeltas.clear();
}
jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java
+47
View File
@@ -1,9 +1,13 @@
package org.jetbrains.jps.incremental.fs;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.io.IOUtil;
import org.jetbrains.annotations.Nullable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.util.*;
/** @noinspection SynchronizationOnLocalVariableOrMethodParameter*/
@@ -13,6 +17,49 @@ final class FilesDelta {
private final Map<File, Set<File>> mySourcesToRecompile = Collections.synchronizedMap(new HashMap<File, Set<File>>()); // srcRoot -> set of sources
private final Map<File, Set<File>> myTestsToRecompile = Collections.synchronizedMap(new HashMap<File, Set<File>>()); // srcRoot -> set of sources
public void save(DataOutput out, final boolean tests) throws IOException {
final Set<String> deleted = tests? myDeletedTests : myDeletedProduction;
out.writeInt(deleted.size());
for (String path : deleted) {
IOUtil.writeString(path, out);
}
final Map<File, Set<File>> recompile = tests? myTestsToRecompile : mySourcesToRecompile;
out.writeInt(recompile.size());
for (Map.Entry<File, Set<File>> entry : recompile.entrySet()) {
final File root = entry.getKey();
IOUtil.writeString(FileUtil.toSystemIndependentName(root.getPath()), out);
final Set<File> files = entry.getValue();
out.writeInt(files.size());
for (File file : files) {
IOUtil.writeString(FileUtil.toSystemIndependentName(file.getPath()), out);
}
}
}
public void load(DataInput in, final boolean tests) throws IOException {
final Set<String> deleted = tests? myDeletedTests : myDeletedProduction;
deleted.clear();
int deletedCount = in.readInt();
while (deletedCount-- > 0) {
deleted.add(IOUtil.readString(in));
}
final Map<File, Set<File>> recompile = tests? myTestsToRecompile : mySourcesToRecompile;
recompile.clear();
int recompileCount = in.readInt();
while (recompileCount-- > 0) {
final File root = new File(IOUtil.readString(in));
Set<File> files = recompile.get(root);
if (files == null) {
files = new HashSet<File>();
recompile.put(root, files);
}
int filesCount = in.readInt();
while (filesCount-- > 0) {
files.add(new File(IOUtil.readString(in)));
}
}
}
public void init(Collection<String> deletedProduction, Collection<String> deletedTests, Map<File, Set<File>> recompileProduction, Map<File, Set<File>> recompileTests) {
myDeletedProduction.clear();
myDeletedProduction.addAll(deletedProduction);