MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

serverless out-of-process make

Browse Source
This commit is contained in:
Eugene Zhuravlev
2012-04-28 17:46:13 +02:00
parent df67832271
commit b781d72c92
12 changed files with 1003 additions and 1759 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java
+81 -87
View File
@@ -31,18 +31,18 @@ import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.components.ApplicationComponent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.fileTypes.FileTypeManager;
import com.intellij.openapi.module.Module;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.project.ProjectManager;
import com.intellij.openapi.project.ProjectManagerAdapter;
import com.intellij.openapi.projectRoots.*;
import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl;
import com.intellij.openapi.roots.*;
import com.intellij.openapi.roots.ModuleRootEvent;
import com.intellij.openapi.roots.ModuleRootListener;
import com.intellij.openapi.roots.OrderRootType;
import com.intellij.openapi.roots.libraries.Library;
import com.intellij.openapi.roots.libraries.LibraryTablesRegistrar;
import com.intellij.openapi.util.JDOMUtil;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.ShutDownTracker;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.registry.Registry;
@@ -69,8 +69,6 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.api.*;
import org.jetbrains.jps.cmdline.BuildMain;
import org.jetbrains.jps.incremental.fs.FSState;
import org.jetbrains.jps.incremental.fs.RootDescriptor;
import org.jetbrains.jps.server.ClasspathBootstrap;
import org.jetbrains.jps.server.Server;
@@ -100,6 +98,7 @@ public class BuildManager implements ApplicationComponent{
private final ProjectManager myProjectManager;
private final Map<RequestFuture, Project> myAutomakeFutures = new HashMap<RequestFuture, Project>();
private final Map<String, RequestFuture> myBuildsInProgress = Collections.synchronizedMap(new HashMap<String, RequestFuture>());
private final CompileServerClasspathManager myClasspathManager = new CompileServerClasspathManager();
private final Executor myPooledThreadExecutor = new Executor() {
@Override
@@ -107,6 +106,8 @@ public class BuildManager implements ApplicationComponent{
ApplicationManager.getApplication().executeOnPooledThread(command);
}
};
private final SequentialTaskExecutor myEventsProcessor = new SequentialTaskExecutor(myPooledThreadExecutor);
private final Map<String, ProjectData> myProjectDataMap = Collections.synchronizedMap(new HashMap<String, ProjectData>());
private final ChannelGroup myAllOpenChannels = new DefaultChannelGroup("build-manager");
@@ -198,80 +199,39 @@ public class BuildManager implements ApplicationComponent{
}
public void notifyFilesChanged(final Collection<String> paths) {
// todo: temporarily commented
//doNotify(paths, false);
doNotify(paths, false);
}
public void notifyFilesDeleted(Collection<String> paths) {
// todo: temporarily commented
//doNotify(paths, true);
doNotify(paths, true);
}
private void doNotify(final Collection<String> paths, final boolean notifyDeletion) {
final Project[] openProjects = myProjectManager.getOpenProjects();
myPooledThreadExecutor.execute(new Runnable() {
// ensure events processed in the order they arrived
myEventsProcessor.submit(new Runnable() {
@Override
public void run() {
List<Pair<ProjectFileIndex, ProjectData>> activeProjects = new ArrayList<Pair<ProjectFileIndex, ProjectData>>();
for (Project project : openProjects) {
if (project.isDisposed() || project.isDefault()) {
continue;
}
final String projectPath = getProjectPath(project);
final ProjectData data = myProjectDataMap.get(projectPath);
if (data != null) {
activeProjects.add(Pair.create(ProjectRootManager.getInstance(project).getFileIndex(), data));
}
}
final LocalFileSystem lfs = LocalFileSystem.getInstance();
for (final Pair<ProjectFileIndex, ProjectData> pair : activeProjects) {
final ProjectFileIndex fileIndex = pair.first;
final FSState fsState = pair.second.fsState;
pair.second.requestQueue.submit(new Runnable() {
@Override
public void run() {
ApplicationManager.getApplication().runReadAction(new Runnable() {
@Override
public void run() {
for (String filePath : paths) {
// todo: estimate performance, probably add a way to obtain all data from file index in a single call
final VirtualFile vFile = lfs.findFileByPath(filePath);
if (vFile == null) {
continue;
}
final Module module = fileIndex.getModuleForFile(vFile);
if (module == null) {
continue;
}
final String moduleName = module.getName();
if (!fsState.isInitialized(moduleName)) { // todo: proper sync!
continue;
}
final VirtualFile srcRoot = fileIndex.getSourceRootForFile(vFile);
if (srcRoot == null) {
return;
}
final boolean inTestSources = fileIndex.isInTestSourceContent(vFile);
if (!inTestSources && !fileIndex.isInSourceContent(vFile)) {
continue;
}
try {
final File file = new File(vFile.getPath());
if (notifyDeletion) {
fsState.registerDeleted(moduleName, file, inTestSources, null);
}
else {
fsState.markDirty(file, new RootDescriptor(moduleName, new File(srcRoot.getPath()), inTestSources, false), null);
}
}
catch (IOException e) {
LOG.error(e);
}
}
}
});
synchronized (myProjectDataMap) {
for (Map.Entry<String, ProjectData> entry : myProjectDataMap.entrySet()) {
final ProjectData data = entry.getValue();
if (notifyDeletion) {
data.addDeleted(paths);
}
});
else {
data.addChanged(paths);
}
final RequestFuture future = myBuildsInProgress.get(entry.getKey());
if (future != null && !future.isCancelled() && !future.isDone()) {
final UUID sessionId = future.getRequestID();
final Channel channel = myMessageDispatcher.getConnectedChannel(sessionId);
if (channel != null) {
final CmdlineRemoteProto.Message.ControllerMessage message =
CmdlineRemoteProto.Message.ControllerMessage.newBuilder().setType(
CmdlineRemoteProto.Message.ControllerMessage.Type.FS_EVENT).setFsEvent(data.createNextEvent()).build();
Channels.write(channel, CmdlineProtoUtil.toMessage(sessionId, message));
}
}
}
}
}
});
@@ -346,8 +306,6 @@ public class BuildManager implements ApplicationComponent{
}
}
// todo: avoid FS scan on every start
@Nullable
public RequestFuture scheduleBuild(
final Project project, final boolean isRebuild,
@@ -366,26 +324,32 @@ public class BuildManager implements ApplicationComponent{
myGlobals = globals;
}
final SequentialTaskExecutor sequential;
final FSState fsState;
CmdlineRemoteProto.Message.ControllerMessage.FSEvent currentFSChanges = null;
final SequentialTaskExecutor projectTaskQueue;
synchronized (myProjectDataMap) {
ProjectData data = myProjectDataMap.get(projectPath);
if (data == null) {
data = new ProjectData(new SequentialTaskExecutor(myPooledThreadExecutor), new FSState());
data = new ProjectData(new SequentialTaskExecutor(myPooledThreadExecutor));
myProjectDataMap.put(projectPath, data);
}
sequential = data.requestQueue;
fsState = data.fsState;
handler = new FSStateMessageHandler(data.fsState, handler);
else {
if (!isRebuild) {
currentFSChanges = data.createNextEvent();
}
else {
data.clearFSChanges();
}
}
projectTaskQueue = data.taskQueue;
}
if (isRebuild) {
params = CmdlineProtoUtil.createRebuildRequest(projectPath, userData, globals, fsState);
params = CmdlineProtoUtil.createRebuildRequest(projectPath, userData, globals);
}
else {
params = isMake ?
CmdlineProtoUtil.createMakeRequest(projectPath, modules, artifacts, userData, globals, fsState) :
CmdlineProtoUtil.createForceCompileRequest(projectPath, modules, artifacts, paths, userData, globals, fsState);
CmdlineProtoUtil.createMakeRequest(projectPath, modules, artifacts, userData, globals, currentFSChanges) :
CmdlineProtoUtil.createForceCompileRequest(projectPath, modules, artifacts, paths, userData, globals, currentFSChanges);
}
myMessageDispatcher.registerBuildMessageHandler(sessionId, handler, params);
@@ -410,7 +374,7 @@ public class BuildManager implements ApplicationComponent{
}
});
sequential.submit(new Runnable() {
projectTaskQueue.submit(new Runnable() {
@Override
public void run() {
try {
@@ -418,6 +382,7 @@ public class BuildManager implements ApplicationComponent{
future.cancel(false);
return;
}
myBuildsInProgress.put(projectPath, future);
final Process process = launchBuildProcess(myListenPort, sessionId);
final OSProcessHandler processHandler = new OSProcessHandler(process, null) {
@Override
@@ -452,6 +417,7 @@ public class BuildManager implements ApplicationComponent{
future.getMessageHandler().sessionTerminated();
}
finally {
myBuildsInProgress.remove(projectPath);
future.setDone();
}
}
@@ -794,12 +760,40 @@ public class BuildManager implements ApplicationComponent{
}
private static class ProjectData {
final SequentialTaskExecutor requestQueue;
final FSState fsState;
final SequentialTaskExecutor taskQueue;
private final Set<String> myChanged = new HashSet<String>();
private final Set<String> myDeleted = new HashSet<String>();
private long myNextEventOrdinal = 0L;
private ProjectData(SequentialTaskExecutor requestQueue, FSState fsState) {
this.requestQueue = requestQueue;
this.fsState = fsState;
private ProjectData(SequentialTaskExecutor taskQueue) {
this.taskQueue = taskQueue;
}
public void addChanged(Collection<String> paths) {
myDeleted.removeAll(paths);
myChanged.addAll(paths);
}
public void addDeleted(Collection<String> paths) {
myChanged.removeAll(paths);
myDeleted.addAll(paths);
}
public CmdlineRemoteProto.Message.ControllerMessage.FSEvent createNextEvent() {
final CmdlineRemoteProto.Message.ControllerMessage.FSEvent.Builder builder =
CmdlineRemoteProto.Message.ControllerMessage.FSEvent.newBuilder();
builder.setOrdinal(++myNextEventOrdinal);
builder.addAllChangedPaths(myChanged);
myChanged.clear();
builder.addAllDeletedPaths(myDeleted);
myDeleted.clear();
return builder.build();
}
public void clearFSChanges() {
myNextEventOrdinal = 0L;
myChanged.clear();
myDeleted.clear();
}
}
java/compiler/impl/src/com/intellij/compiler/server/BuildMessageDispatcher.java
+16 -6
View File
@@ -52,16 +52,26 @@ class BuildMessageDispatcher extends SimpleChannelHandler {
public void cancelSession(UUID sessionId) {
if (myCanceledSessions.add(sessionId)) {
final SessionData data = myMessageHandlers.get(sessionId);
if (data != null) {
final Channel channel = data.channel;
if (channel != null) {
Channels.write(channel, CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createCancelCommand()));
}
final Channel channel = getConnectedChannel(sessionId);
if (channel != null) {
Channels.write(channel, CmdlineProtoUtil.toMessage(sessionId, CmdlineProtoUtil.createCancelCommand()));
}
}
}
@Nullable
public Channel getConnectedChannel(final UUID sessionId) {
final SessionData data = myMessageHandlers.get(sessionId);
if (data != null) {
final Channel channel = data.channel;
if (channel != null && channel.isConnected()) {
return channel;
}
}
return null;
}
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
final CmdlineRemoteProto.Message message = (CmdlineRemoteProto.Message)e.getMessage();
java/compiler/impl/src/com/intellij/compiler/server/DefaultMessageHandler.java
-6
View File
@@ -31,15 +31,9 @@ public abstract class DefaultMessageHandler implements BuilderMessageHandler {
case COMPILE_MESSAGE:
handleCompileMessage(msg.getCompileMessage());
break;
case FS_STATE:
handleFSStateMessage(msg.getFsstateMessage());
break;
}
}
protected void handleFSStateMessage(CmdlineRemoteProto.Message.FSStateMessage message) {
}
protected abstract void handleCompileMessage(CmdlineRemoteProto.Message.BuilderMessage.CompileMessage message);
protected abstract void handleBuildEvent(CmdlineRemoteProto.Message.BuilderMessage.BuildEvent event);
java/compiler/impl/src/com/intellij/compiler/server/FSStateMessageHandler.java
-83
View File
@@ -1,83 +0,0 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.compiler.server;
import org.jetbrains.jps.api.CmdlineRemoteProto;
import org.jetbrains.jps.incremental.fs.FSState;
import java.io.File;
import java.util.*;
/**
* @author Eugene Zhuravlev
* Date: 4/18/12
*/
public class FSStateMessageHandler extends DefaultMessageHandler {
private final FSState myFsState;
private final DefaultMessageHandler myDelegate;
protected FSStateMessageHandler(FSState fsState, DefaultMessageHandler delegate) {
myFsState = fsState;
myDelegate = delegate;
}
@Override
public void handleFailure(CmdlineRemoteProto.Message.Failure failure) {
myDelegate.handleFailure(failure);
}
@Override
public void sessionTerminated() {
myDelegate.sessionTerminated();
}
protected void handleFSStateMessage(CmdlineRemoteProto.Message.FSStateMessage message) {
try {
myDelegate.handleFSStateMessage(message);
}
finally {
final String moduleName = message.getModuleName();
final List<String> deleteProduction = message.getDeletedProductionList();
final List<String> deletedTests = message.getDeletedTestsList();
final Map<File,Set<File>> recompileProduction = new HashMap<File, Set<File>>();
final Map<File, Set<File>> recompileTests = new HashMap<File, Set<File>>();
for (CmdlineRemoteProto.Message.FSStateMessage.RootDelta delta : message.getRecompileDeltaList()) {
final File root = new File(delta.getRoot());
final Map<File, Set<File>> map = delta.getTestSources()? recompileTests : recompileProduction;
Set<File> files = map.get(root);
if (files == null) {
files = new HashSet<File>();
map.put(root, files);
}
for (String path : delta.getPathList()) {
files.add(new File(path));
}
}
myFsState.init(moduleName, deleteProduction, deletedTests, recompileProduction, recompileTests);
}
}
protected void handleCompileMessage(CmdlineRemoteProto.Message.BuilderMessage.CompileMessage message) {
myDelegate.handleCompileMessage(message);
}
protected void handleBuildEvent(CmdlineRemoteProto.Message.BuilderMessage.BuildEvent event) {
myDelegate.handleBuildEvent(event);
}
}