external javac server refactoring

This commit is contained in:
Eugene Zhuravlev
2015-06-17 10:01:27 +02:00
parent ee04a99651
commit d850e92f44
6 changed files with 227 additions and 224 deletions
@@ -52,7 +52,7 @@ import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration;
import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping;
import org.jetbrains.jps.incremental.storage.OutputToTargetRegistry;
import org.jetbrains.jps.indices.ModuleExcludeIndex;
import org.jetbrains.jps.javac.ExternalJavacServer;
import org.jetbrains.jps.javac.ExternalJavacManager;
import org.jetbrains.jps.javac.JavacMain;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.java.compiler.JpsJavaCompilerConfiguration;
@@ -313,10 +313,10 @@ public class IncProjectBuilder {
pd.timestamps.getStorage().force();
pd.dataManager.flush(false);
}
final ExternalJavacServer server = ExternalJavacServer.KEY.get(context);
final ExternalJavacManager server = ExternalJavacManager.KEY.get(context);
if (server != null) {
server.stop();
ExternalJavacServer.KEY.set(context, null);
ExternalJavacManager.KEY.set(context, null);
}
}
@@ -59,8 +59,10 @@ import org.jetbrains.jps.model.module.JpsModuleType;
import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService;
import org.jetbrains.jps.model.serialization.PathMacroUtil;
import org.jetbrains.jps.service.JpsServiceManager;
import org.jetbrains.jps.service.SharedThreadPool;
import javax.tools.*;
import javax.tools.Diagnostic;
import javax.tools.JavaFileObject;
import java.io.*;
import java.net.ServerSocket;
import java.util.*;
@@ -392,9 +394,9 @@ public class JavaBuilder extends ModuleLevelBuilder {
}
final List<String> vmOptions = getCompilationVMOptions(context, compilingTool);
final ExternalJavacServer server = ensureJavacServerStarted(context);
final ExternalJavacManager server = ensureJavacServerStarted(context);
rc = server.forkJavac(
context, options, vmOptions, files, classpath, _platformCp, sourcePath, outs, diagnosticSink, classesConsumer, sdkHome, compilingTool
sdkHome, getExternalJavacHeapSize(context), vmOptions, options, _platformCp, classpath, sourcePath, files, outs, diagnosticSink, classesConsumer, compilingTool, context.getCancelStatus()
);
}
return rc;
@@ -404,6 +406,12 @@ public class JavaBuilder extends ModuleLevelBuilder {
}
}
private static int getExternalJavacHeapSize(CompileContext context) {
final JpsProject project = context.getProjectDescriptor().getProject();
final JpsJavaCompilerConfiguration config = JpsJavaExtensionService.getInstance().getOrCreateCompilerConfiguration(project);
final JpsJavaCompilerOptions options = config.getCurrentCompilerOptions();
return options.MAXIMUM_HEAP_SIZE;
}
@Nullable
public static String validateCycle(ModuleChunk chunk,
JpsJavaExtensionService javaExt,
@@ -517,15 +525,15 @@ public class JavaBuilder extends ModuleLevelBuilder {
});
}
private static synchronized ExternalJavacServer ensureJavacServerStarted(@NotNull CompileContext context) throws Exception {
ExternalJavacServer server = ExternalJavacServer.KEY.get(context);
private static synchronized ExternalJavacManager ensureJavacServerStarted(@NotNull CompileContext context) throws Exception {
ExternalJavacManager server = ExternalJavacManager.KEY.get(context);
if (server != null) {
return server;
}
final int listenPort = findFreePort();
server = new ExternalJavacServer();
server = new ExternalJavacManager(Utils.getSystemRoot(), SharedThreadPool.getInstance());
server.start(listenPort);
ExternalJavacServer.KEY.set(context, server);
ExternalJavacManager.KEY.set(context, server);
return server;
}
@@ -584,7 +592,7 @@ public class JavaBuilder extends ModuleLevelBuilder {
}
catch (IOException e) {
e.printStackTrace(System.err);
return ExternalJavacServer.DEFAULT_SERVER_PORT;
return ExternalJavacManager.DEFAULT_SERVER_PORT;
}
}
@@ -903,7 +911,15 @@ public class JavaBuilder extends ModuleLevelBuilder {
public void outputLineAvailable(String line) {
if (!StringUtil.isEmpty(line)) {
if (line.contains("java.lang.OutOfMemoryError")) {
if (line.startsWith(ExternalJavacManager.STDOUT_LINE_PREFIX)) {
//noinspection UseOfSystemOutOrSystemErr
System.out.println(line);
}
else if (line.startsWith(ExternalJavacManager.STDERR_LINE_PREFIX)) {
//noinspection UseOfSystemOutOrSystemErr
System.err.println(line);
}
else if (line.contains("java.lang.OutOfMemoryError")) {
myContext.processMessage(new CompilerMessage(BUILDER_NAME, BuildMessage.Kind.ERROR, "OutOfMemoryError: insufficient memory"));
myErrorCount++;
}
@@ -15,7 +15,15 @@
*/
package org.jetbrains.jps.javac;
import com.intellij.execution.process.BaseOSProcessHandler;
import com.intellij.execution.process.ProcessAdapter;
import com.intellij.execution.process.ProcessEvent;
import com.intellij.execution.process.ProcessOutputTypes;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.concurrency.Semaphore;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.*;
@@ -32,19 +40,17 @@ import io.netty.util.AttributeKey;
import io.netty.util.concurrent.ImmediateEventExecutor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.api.CanceledStatus;
import org.jetbrains.jps.builders.java.JavaCompilingTool;
import org.jetbrains.jps.incremental.CompileContext;
import org.jetbrains.jps.cmdline.ClasspathBootstrap;
import org.jetbrains.jps.incremental.GlobalContextKey;
import org.jetbrains.jps.incremental.Utils;
import org.jetbrains.jps.model.JpsProject;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.java.compiler.JpsJavaCompilerConfiguration;
import org.jetbrains.jps.model.java.compiler.JpsJavaCompilerOptions;
import org.jetbrains.jps.service.SharedThreadPool;
import org.jetbrains.jps.service.ThreadExecutor;
import javax.tools.*;
import javax.tools.Diagnostic;
import java.io.File;
import java.util.*;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
/**
@@ -52,17 +58,26 @@ import java.util.concurrent.TimeUnit;
* Date: 1/22/12
*/
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public class ExternalJavacServer {
public class ExternalJavacManager {
private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.javac.ExternalJavacServer");
public static final GlobalContextKey<ExternalJavacServer> KEY = GlobalContextKey.create("_external_javac_server_");
public static final GlobalContextKey<ExternalJavacManager> KEY = GlobalContextKey.create("_external_javac_server_");
public static final int DEFAULT_SERVER_PORT = 7878;
public static final String STDOUT_LINE_PREFIX = "JAVAC_PROCESS[STDOUT]";
public static final String STDERR_LINE_PREFIX = "JAVAC_PROCESS[STDERR]";
private static final AttributeKey<JavacProcessDescriptor> SESSION_DESCRIPTOR = AttributeKey.valueOf("ExternalJavacServer.JavacProcessDescriptor");
private final File mySystemRoot;
private final ThreadExecutor myThreadExecutor;
private ChannelRegistrar myChannelRegistrar;
private final Map<UUID, JavacProcessDescriptor> myMessageHandlers = new HashMap<UUID, JavacProcessDescriptor>();
private int myListenPort = DEFAULT_SERVER_PORT;
public ExternalJavacManager(final File systemRoot, @NotNull ThreadExecutor threadExecutor) {
mySystemRoot = systemRoot;
myThreadExecutor = threadExecutor;
}
public void start(int listenPort) {
final ServerBootstrap bootstrap = new ServerBootstrap().group(new NioEventLoopGroup(1, SharedThreadPool.getInstance())).channel(NioServerSocketChannel.class);
bootstrap.childOption(ChannelOption.TCP_NODELAY, true).childOption(ChannelOption.SO_KEEPALIVE, true);
@@ -83,23 +98,16 @@ public class ExternalJavacServer {
myListenPort = listenPort;
}
private static int getExternalJavacHeapSize(CompileContext context) {
final JpsProject project = context.getProjectDescriptor().getProject();
final JpsJavaCompilerConfiguration config = JpsJavaExtensionService.getInstance().getOrCreateCompilerConfiguration(project);
final JpsJavaCompilerOptions options = config.getCurrentCompilerOptions();
return options.MAXIMUM_HEAP_SIZE;
}
public boolean forkJavac(CompileContext context, List<String> options,
List<String> vmOptions, Collection<File> files,
Collection<File> classpath,
public boolean forkJavac(final String javaHome, final int heapSize, List<String> vmOptions, List<String> options,
Collection<File> platformCp,
Collection<File> classpath,
Collection<File> sourcePath,
Collection<File> files,
Map<File, Set<File>> outs,
DiagnosticOutputConsumer diagnosticSink,
OutputFileConsumer outputSink,
final String javaHome, final JavaCompilingTool compilingTool) {
final DiagnosticOutputConsumer diagnosticSink, OutputFileConsumer outputSink,
final JavaCompilingTool compilingTool,
final CanceledStatus cancelStatus) {
final ExternalJavacMessageHandler rh = new ExternalJavacMessageHandler(diagnosticSink, outputSink, getEncodingName(options));
final JavacRemoteProto.Message.Request request = JavacProtoUtil.createCompilationRequest(options, files, classpath, platformCp, sourcePath, outs);
final UUID uuid = UUID.randomUUID();
@@ -108,9 +116,27 @@ public class ExternalJavacServer {
myMessageHandlers.put(uuid, processDescriptor);
}
try {
final JavacServerBootstrap.ExternalJavacProcessHandler processHandler = JavacServerBootstrap.launchExternalJavacProcess(
uuid, javaHome, getExternalJavacHeapSize(context), myListenPort, Utils.getSystemRoot(), vmOptions, compilingTool
final ExternalJavacProcessHandler processHandler = launchExternalJavacProcess(
uuid, javaHome, heapSize, myListenPort, mySystemRoot, vmOptions, compilingTool
);
processHandler.addProcessListener(new ProcessAdapter() {
public void onTextAvailable(ProcessEvent event, Key outputType) {
final String text = event.getText();
if (!StringUtil.isEmptyOrSpaces(text)) {
String prefix = null;
if (outputType == ProcessOutputTypes.STDOUT) {
prefix = STDOUT_LINE_PREFIX;
}
else if (outputType == ProcessOutputTypes.STDERR) {
prefix = STDERR_LINE_PREFIX;
}
if (prefix != null) {
diagnosticSink.outputLineAvailable(prefix + ": " + text);
}
}
}
});
processHandler.startNotify();
while (!processDescriptor.waitFor(300L)) {
if (processHandler.isProcessTerminated() && processDescriptor.channel == null && processHandler.getExitCode() != 0) {
@@ -118,7 +144,7 @@ public class ExternalJavacServer {
processDescriptor.setDone();
break;
}
if (context.getCancelStatus().isCanceled()) {
if (cancelStatus.isCanceled()) {
processDescriptor.cancelBuild();
}
}
@@ -163,6 +189,130 @@ public class ExternalJavacServer {
myChannelRegistrar.close().awaitUninterruptibly();
}
private ExternalJavacProcessHandler launchExternalJavacProcess(UUID uuid, String sdkHomePath,
int heapSize,
int port,
File workingDir,
List<String> vmOptions,
JavaCompilingTool compilingTool) throws Exception {
final List<String> cmdLine = new ArrayList<String>();
appendParam(cmdLine, getVMExecutablePath(sdkHomePath));
//appendParam(cmdLine, "-XX:MaxPermSize=150m");
//appendParam(cmdLine, "-XX:ReservedCodeCacheSize=64m");
appendParam(cmdLine, "-Djava.awt.headless=true");
final int xms = heapSize / 2;
if (xms > 32) {
appendParam(cmdLine, "-Xms" + xms + "m");
}
appendParam(cmdLine, "-Xmx" + heapSize + "m");
// debugging
//appendParam(cmdLine, "-XX:+HeapDumpOnOutOfMemoryError");
//appendParam(cmdLine, "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5009");
// javac's VM should use the same default locale that IDEA uses in order for javac to print messages in 'correct' language
final String encoding = System.getProperty("file.encoding");
if (encoding != null) {
appendParam(cmdLine, "-Dfile.encoding=" + encoding);
}
final String lang = System.getProperty("user.language");
if (lang != null) {
//noinspection HardCodedStringLiteral
appendParam(cmdLine, "-Duser.language=" + lang);
}
final String country = System.getProperty("user.country");
if (country != null) {
//noinspection HardCodedStringLiteral
appendParam(cmdLine, "-Duser.country=" + country);
}
//noinspection HardCodedStringLiteral
final String region = System.getProperty("user.region");
if (region != null) {
//noinspection HardCodedStringLiteral
appendParam(cmdLine, "-Duser.region=" + region);
}
appendParam(cmdLine, "-D" + ExternalJavacProcess.JPS_JAVA_COMPILING_TOOL_PROPERTY + "=" + compilingTool.getId());
// this will disable standard extensions to ensure javac is loaded from the right tools.jar
appendParam(cmdLine, "-Djava.ext.dirs=");
appendParam(cmdLine, "-Dlog4j.defaultInitOverride=true");
for (String option : vmOptions) {
appendParam(cmdLine, option);
}
appendParam(cmdLine, "-classpath");
final List<File> cp = ClasspathBootstrap.getExternalJavacProcessClasspath(sdkHomePath, compilingTool);
final StringBuilder classpath = new StringBuilder();
for (File file : cp) {
if (classpath.length() > 0) {
classpath.append(File.pathSeparator);
}
classpath.append(file.getPath());
}
appendParam(cmdLine, classpath.toString());
appendParam(cmdLine, ExternalJavacProcess.class.getName());
appendParam(cmdLine, uuid.toString());
appendParam(cmdLine, "127.0.0.1");
appendParam(cmdLine, Integer.toString(port));
workingDir.mkdirs();
appendParam(cmdLine, FileUtil.toSystemIndependentName(workingDir.getPath()));
final ProcessBuilder builder = new ProcessBuilder(cmdLine);
builder.directory(workingDir);
final Process process = builder.start();
return new ExternalJavacProcessHandler(process, myThreadExecutor);
}
private static void appendParam(List<String> cmdLine, String param) {
if (SystemInfo.isWindows) {
if (param.contains("\"")) {
param = StringUtil.replace(param, "\"", "\\\"");
}
else if (param.length() == 0) {
param = "\"\"";
}
}
cmdLine.add(param);
}
private static String getVMExecutablePath(String sdkHome) {
return sdkHome + "/bin/java";
}
private static class ExternalJavacProcessHandler extends BaseOSProcessHandler {
@NotNull
private final ThreadExecutor myExecutorService;
private volatile int myExitCode;
ExternalJavacProcessHandler(Process process, @NotNull ThreadExecutor executorService) {
super(process, null, null);
myExecutorService = executorService;
addProcessListener(new ProcessAdapter() {
@Override
public void processTerminated(ProcessEvent event) {
myExitCode = event.getExitCode();
}
});
}
@Override
protected Future<?> executeOnPooledThread(Runnable task) {
return myExecutorService.executeOnPooledThread(task);
}
public int getExitCode() {
return myExitCode;
}
}
@ChannelHandler.Sharable
private class CompilationRequestsHandler extends SimpleChannelInboundHandler<JavacRemoteProto.Message> {
@Override
@@ -1,182 +0,0 @@
/*
* Copyright 2000-2012 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jps.javac;
import com.intellij.execution.process.BaseOSProcessHandler;
import com.intellij.execution.process.ProcessAdapter;
import com.intellij.execution.process.ProcessEvent;
import com.intellij.execution.process.ProcessOutputTypes;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import org.jetbrains.jps.builders.java.JavaCompilingTool;
import org.jetbrains.jps.cmdline.ClasspathBootstrap;
import org.jetbrains.jps.service.SharedThreadPool;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Future;
/**
* @author Eugene Zhuravlev
* Date: 1/24/12
*/
public class JavacServerBootstrap {
public static ExternalJavacProcessHandler launchExternalJavacProcess(UUID uuid, String sdkHomePath,
int heapSize,
int port,
File workingDir,
List<String> vmOptions,
JavaCompilingTool compilingTool) throws Exception {
final List<String> cmdLine = new ArrayList<String>();
appendParam(cmdLine, getVMExecutablePath(sdkHomePath));
//appendParam(cmdLine, "-XX:MaxPermSize=150m");
//appendParam(cmdLine, "-XX:ReservedCodeCacheSize=64m");
appendParam(cmdLine, "-Djava.awt.headless=true");
final int xms = heapSize / 2;
if (xms > 32) {
appendParam(cmdLine, "-Xms" + xms + "m");
}
appendParam(cmdLine, "-Xmx" + heapSize + "m");
// debugging
//appendParam(cmdLine, "-XX:+HeapDumpOnOutOfMemoryError");
//appendParam(cmdLine, "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5009");
// javac's VM should use the same default locale that IDEA uses in order for javac to print messages in 'correct' language
final String encoding = System.getProperty("file.encoding");
if (encoding != null) {
appendParam(cmdLine, "-Dfile.encoding=" + encoding);
}
final String lang = System.getProperty("user.language");
if (lang != null) {
//noinspection HardCodedStringLiteral
appendParam(cmdLine, "-Duser.language=" + lang);
}
final String country = System.getProperty("user.country");
if (country != null) {
//noinspection HardCodedStringLiteral
appendParam(cmdLine, "-Duser.country=" + country);
}
//noinspection HardCodedStringLiteral
final String region = System.getProperty("user.region");
if (region != null) {
//noinspection HardCodedStringLiteral
appendParam(cmdLine, "-Duser.region=" + region);
}
appendParam(cmdLine, "-D" + ExternalJavacProcess.JPS_JAVA_COMPILING_TOOL_PROPERTY + "=" + compilingTool.getId());
// this will disable standard extensions to ensure javac is loaded from the right tools.jar
appendParam(cmdLine, "-Djava.ext.dirs=");
appendParam(cmdLine, "-Dlog4j.defaultInitOverride=true");
for (String option : vmOptions) {
appendParam(cmdLine, option);
}
appendParam(cmdLine, "-classpath");
final List<File> cp = ClasspathBootstrap.getExternalJavacProcessClasspath(sdkHomePath, compilingTool);
final StringBuilder classpath = new StringBuilder();
for (File file : cp) {
if (classpath.length() > 0) {
classpath.append(File.pathSeparator);
}
classpath.append(file.getPath());
}
appendParam(cmdLine, classpath.toString());
appendParam(cmdLine, org.jetbrains.jps.javac.ExternalJavacProcess.class.getName());
appendParam(cmdLine, uuid.toString());
appendParam(cmdLine, "127.0.0.1");
appendParam(cmdLine, Integer.toString(port));
workingDir.mkdirs();
appendParam(cmdLine, FileUtil.toSystemIndependentName(workingDir.getPath()));
final ProcessBuilder builder = new ProcessBuilder(cmdLine);
builder.directory(workingDir);
final Process process = builder.start();
final ExternalJavacProcessHandler processHandler = new ExternalJavacProcessHandler(process);
processHandler.addProcessListener(new ProcessAdapter() {
@Override
public void processTerminated(ProcessEvent event) {
processHandler.setExitCode(event.getExitCode());
}
public void onTextAvailable(ProcessEvent event, Key outputType) {
final String text = event.getText();
if (!StringUtil.isEmptyOrSpaces(text)) {
if (outputType == ProcessOutputTypes.STDOUT) {
System.out.print("JAVAC_PROCESS: " + text);
}
else if (outputType == ProcessOutputTypes.STDERR) {
System.err.print("JAVAC_PROCESS: " + text);
}
}
}
});
processHandler.startNotify();
return processHandler;
}
private static void appendParam(List<String> cmdLine, String param) {
if (SystemInfo.isWindows) {
if (param.contains("\"")) {
param = StringUtil.replace(param, "\"", "\\\"");
}
else if (param.length() == 0) {
param = "\"\"";
}
}
cmdLine.add(param);
}
public static String getVMExecutablePath(String sdkHome) {
return sdkHome + "/bin/java";
}
public static class ExternalJavacProcessHandler extends BaseOSProcessHandler {
private volatile int myExitCode;
ExternalJavacProcessHandler(Process process) {
super(process, null, null);
}
@Override
protected Future<?> executeOnPooledThread(Runnable task) {
return SharedThreadPool.getInstance().executeOnPooledThread(task);
}
void setExitCode(int exitCode) {
myExitCode = exitCode;
}
public int getExitCode() {
return myExitCode;
}
}
}
@@ -15,16 +15,12 @@
*/
package org.jetbrains.jps.service;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
/**
* @author nik
*/
public abstract class SharedThreadPool implements Executor {
public abstract class SharedThreadPool implements ThreadExecutor {
public static SharedThreadPool getInstance() {
return JpsServiceManager.getInstance().getService(SharedThreadPool.class);
}
public abstract Future<?> executeOnPooledThread(Runnable action);
}
@@ -0,0 +1,23 @@
/*
* Copyright 2000-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jps.service;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
public interface ThreadExecutor extends Executor {
Future<?> executeOnPooledThread(Runnable action);
}