diff --git a/java/compiler/impl/src/com/intellij/compiler/CompilerManagerImpl.java b/java/compiler/impl/src/com/intellij/compiler/CompilerManagerImpl.java index e2a20c46b6b8..df9bd71ed834 100644 --- a/java/compiler/impl/src/com/intellij/compiler/CompilerManagerImpl.java +++ b/java/compiler/impl/src/com/intellij/compiler/CompilerManagerImpl.java @@ -4,6 +4,8 @@ package com.intellij.compiler; import com.intellij.codeInspection.InspectionManager; import com.intellij.compiler.impl.*; import com.intellij.compiler.server.BuildManager; +import com.intellij.execution.process.ProcessIOExecutorService; +import com.intellij.ide.IdeEventQueue; import com.intellij.openapi.Disposable; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.compiler.Compiler; @@ -20,6 +22,7 @@ import com.intellij.openapi.projectRoots.*; import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.profile.codeInspection.InspectionProjectProfileManager; @@ -36,12 +39,10 @@ import org.jetbrains.annotations.TestOnly; import org.jetbrains.jps.api.CanceledStatus; import org.jetbrains.jps.builders.impl.java.JavacCompilerTool; import org.jetbrains.jps.incremental.BinaryContent; -import org.jetbrains.jps.javac.DiagnosticOutputConsumer; -import org.jetbrains.jps.javac.ExternalJavacManager; -import org.jetbrains.jps.javac.OutputFileConsumer; -import org.jetbrains.jps.javac.OutputFileObject; +import org.jetbrains.jps.javac.*; -import javax.tools.*; +import javax.tools.Diagnostic; +import javax.tools.JavaFileObject; import java.io.File; import java.io.IOException; import java.lang.reflect.Array; @@ -52,6 +53,7 @@ import java.util.concurrent.TimeUnit; public class CompilerManagerImpl extends CompilerManager { private static final Logger LOG = Logger.getInstance("#com.intellij.compiler.CompilerManagerImpl"); + private static final int IDLE_PROCESSES_CHECK_PERIOD = 10000; // check idle javac processes every 10 second when IDE is idle private final Project myProject; @@ -367,7 +369,6 @@ public class CompilerManagerImpl extends CompilerManager { Collection sourcePath, Collection files, File outputDir) throws IOException, CompilationException { - final Pair runtime = BuildManager.getJavacRuntimeSdk(myProject); final Sdk sdk = runtime.getFirst(); @@ -407,10 +408,12 @@ public class CompilerManagerImpl extends CompilerManager { final Map> outs = Collections.singletonMap(outputDir, sourceRoots); final ExternalJavacManager javacManager = getJavacManager(); + final CompilationPaths paths = CompilationPaths.create(platformCp, classpath, upgradeModulePath, modulePath, sourcePath); + // do not keep process alive in tests since every test expects all spawned processes to terminate in teardown boolean compiledOk = javacManager != null && javacManager.forkJavac( - javaHome, -1, Collections.emptyList(), options, platformCp, classpath, upgradeModulePath, modulePath, sourcePath, files, outs, diagnostic, outputCollector, - new JavacCompilerTool(), CanceledStatus.NULL - ); + javaHome, -1, Collections.emptyList(), options, paths, files, outs, diagnostic, outputCollector, + new JavacCompilerTool(), CanceledStatus.NULL, !ApplicationManager.getApplication().isUnitTestMode() + ).get(); if (!compiledOk) { final List messages = new SmartList<>(); @@ -460,9 +463,26 @@ public class CompilerManagerImpl extends CompilerManager { return null; // should not happen for real projects } final int listenPort = NetUtils.findAvailableSocketPort(); - manager = new ExternalJavacManager(compilerWorkingDir); + manager = new ExternalJavacManager( + compilerWorkingDir, ProcessIOExecutorService.INSTANCE, Registry.intValue("compiler.external.javac.keep.alive.timeout", 5*60*1000) + ); manager.start(listenPort); myExternalJavacManager = manager; + IdeEventQueue.getInstance().addIdleListener(new Runnable() { + @Override + public void run() { + final ExternalJavacManager manager; + synchronized (CompilerManagerImpl.this) { + manager = myExternalJavacManager; + } + if (manager != null) { + manager.shutdownIdleProcesses(); + } + else { + IdeEventQueue.getInstance().removeIdleListener(this); + } + } + }, IDLE_PROCESSES_CHECK_PERIOD); } } } diff --git a/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacMessageHandler.java b/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacMessageHandler.java index c84d5d33374a..d49a630b1849 100644 --- a/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacMessageHandler.java +++ b/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacMessageHandler.java @@ -20,7 +20,8 @@ import com.google.protobuf.MessageLite; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.incremental.BinaryContent; -import javax.tools.*; +import javax.tools.Diagnostic; +import javax.tools.JavaFileObject; import java.io.File; import java.net.URI; import java.util.Collection; @@ -44,6 +45,10 @@ public class ExternalJavacMessageHandler { myEncodingName = encodingName; } + public DiagnosticOutputConsumer getDiagnosticSink() { + return myDiagnosticSink; + } + public boolean handleMessage(MessageLite message) { try { final JavacRemoteProto.Message msg = (JavacRemoteProto.Message)message; diff --git a/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacProcess.java b/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacProcess.java index c3e17633357d..4c81ee60f154 100644 --- a/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacProcess.java +++ b/jps/jps-builders-6/src/org/jetbrains/jps/javac/ExternalJavacProcess.java @@ -19,12 +19,11 @@ import org.jetbrains.jps.api.CanceledStatus; import org.jetbrains.jps.builders.impl.java.JavacCompilerTool; import org.jetbrains.jps.builders.java.JavaCompilingTool; -import javax.tools.*; +import javax.tools.Diagnostic; +import javax.tools.JavaFileObject; import java.io.File; import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; /** * @author Eugene Zhuravlev @@ -33,8 +32,9 @@ public class ExternalJavacProcess { public static final String JPS_JAVA_COMPILING_TOOL_PROPERTY = "jps.java.compiling.tool"; private final ChannelInitializer myChannelInitializer; private final EventLoopGroup myEventLoopGroup; + private final boolean myKeepRunning; private volatile ChannelFuture myConnectFuture; - private volatile CancelHandler myCancelHandler; + private final ConcurrentMap myCanceled = new ConcurrentHashMap(); private final ExecutorService myThreadPool = Executors.newCachedThreadPool(); static { @@ -46,7 +46,8 @@ public class ExternalJavacProcess { InternalLoggerFactory.setDefaultFactory(new Log4JLoggerFactory()); } - public ExternalJavacProcess() { + public ExternalJavacProcess(boolean keepRunning) { + myKeepRunning = keepRunning; final JavacRemoteProto.Message msgDefaultInstance = JavacRemoteProto.Message.getDefaultInstance(); myEventLoopGroup = new NioEventLoopGroup(1, myThreadPool); @@ -65,12 +66,16 @@ public class ExternalJavacProcess { //static volatile long myGlobalStart; + /** + * @param args: SessionUUID, host, port, + */ public static void main(String[] args) { //myGlobalStart = System.currentTimeMillis(); UUID uuid = null; String host = null; int port = -1; - if (args.length > 0) { + boolean keepRunning = false; // keep running after compilation ends until explicit shutdown + if (args.length >= 3) { try { uuid = UUID.fromString(args[0]); } @@ -88,13 +93,17 @@ public class ExternalJavacProcess { System.err.println("Error parsing port: " + e.getMessage()); System.exit(-1); } + + if (args.length > 3) { + keepRunning = Boolean.valueOf(args[3]); + } } else { - System.err.println("Insufficient parameters"); + System.err.println("Insufficient number of parameters"); System.exit(-1); } - final ExternalJavacProcess process = new ExternalJavacProcess(); + final ExternalJavacProcess process = new ExternalJavacProcess(keepRunning); try { //final long connectStart = System.currentTimeMillis(); if (process.connect(host, port)) { @@ -137,7 +146,7 @@ public class ExternalJavacProcess { Collection sourcePath, Map> outs, final CanceledStatus canceledStatus) { - //final long compileStart = System.currentTimeMillis(); + final long compileStart = System.currentTimeMillis(); //System.err.println("Compile start; since global start: " + (compileStart - myGlobalStart)); final DiagnosticOutputConsumer diagnostic = new DiagnosticOutputConsumer() { @Override @@ -188,10 +197,11 @@ public class ExternalJavacProcess { e.printStackTrace(System.err); return JavacProtoUtil.toMessage(sessionId, JavacProtoUtil.createFailure(e.getMessage(), e)); } - //finally { - // final long compileEnd = System.currentTimeMillis(); - // System.err.println("Compiled in " + (compileEnd - compileStart) + " ms; since global start: " + (compileEnd - myGlobalStart)); - //} + finally { + final long compileEnd = System.currentTimeMillis(); + System.err.println("Compiled in " + (compileEnd - compileStart) + " ms"); + //System.err.println("Compiled in " + (compileEnd - compileStart) + " ms; since global start: " + (compileEnd - myGlobalStart)); + } } private static JavaCompilingTool getCompilingTool() { @@ -221,7 +231,7 @@ public class ExternalJavacProcess { final JavacRemoteProto.Message.Request request = message.getRequest(); final JavacRemoteProto.Message.Request.Type requestType = request.getRequestType(); if (requestType == JavacRemoteProto.Message.Request.Type.COMPILE) { - if (myCancelHandler == null) { // if not running yet + if (myCanceled.putIfAbsent(sessionId, Boolean.FALSE) == null) { // if not running yet final List options = request.getOptionList(); final List files = toFiles(request.getFileList()); final List cp = toFiles(request.getClasspathList()); @@ -238,20 +248,23 @@ public class ExternalJavacProcess { } outs.put(new File(outputGroup.getOutputRoot()), srcRoots); } - - final CancelHandler cancelHandler = new CancelHandler(); - myCancelHandler = cancelHandler; myThreadPool.submit(new Runnable() { @Override public void run() { try { - context.channel().writeAndFlush( - compile(context, sessionId, options, files, cp, platformCp, modulePath, upgradeModulePath, srcPath, outs, cancelHandler) - ).awaitUninterruptibly(); + final JavacRemoteProto.Message result = compile(context, sessionId, options, files, cp, platformCp, modulePath, upgradeModulePath, srcPath, outs, new CanceledStatus() { + @Override + public boolean isCanceled() { + return Boolean.TRUE.equals(myCanceled.get(sessionId)); + } + }); + context.channel().writeAndFlush(result).awaitUninterruptibly(); } finally { - myCancelHandler = null; - ExternalJavacProcess.this.stop(); + myCanceled.remove(sessionId); // state cleanup + if (!myKeepRunning) { // todo: also check that no other process is running + ExternalJavacProcess.this.stop(); + } Thread.interrupted(); // reset interrupted status } } @@ -259,10 +272,15 @@ public class ExternalJavacProcess { } } else if (requestType == JavacRemoteProto.Message.Request.Type.CANCEL){ - cancelBuild(); + cancelBuild(sessionId); } else if (requestType == JavacRemoteProto.Message.Request.Type.SHUTDOWN){ - cancelBuild(); + // cancel all running builds + // todo: optionally wait for all builds to complete and only then shutdown + for (UUID uuid : myCanceled.keySet()) { + // todo: do we really need to wait for cancelled sessions to terminate? + cancelBuild(uuid); + } new Thread("StopThread") { @Override public void run() { @@ -313,26 +331,8 @@ public class ExternalJavacProcess { return files; } - public void cancelBuild() { - final CancelHandler cancelHandler = myCancelHandler; - if (cancelHandler != null) { - cancelHandler.cancel(); - } + public void cancelBuild(UUID sessionId) { + myCanceled.replace(sessionId, Boolean.FALSE, Boolean.TRUE); } - private static class CancelHandler implements CanceledStatus { - private volatile boolean myIsCanceled = false; - - private CancelHandler() { - } - - public void cancel() { - myIsCanceled = true; - } - - @Override - public boolean isCanceled() { - return myIsCanceled; - } - } } \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java index 0a19a562cc9b..6c542cf48a06 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java @@ -55,7 +55,8 @@ import org.jetbrains.jps.model.serialization.PathMacroUtil; import org.jetbrains.jps.service.JpsServiceManager; import org.jetbrains.jps.service.SharedThreadPool; -import javax.tools.*; +import javax.tools.Diagnostic; +import javax.tools.JavaFileObject; import java.io.File; import java.io.FileFilter; import java.io.IOException; @@ -452,12 +453,11 @@ public class JavaBuilder extends ModuleLevelBuilder { else { updateCompilerUsageStatistics(context, "javac " + forkSdk.getSecond(), chunk); final ExternalJavacManager server = ensureJavacServerStarted(context); + final CompilationPaths paths = CompilationPaths.create(platformCp, classPath, upgradeModulePath, modulePath, sourcePath); rc = server.forkJavac( - forkSdk.getFirst(), - Utils.suggestForkedCompilerHeapSize(), - vmOptions, options, platformCp, classPath, upgradeModulePath, modulePath, sourcePath, - files, outs, diagnosticSink, classesConsumer, compilingTool, context.getCancelStatus() - ); + forkSdk.getFirst(), Utils.suggestForkedCompilerHeapSize(), + vmOptions, options, paths, files, outs, diagnosticSink, classesConsumer, compilingTool, context.getCancelStatus(), false + ).get(); } return rc; } @@ -671,13 +671,13 @@ public class JavaBuilder extends ModuleLevelBuilder { return server; } final int listenPort = findFreePort(); - server = new ExternalJavacManager(Utils.getSystemRoot()) { + server = new ExternalJavacManager(Utils.getSystemRoot(), SharedThreadPool.getInstance()) { @Override - protected ExternalJavacProcessHandler createProcessHandler(@NotNull Process process, @NotNull String commandLine) { - return new ExternalJavacProcessHandler(process, commandLine) { - @Override + protected ExternalJavacProcessHandler createProcessHandler(UUID processId, @NotNull Process process, @NotNull String commandLine, boolean keepProcessAlive) { + return new ExternalJavacProcessHandler(processId, process, commandLine, keepProcessAlive) { @NotNull - protected Future executeOnPooledThread(@NotNull Runnable task) { + @Override + public Future executeTask(@NotNull Runnable task) { return SharedThreadPool.getInstance().executeOnPooledThread(task); } }; diff --git a/jps/jps-builders/src/org/jetbrains/jps/javac/CompilationPaths.java b/jps/jps-builders/src/org/jetbrains/jps/javac/CompilationPaths.java new file mode 100644 index 000000000000..5816eb404ef2 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/javac/CompilationPaths.java @@ -0,0 +1,121 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.jps.javac; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.util.Collection; +import java.util.Collections; + +/** + * @author Eugene Zhuravlev + * Date: 14-Nov-18 + */ +public class CompilationPaths { + private final Collection myPlatformClasspath; + private final Collection myClasspath; + private final Collection myUpgradeModulePath; + private final Collection myModulePath; + private final Collection mySourcePath; + + public CompilationPaths(Collection platformClasspath, Collection classpath, Collection upgradeModulePath, Collection modulePath, Collection sourcePath) { + myPlatformClasspath = constCollection(platformClasspath); + myClasspath = constCollection(classpath); + myUpgradeModulePath = constCollection(upgradeModulePath); + myModulePath = constCollection(modulePath); + mySourcePath = constCollection(sourcePath); + } + + private static Collection constCollection(Collection col) { + return col == null || col.isEmpty()? Collections.emptyList() : Collections.unmodifiableCollection(col); + } + + @NotNull + public Collection getPlatformClasspath() { + return myPlatformClasspath; + } + + @NotNull + public Collection getClasspath() { + return myClasspath; + } + + @NotNull + public Collection getUpgradeModulePath() { + return myUpgradeModulePath; + } + + @NotNull + public Collection getModulePath() { + return myModulePath; + } + + @NotNull + public Collection getSourcePath() { + return mySourcePath; + } + + public interface Builder { + CompilationPaths create(); + + Builder setPlatformClasspath(Collection path); + Builder setClasspath(Collection path); + Builder setUpgradeModulePath(Collection path); + Builder setModulePath(Collection path); + Builder setSourcePath(Collection path); + } + + public static CompilationPaths create(@Nullable Collection platformCp, + @Nullable Collection cp, + @Nullable Collection upgradeModCp, + @Nullable Collection modulePath, + @Nullable Collection sourcePath) { + return new CompilationPaths(platformCp, cp, upgradeModCp, modulePath, sourcePath); + } + + public static Builder builder() { + return new Builder() { + private Collection mySourcePath; + private Collection myModulePath; + private Collection myUpgradeModulePath; + private Collection myClasspath; + private Collection myPlatformCp; + + @Override + public CompilationPaths create() { + return CompilationPaths.create(myPlatformCp, myClasspath, myUpgradeModulePath, myModulePath, mySourcePath); + } + + @Override + public Builder setPlatformClasspath(Collection path) { + myPlatformCp = path; + return this; + } + + @Override + public Builder setClasspath(Collection path) { + myClasspath = path; + return this; + } + + @Override + public Builder setUpgradeModulePath(Collection path) { + myUpgradeModulePath = path; + return this; + } + + @Override + public Builder setModulePath(Collection path) { + myModulePath = path; + return this; + } + + @Override + public Builder setSourcePath(Collection path) { + mySourcePath = path; + return this; + } + }; + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/javac/ExternalJavacManager.java b/jps/jps-builders/src/org/jetbrains/jps/javac/ExternalJavacManager.java index 272875ce8fb4..2b6d9ba06ced 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/javac/ExternalJavacManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/javac/ExternalJavacManager.java @@ -9,7 +9,7 @@ import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.ConcurrencyUtil; import com.intellij.util.concurrency.Semaphore; -import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.io.BaseOutputReader; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.*; import io.netty.channel.group.ChannelGroup; @@ -31,19 +31,21 @@ import org.jetbrains.jps.builders.java.JavaCompilingTool; import org.jetbrains.jps.cmdline.ClasspathBootstrap; import org.jetbrains.jps.incremental.GlobalContextKey; -import javax.tools.*; +import javax.tools.Diagnostic; import java.io.File; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.*; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; /** * @author Eugene Zhuravlev */ -public class ExternalJavacManager { +public class ExternalJavacManager extends ProcessAdapter { private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.javac.ExternalJavacServer"); public static final GlobalContextKey KEY = GlobalContextKey.create("_external_javac_server_"); @@ -51,18 +53,37 @@ public class ExternalJavacManager { public static final String STDOUT_LINE_PREFIX = "JAVAC_PROCESS[STDOUT]"; public static final String STDERR_LINE_PREFIX = "JAVAC_PROCESS[STDERR]"; - private static final AttributeKey SESSION_DESCRIPTOR = AttributeKey.valueOf("ExternalJavacServer.JavacProcessDescriptor"); + private static final AttributeKey PROCESS_ID_KEY = AttributeKey.valueOf("ExternalJavacServer.ProcessId"); + private static final Key PROCESS_HASH = Key.create("ExternalJavacServer.SdkHomePath"); private final File myWorkingDir; private final ChannelRegistrar myChannelRegistrar; - private final Map myMessageHandlers = new HashMap<>(); private int myListenPort = DEFAULT_SERVER_PORT; - private final Set myRunningHandlers = ContainerUtil.newConcurrentSet(); - private final ThreadPoolExecutor myExecutor = ConcurrencyUtil.newSingleThreadExecutor("Javac server event loop pool"); + private final Map mySessions = Collections.synchronizedMap(new HashMap<>()); + private final Map myRunningProcesses = Collections.synchronizedMap(new HashMap<>()); + private final Map myConnections = Collections.synchronizedMap(new HashMap<>()); // processId->channel + private final Executor myExecutor; + private boolean myOwnExecutor; + private final long myKeepAliveTimeout; + /** + * @deprecated: use {@link #ExternalJavacManager(File, Executor)} instead with explicit executor parameter + */ + @Deprecated public ExternalJavacManager(@NotNull final File workingDir) { + this(workingDir, ConcurrencyUtil.newSingleThreadExecutor("Javac server event loop pool")); + myOwnExecutor = true; + } + + public ExternalJavacManager(@NotNull final File workingDir, @NotNull Executor executor) { + this(workingDir, executor, 5 * 60 * 1000L /* 5 minutes default*/); + } + + public ExternalJavacManager(@NotNull final File workingDir, @NotNull Executor executor, long keepAliveTimeout) { myWorkingDir = workingDir; myChannelRegistrar = new ChannelRegistrar(); + myExecutor = executor; + myKeepAliveTimeout = keepAliveTimeout; } public void start(int listenPort) { @@ -93,6 +114,10 @@ public class ExternalJavacManager { } } + /** + * @deprecated Use {@link #forkJavac(String, int, List, List, CompilationPaths, Collection, Map, DiagnosticOutputConsumer, OutputFileConsumer, JavaCompilingTool, CanceledStatus, boolean)} instead + */ + @Deprecated public boolean forkJavac(String javaHome, int heapSize, List vmOptions, @@ -108,70 +133,54 @@ public class ExternalJavacManager { OutputFileConsumer outputSink, JavaCompilingTool compilingTool, CanceledStatus cancelStatus) { - final ExternalJavacMessageHandler rh = new ExternalJavacMessageHandler(diagnosticSink, outputSink, getEncodingName(options)); - final JavacRemoteProto.Message.Request request = JavacProtoUtil.createCompilationRequest( - options, files, classpath, platformCp, modulePath, upgradeModulePath, sourcePath, outs); - final UUID uuid = UUID.randomUUID(); - final JavacProcessDescriptor processDescriptor = new JavacProcessDescriptor(uuid, rh, request); - synchronized (myMessageHandlers) { - myMessageHandlers.put(uuid, processDescriptor); - } + return forkJavac( + javaHome, heapSize, vmOptions, options, + CompilationPaths.create(platformCp, classpath, upgradeModulePath, modulePath, sourcePath), + files, outs, diagnosticSink, outputSink, compilingTool, cancelStatus, false + ).get(); + } + + + public ExternalJavacRunResult forkJavac(String javaHome, + int heapSize, + List vmOptions, + List options, + CompilationPaths paths, + Collection files, + Map> outs, + DiagnosticOutputConsumer diagnosticSink, + OutputFileConsumer outputSink, + JavaCompilingTool compilingTool, + CanceledStatus cancelStatus, final boolean keepProcessAlive) { try { - final ExternalJavacProcessHandler processHandler = launchExternalJavacProcess( - uuid, javaHome, heapSize, myListenPort, myWorkingDir, vmOptions, compilingTool + final ExternalJavacProcessHandler running = findRunningProcess(processHash(javaHome, vmOptions, compilingTool)); + final ExternalJavacProcessHandler processHandler = running != null && running.lock()? running : launchExternalJavacProcess( + javaHome, heapSize, myListenPort, myWorkingDir, vmOptions, compilingTool, running == null && keepProcessAlive ); - myRunningHandlers.add(processHandler); - processHandler.addProcessListener(new ProcessAdapter() { - @Override - public void processTerminated(@NotNull ProcessEvent event) { - myRunningHandlers.remove(processHandler); - } - @Override - public void onTextAvailable(@NotNull ProcessEvent event, @NotNull Key outputType) { - final String text = event.getText(); - if (!StringUtil.isEmptyOrSpaces(text)) { - String prefix = null; - if (outputType == ProcessOutputTypes.STDOUT) { - prefix = STDOUT_LINE_PREFIX; - } - else if (outputType == ProcessOutputTypes.STDERR) { - prefix = STDERR_LINE_PREFIX; - } - if (prefix != null) { - diagnosticSink.outputLineAvailable(prefix + ": " + text); - } - } - } - }); - processHandler.startNotify(); - - while (!processDescriptor.waitFor(300L)) { - if (processHandler.isProcessTerminated() && processDescriptor.channel == null && processHandler.getExitCode() != 0) { - // process terminated abnormally and no communication took place - processDescriptor.setDone(); - break; - } - if (cancelStatus.isCanceled()) { - processDescriptor.cancelBuild(); - } + final Channel channel = lookupChannel(processHandler.getProcessId()); + if (channel != null) { + final CompileSession session = new CompileSession( + processHandler.getProcessId(), new ExternalJavacMessageHandler(diagnosticSink, outputSink, getEncodingName(options)), cancelStatus + ); + mySessions.put(session.getId(), session); + channel.writeAndFlush(JavacProtoUtil.toMessage(session.getId(), JavacProtoUtil.createCompilationRequest( + options, files, paths.getClasspath(), paths.getPlatformClasspath(), paths.getModulePath(), paths.getUpgradeModulePath(), paths.getSourcePath(), outs + ))); + return session; } - return rh.isTerminatedSuccessfully(); } catch (Throwable e) { LOG.info(e); diagnosticSink.report(new PlainMessageDiagnostic(Diagnostic.Kind.ERROR, e.getMessage())); } - finally { - unregisterMessageHandler(uuid); - } - return false; + return ExternalJavacRunResult.FAILURE; } // returns true if all process handlers terminated @TestOnly public boolean waitForAllProcessHandlers(long time, @NotNull TimeUnit unit) { - for (ProcessHandler handler : myRunningHandlers) { + for (ProcessHandler handler : myRunningProcesses.values()) { if (!handler.waitFor(unit.toMillis(time))) { return false; } @@ -181,22 +190,47 @@ public class ExternalJavacManager { @TestOnly public boolean awaitNettyThreadPoolTermination(long time, @NotNull TimeUnit unit) { - try { - return myExecutor.awaitTermination(time, unit); + if (myOwnExecutor && myExecutor instanceof ExecutorService) { + try { + return ((ExecutorService)myExecutor).awaitTermination(time, unit); + } + catch (InterruptedException ignored) { + } } - catch (InterruptedException ignored) { - return true; + return true; + } + + private ExternalJavacProcessHandler findRunningProcess(int processHash) { + List idleProcesses = null; + try { + synchronized (myRunningProcesses) { + for (Map.Entry entry : myRunningProcesses.entrySet()) { + final ExternalJavacProcessHandler process = entry.getValue(); + final Integer hash = PROCESS_HASH.get(process); + if (hash != null && hash == processHash) { + return process; + } + if (process.getIdleTime() > myKeepAliveTimeout) { + if (idleProcesses == null) { + idleProcesses = new ArrayList<>(); + } + idleProcesses.add(process); + } + } + } + return null; + } + finally { + if (idleProcesses != null) { + for (ExternalJavacProcessHandler process : idleProcesses) { + shutdownProcess(process); + } + } } } - private void unregisterMessageHandler(UUID uuid) { - final JavacProcessDescriptor descriptor; - synchronized (myMessageHandlers) { - descriptor = myMessageHandlers.remove(uuid); - } - if (descriptor != null) { - descriptor.setDone(); - } + private static int processHash(String sdkHomePath, List vmOptions, JavaCompilingTool tool) { + return Objects.hash(sdkHomePath.replace(File.separatorChar, '/'), vmOptions, tool.getId()); } @Nullable @@ -206,17 +240,54 @@ public class ExternalJavacManager { } public void stop() { + synchronized (myConnections) { + for (Map.Entry entry : myConnections.entrySet()) { + entry.getValue().writeAndFlush(JavacProtoUtil.toMessage(entry.getKey(), JavacProtoUtil.createShutdownRequest())); + } + } myChannelRegistrar.close().awaitUninterruptibly(); - myExecutor.shutdown(); + if (myOwnExecutor && myExecutor instanceof ExecutorService) { + ((ExecutorService)myExecutor).shutdown(); + } } - private ExternalJavacProcessHandler launchExternalJavacProcess(UUID uuid, - String sdkHomePath, + public void shutdownIdleProcesses() { + List idleProcesses = null; + synchronized (myRunningProcesses) { + for (ExternalJavacProcessHandler process : myRunningProcesses.values()) { + final long idle = process.getIdleTime(); + if (idle > myKeepAliveTimeout) { + if (idleProcesses == null) { + idleProcesses = new ArrayList<>(); + } + idleProcesses.add(process); + } + } + } + if (idleProcesses != null) { + for (ExternalJavacProcessHandler process : idleProcesses) { + shutdownProcess(process); + } + } + } + + private boolean shutdownProcess(ExternalJavacProcessHandler process) { + final Channel conn = myConnections.get(process.getProcessId()); + if (conn != null && process.lock()) { + conn.writeAndFlush(JavacProtoUtil.toMessage(process.getProcessId(), JavacProtoUtil.createShutdownRequest())); + return true; + } + return false; + } + + private ExternalJavacProcessHandler launchExternalJavacProcess(String sdkHomePath, int heapSize, int port, File workingDir, List vmOptions, - JavaCompilingTool compilingTool) throws Exception { + JavaCompilingTool compilingTool, + final boolean keepProcessAlive) throws Exception { + final UUID processId = UUID.randomUUID(); final List cmdLine = new ArrayList<>(); appendParam(cmdLine, getVMExecutablePath(sdkHomePath)); @@ -262,24 +333,80 @@ public class ExternalJavacManager { appendParam(cmdLine, cp.stream().map(File::getPath).collect(Collectors.joining(File.pathSeparator))); appendParam(cmdLine, ExternalJavacProcess.class.getName()); - appendParam(cmdLine, uuid.toString()); + appendParam(cmdLine, processId.toString()); appendParam(cmdLine, "127.0.0.1"); appendParam(cmdLine, Integer.toString(port)); + appendParam(cmdLine, Boolean.toString(keepProcessAlive)); // keep in memory after build finished appendParam(cmdLine, FileUtil.toSystemIndependentName(workingDir.getPath())); if (LOG.isDebugEnabled()) { LOG.debug("starting external compiler: " + cmdLine); } - FileUtil.createDirectory(workingDir); - Process process = new ProcessBuilder(cmdLine).directory(workingDir).start(); - return createProcessHandler(process, StringUtil.join(cmdLine, " ")); + final int processHash = processHash(sdkHomePath, vmOptions, compilingTool); + final ExternalJavacProcessHandler processHandler = createProcessHandler(processId, new ProcessBuilder(cmdLine).directory(workingDir).start(), StringUtil.join(cmdLine, " "), keepProcessAlive); + PROCESS_HASH.set(processHandler, processHash); + processHandler.lock(); + myRunningProcesses.put(processId, processHandler); + processHandler.addProcessListener(this); + processHandler.startNotify(); + return processHandler; } - protected ExternalJavacProcessHandler createProcessHandler(@NotNull Process process, @NotNull String commandLine) { - return new ExternalJavacProcessHandler(process, commandLine); + @Override + public void processTerminated(@NotNull ProcessEvent event) { + final UUID processId = ((ExternalJavacProcessHandler)event.getProcessHandler()).getProcessId(); + myRunningProcesses.remove(processId); + synchronized (mySessions) { + for (Iterator> it = mySessions.entrySet().iterator(); it.hasNext(); ) { + final CompileSession session = it.next().getValue(); + if (processId.equals(session.getProcessId())) { + session.setDone(); + it.remove(); + } + } + } + } + + @Override + public void onTextAvailable(@NotNull ProcessEvent event, @NotNull Key outputType) { + final String text = event.getText(); + if (!StringUtil.isEmptyOrSpaces(text)) { + String prefix = null; + if (outputType == ProcessOutputTypes.STDOUT) { + prefix = STDOUT_LINE_PREFIX; + } + else if (outputType == ProcessOutputTypes.STDERR) { + prefix = STDERR_LINE_PREFIX; + } + if (prefix != null) { + List consumers = null; + final UUID processId = ((ExternalJavacProcessHandler)event.getProcessHandler()).getProcessId(); + synchronized (mySessions) { + for (CompileSession session : mySessions.values()) { + if (processId.equals(session.getProcessId())) { + if (consumers == null) { + consumers = new ArrayList<>(); + } + consumers.add(session.myHandler.getDiagnosticSink()); + } + } + } + if (consumers != null) { + final String msg = prefix + ": " + text; + for (DiagnosticOutputConsumer consumer : consumers) { + consumer.outputLineAvailable(msg); + } + } + } + } + } + + + protected ExternalJavacProcessHandler createProcessHandler(UUID processId, @NotNull Process process, @NotNull String commandLine, boolean keepProcessAlive) { + return new ExternalJavacProcessHandler(processId, process, commandLine, keepProcessAlive); } private static void appendParam(List cmdLine, String parameter) { @@ -306,57 +433,78 @@ public class ExternalJavacManager { } protected static class ExternalJavacProcessHandler extends BaseOSProcessHandler { - private volatile int myExitCode; + private long myIdleSince; + private final UUID myProcessId; + private final boolean myKeepProcessAlive; + private boolean myIsBusy; - protected ExternalJavacProcessHandler(@NotNull Process process, @NotNull String commandLine) { + protected ExternalJavacProcessHandler(UUID processId, @NotNull Process process, @NotNull String commandLine, boolean keepProcessAlive) { super(process, commandLine, null); - addProcessListener(new ProcessAdapter() { - @Override - public void processTerminated(@NotNull ProcessEvent event) { - myExitCode = event.getExitCode(); - } - }); + myProcessId = processId; + myKeepProcessAlive = keepProcessAlive; + } + + public UUID getProcessId() { + return myProcessId; + } + + public synchronized long getIdleTime() { + final long idleSince = myIdleSince; + return idleSince <= 0L? 0L : (System.currentTimeMillis() - idleSince); + } + + public synchronized void unlock() { + myIdleSince = System.currentTimeMillis(); + myIsBusy = false; + } + + public synchronized boolean lock() { + myIdleSince = 0L; + return !myIsBusy && (myIsBusy = true); + } + + public boolean isKeepProcessAlive() { + return myKeepProcessAlive; } - @Override @NotNull - public Integer getExitCode() { - return myExitCode; + @Override + protected BaseOutputReader.Options readerOptions() { + // if keepAlive requested, that means the process will be waiting considerable periods of time without any output + return myKeepProcessAlive? BaseOutputReader.Options.BLOCKING : super.readerOptions(); } } @ChannelHandler.Sharable private class CompilationRequestsHandler extends SimpleChannelInboundHandler { + + @Override + public void channelRegistered(ChannelHandlerContext ctx) throws Exception { + super.channelRegistered(ctx); + } + + @Override + public void channelActive(@NotNull ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + } + @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { - JavacProcessDescriptor descriptor = ctx.channel().attr(SESSION_DESCRIPTOR).getAndSet(null); - if (descriptor != null) { - descriptor.setDone(); + final Channel channel = ctx.channel(); + final UUID processId = channel.attr(PROCESS_ID_KEY).get(); + if (processId != null) { + myConnections.remove(processId); } super.channelUnregistered(ctx); } @Override public void channelRead0(final ChannelHandlerContext context, JavacRemoteProto.Message message) throws Exception { - JavacProcessDescriptor descriptor = context.channel().attr(SESSION_DESCRIPTOR).get(); - - UUID sessionId; - if (descriptor == null) { - // this is the first message for this session, so fill session data with missing info - sessionId = JavacProtoUtil.fromProtoUUID(message.getSessionId()); - - descriptor = myMessageHandlers.get(sessionId); - if (descriptor != null) { - descriptor.channel = context.channel(); - context.channel().attr(SESSION_DESCRIPTOR).set(descriptor); - } - } - else { - sessionId = descriptor.sessionId; - } - - final ExternalJavacMessageHandler handler = descriptor != null? descriptor.handler : null; + // in case of REQUEST_ACK this is a process ID, otherwise this is a sessionId + final UUID msgUuid = JavacProtoUtil.fromProtoUUID(message.getSessionId()); + CompileSession session = mySessions.get(msgUuid); + final ExternalJavacMessageHandler handler = session != null? session.myHandler : null; final JavacRemoteProto.Message.Type messageType = message.getMessageType(); JavacRemoteProto.Message reply = null; @@ -364,27 +512,38 @@ public class ExternalJavacManager { if (messageType == JavacRemoteProto.Message.Type.RESPONSE) { final JavacRemoteProto.Message.Response response = message.getResponse(); final JavacRemoteProto.Message.Response.Type responseType = response.getResponseType(); - if (handler != null) { - if (responseType == JavacRemoteProto.Message.Response.Type.REQUEST_ACK) { - final JavacRemoteProto.Message.Request request = descriptor.request; - if (request != null) { - reply = JavacProtoUtil.toMessage(sessionId, request); - descriptor.request = null; + if (responseType == JavacRemoteProto.Message.Response.Type.REQUEST_ACK) { + // in this case msgUuid is a process ID, so we need to save the channel, associated with the process + final Channel channel = context.channel(); + channel.attr(PROCESS_ID_KEY).set(msgUuid); + synchronized (myConnections) { + myConnections.put(msgUuid, channel); + myConnections.notifyAll(); + } + } + else if (handler != null) { + final boolean terminateOk = handler.handleMessage(message); + if (terminateOk) { + session.setDone(); + mySessions.remove(session.getId()); + final ExternalJavacProcessHandler process = myRunningProcesses.get(session.getProcessId()); + if (process != null) { + process.unlock(); + // todo: submit running process GC task? + //if (process.isKeepProcessAlive()) { + //} } } - else { - final boolean terminateOk = handler.handleMessage(message); - if (terminateOk) { - descriptor.setDone(); - } + else if (session.isCancelRequested()) { + reply = JavacProtoUtil.toMessage(msgUuid, JavacProtoUtil.createCancelRequest()); } } else { - reply = JavacProtoUtil.toMessage(sessionId, JavacProtoUtil.createCancelRequest()); + reply = JavacProtoUtil.toMessage(msgUuid, JavacProtoUtil.createCancelRequest()); } } else { - reply = JavacProtoUtil.toMessage(sessionId, JavacProtoUtil.createFailure("Unsupported message: " + messageType.name(), null)); + reply = JavacProtoUtil.toMessage(msgUuid, JavacProtoUtil.createFailure("Unsupported message: " + messageType.name(), null)); } } finally { @@ -395,6 +554,25 @@ public class ExternalJavacManager { } } + private Channel lookupChannel(UUID processId) { + Channel channel = null; + synchronized (myConnections) { + channel = myConnections.get(processId); + while (channel == null) { + if (!myRunningProcesses.containsKey(processId)) { + break; // the process is already gone + } + try { + myConnections.wait(300L); + } + catch (InterruptedException ignored) { + } + channel = myConnections.get(processId); + } + } + return channel; + } + @ChannelHandler.Sharable private static final class ChannelRegistrar extends ChannelInboundHandlerAdapter { private final ChannelGroup openChannels = new DefaultChannelGroup(ImmediateEventExecutor.INSTANCE); @@ -434,34 +612,93 @@ public class ExternalJavacManager { } } - private static class JavacProcessDescriptor { - private final UUID sessionId; - private final ExternalJavacMessageHandler handler; - private volatile JavacRemoteProto.Message.Request request; - private volatile Channel channel; + private class CompileSession extends ExternalJavacRunResult{ + private final UUID myId; + private final UUID myProcessId; + private final CanceledStatus myCancelStatus; + private final ExternalJavacMessageHandler myHandler; private final Semaphore myDone = new Semaphore(); - JavacProcessDescriptor(@NotNull UUID sessionId, - @NotNull ExternalJavacMessageHandler handler, - @NotNull JavacRemoteProto.Message.Request request) { - this.sessionId = sessionId; - this.handler = handler; - this.request = request; + CompileSession(@NotNull UUID processId, @NotNull ExternalJavacMessageHandler handler, CanceledStatus cancelStatus) { + myProcessId = processId; + myCancelStatus = cancelStatus; + myId = UUID.randomUUID(); + myHandler = handler; myDone.down(); } + + @NotNull + public UUID getId() { + return myId; + } - public void cancelBuild() { - if (channel != null) { - channel.writeAndFlush(JavacProtoUtil.toMessage(sessionId, JavacProtoUtil.createCancelRequest())); - } + @NotNull + public UUID getProcessId() { + return myProcessId; + } + + @Override + public boolean isDone() { + return myDone.isUp(); } public void setDone() { myDone.up(); } - public boolean waitFor(long timeout) { - return myDone.waitFor(timeout); + public boolean isTerminatedSuccessfully() { + return myHandler.isTerminatedSuccessfully(); } + + boolean isCancelRequested() { + return myCancelStatus.isCanceled(); + } + + @NotNull + @Override + public Boolean get() { + while (true) { + try { + if (myDone.waitForUnsafe(300L)) { + break; + } + } + catch (InterruptedException ignored) { + } + if (checkStopConditions()) { + break; + } + } + return isTerminatedSuccessfully(); + } + + @NotNull + @Override + public Boolean get(long timeout, @NotNull TimeUnit unit) throws InterruptedException, TimeoutException { + if (!myDone.waitForUnsafe(unit.toMillis(timeout))) { + if (!checkStopConditions()) { + // if execution continues, just notify about timeout + throw new TimeoutException(); + } + } + return isTerminatedSuccessfully(); + } + + private boolean checkStopConditions() { + if (!myRunningProcesses.containsKey(myProcessId)) { + // process terminated + setDone(); + mySessions.remove(myId); + return true; + } + if (isCancelRequested()) { + final Channel channel = myConnections.get(myProcessId); + if (channel != null) { + channel.writeAndFlush(JavacProtoUtil.toMessage(myId, JavacProtoUtil.createCancelRequest())); + } + } + return false; + } + } } \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/javac/ExternalJavacRunResult.java b/jps/jps-builders/src/org/jetbrains/jps/javac/ExternalJavacRunResult.java new file mode 100644 index 000000000000..76d20d0c4fc3 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/javac/ExternalJavacRunResult.java @@ -0,0 +1,47 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package org.jetbrains.jps.javac; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public abstract class ExternalJavacRunResult implements Future { + public static final ExternalJavacRunResult FAILURE = new ExternalJavacRunResult() { + @Override + public boolean isDone() { + return true; + } + + @NotNull + @Override + public Boolean get() { + return Boolean.FALSE; + } + + @NotNull + @Override + public Boolean get(long timeout, @NotNull TimeUnit unit){ + return Boolean.FALSE; + } + }; + + @Override + public final boolean cancel(boolean mayInterruptIfRunning) { + return false; // not supported + } + + @Override + public final boolean isCancelled() { + return false; // not supported, as cancel is handled via CancelStatus + } + + @Override + @NotNull + public abstract Boolean get(); + + @Override + @NotNull + public abstract Boolean get(long timeout, @NotNull TimeUnit unit) throws InterruptedException, TimeoutException; +} diff --git a/platform/util/resources/misc/registry.properties b/platform/util/resources/misc/registry.properties index 51e48a2d50c3..33925c837cef 100644 --- a/platform/util/resources/misc/registry.properties +++ b/platform/util/resources/misc/registry.properties @@ -492,6 +492,9 @@ compiler.automake.allow.parallel=true compiler.automake.allow.parallel.description=The option allows to force automatically started builds to run in single-threaded mode even if 'Compile independent modules in parallel' is on\n\ This might help to save CPU resources for the foreground processes. +compiler.external.javac.keep.alive.timeout=300000 +compiler.external.javac.keep.alive.timeout.description=If not used for the specified period of time or longer, the cached javac compilation process will be shut down by the IDE + vcs.annotations.preload=false vcs.showConsole=true vcs.log.bek.sort.disabled=false