MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

support simultaneous modulePath & classpath compilation for JPMS projects (IDEA-223774)

Browse Source 
GitOrigin-RevId: 5bb3b635b6d31d941633ffbdf28fc17e6d9f1b12
This commit is contained in:
Eugene Zhuravlev
2019-10-01 14:32:52 +00:00
committed by intellij-monorepo-bot
parent 01d32acced
commit c4428f6ee8
16 changed files with 289 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/libraries/qdox_java_parser.xml
Generated
+12
View File
@@ -0,0 +1,12 @@
<component name="libraryTable">
<library name="qdox-java-parser" type="repository">
<properties maven-id="com.thoughtworks.qdox:qdox:2.0-M10" />
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/thoughtworks/qdox/qdox/2.0-M10/qdox-2.0-M10.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/thoughtworks/qdox/qdox/2.0-M10/qdox-2.0-M10-sources.jar!/" />
</SOURCES>
</library>
</component>
build/groovy/org/jetbrains/intellij/build/CommunityStandaloneJpsBuilder.groovy
+1 -1
View File
@@ -75,7 +75,7 @@ class CommunityStandaloneJpsBuilder {
[
"JDOM", "jna", "OroMatcher", "Trove4j", "ASM", "NanoXML", "protobuf", "cli-parser", "Log4J", "jgoodies-forms", "Eclipse",
"netty-codec-http", "lz4-java", "commons-codec", "commons-logging", "http-client", "Slf4j", "Guava", "plexus-utils",
"jetbrains-annotations-java5", "maven-model"
"jetbrains-annotations-java5", "maven-model", "qdox-java-parser"
].each {
projectLibrary(it)
}
java/java-tests/testData/compileServer/incremental/java9-features/splitModulePath/lib/module_lib_1.jar
BIN
View File
Binary file not shown.
java/java-tests/testData/compileServer/incremental/java9-features/splitModulePath/lib/module_lib_2.jar
BIN
View File
Binary file not shown.
java/java-tests/testData/compileServer/incremental/java9-features/splitModulePath/lib/module_lib_3.jar
BIN
View File
Binary file not shown.
java/java-tests/testData/compileServer/incremental/java9-features/splitModulePath/lib/module_lib_util.jar
BIN
View File
Binary file not shown.
java/java-tests/testData/compileServer/incremental/java9-features/splitModulePath/moduleA/src/module-info.java
+5
View File
@@ -0,0 +1,5 @@
module A {
requires module_lib1;
requires mod.lib2;
}
java/java-tests/testData/compileServer/incremental/java9-features/splitModulePath/moduleA/src/ppp/Main.java
+7
View File
@@ -0,0 +1,7 @@
package ppp;
public class Main {
public static void main(String[] args) {
System.out.println("Hello");
}
}
jps/jps-builders-6/src/org/jetbrains/jps/javac/JavacMain.java
+18 -17
View File
@@ -97,23 +97,6 @@ public class JavacMain {
return false;
}
if (!classpath.isEmpty()) {
try {
fileManager.setLocation(StandardLocation.CLASS_PATH, classpath);
if (!usingJavac &&
isAnnotationProcessingEnabled(_options) &&
!_options.contains("-processorpath") &&
(javacBefore9 || !_options.contains("--processor-module-path"))) {
// for non-javac file manager ensure annotation processor path defaults to classpath
fileManager.setLocation(StandardLocation.ANNOTATION_PROCESSOR_PATH, classpath);
}
}
catch (IOException e) {
fileManager.getContext().reportMessage(Diagnostic.Kind.ERROR, e.getMessage());
return false;
}
}
if (!platformClasspath.isEmpty()) {
try {
fileManager.handleOption("-bootclasspath", Collections.singleton("").iterator()); // this will clear cached stuff
@@ -156,6 +139,24 @@ public class JavacMain {
}
}
if (!classpath.isEmpty()) {
// because module path has priority if present, initialize classpath after the module path
try {
fileManager.setLocation(StandardLocation.CLASS_PATH, classpath);
if (!usingJavac &&
isAnnotationProcessingEnabled(_options) &&
!_options.contains("-processorpath") &&
(javacBefore9 || (!_options.contains("--processor-module-path") && getLocation(fileManager, "ANNOTATION_PROCESSOR_MODULE_PATH") == null))) {
// for non-javac file manager ensure annotation processor path defaults to classpath
fileManager.setLocation(StandardLocation.ANNOTATION_PROCESSOR_PATH, classpath);
}
}
catch (IOException e) {
fileManager.getContext().reportMessage(Diagnostic.Kind.ERROR, e.getMessage());
return false;
}
}
if (javacBefore9 || !sourcePath.isEmpty() || modulePath.isEmpty()) {
try {
// ensure the source path is set;
jps/jps-builders/intellij.platform.jps.build.iml
+1
View File
@@ -34,5 +34,6 @@
<orderEntry type="library" name="Guava" level="project" />
<orderEntry type="library" name="netty-codec-http" level="project" />
<orderEntry type="library" name="javax.annotation-api" level="project" />
<orderEntry type="library" name="qdox-java-parser" level="project" />
</component>
</module>
jps/jps-builders/src/org/jetbrains/jps/cmdline/ClasspathBootstrap.java
+2
View File
@@ -11,6 +11,7 @@ import com.intellij.uiDesigner.core.GridConstraints;
import com.intellij.util.SystemProperties;
import com.intellij.util.containers.ContainerUtil;
import com.jgoodies.forms.layout.CellConstraints;
import com.thoughtworks.qdox.JavaProjectBuilder;
import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.EventLoopGroup;
import io.netty.handler.codec.protobuf.ProtobufDecoder;
@@ -72,6 +73,7 @@ public class ClasspathBootstrap {
cp.add(getResourcePath(CellConstraints.class)); // jGoodies-forms
cp.addAll(getInstrumentationUtilRoots());
cp.add(getResourcePath(IXMLBuilder.class)); // nano-xml
cp.add(getResourcePath(JavaProjectBuilder.class)); // qdox lightweight java parser
cp.addAll(ContainerUtil.map(ArtifactRepositoryManager.getClassesFromDependencies(), ClasspathBootstrap::getResourcePath));
jps/jps-builders/src/org/jetbrains/jps/incremental/java/JavaBuilder.java
+38 -16
View File
@@ -59,8 +59,7 @@ import org.jetbrains.jps.model.serialization.PathMacroUtil;
import org.jetbrains.jps.service.JpsServiceManager;
import org.jetbrains.jps.service.SharedThreadPool;
import javax.tools.Diagnostic;
import javax.tools.JavaFileObject;
import javax.tools.*;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
@@ -82,6 +81,8 @@ public class JavaBuilder extends ModuleLevelBuilder {
private static final Logger LOG = Logger.getInstance("#org.jetbrains.jps.incremental.java.JavaBuilder");
private static final String JAVA_EXTENSION = "java";
private static final String USE_MODULE_PATH_ONLY_OPTION = "compiler.force.module.path";
public static final String BUILDER_NAME = "java";
public static final Key<Boolean> IS_ENABLED = Key.create("_java_compiler_enabled_");
public static final FileFilter JAVA_SOURCES_FILTER = FileFilters.withExtension(JAVA_EXTENSION);
@@ -90,6 +91,7 @@ public class JavaBuilder extends ModuleLevelBuilder {
private static final Key<Boolean> PREFER_TARGET_JDK_COMPILER = GlobalContextKey.create("_prefer_target_jdk_javac_");
private static final Key<JavaCompilingTool> COMPILING_TOOL = Key.create("_java_compiling_tool_");
private static final Key<ConcurrentMap<String, Collection<String>>> COMPILER_USAGE_STATISTICS = Key.create("_java_compiler_usage_stats_");
private static final Key<ModulePathSplitter> MODULE_PATH_SPLITTER = GlobalContextKey.create("_module_path_splitter_");
private static final List<String> COMPILABLE_EXTENSIONS = Collections.singletonList(JAVA_EXTENSION);
private static final Set<String> FILTERED_OPTIONS = ContainerUtil.newHashSet(
@@ -145,6 +147,7 @@ public class JavaBuilder extends ModuleLevelBuilder {
if (LOG.isDebugEnabled()) {
LOG.debug("Java compiler ID: " + compilerId);
}
MODULE_PATH_SPLITTER.set(context, new ModulePathSplitter());
JavaCompilingTool compilingTool = JavaBuilderUtil.findCompilingTool(compilerId);
COMPILING_TOOL.set(context, compilingTool);
COMPILER_USAGE_STATISTICS.set(context, new ConcurrentHashMap<>());
@@ -230,12 +233,17 @@ public class JavaBuilder extends ModuleLevelBuilder {
return true;
});
File moduleInfoFile = null;
int javaModulesCount = 0;
if ((!filesToCompile.isEmpty() || dirtyFilesHolder.hasRemovedFiles()) &&
getTargetPlatformLanguageVersion(chunk.representativeTarget().getModule()) >= 9) {
for (ModuleBuildTarget target : chunk.getTargets()) {
if (JavaBuilderUtil.findModuleInfoFile(context, target) != null) {
final File moduleInfo = JavaBuilderUtil.findModuleInfoFile(context, target);
if (moduleInfo != null) {
javaModulesCount++;
if (moduleInfoFile == null) {
moduleInfoFile = moduleInfo;
}
}
}
}
@@ -254,7 +262,7 @@ public class JavaBuilder extends ModuleLevelBuilder {
return ExitCode.ABORT;
}
return compile(context, chunk, dirtyFilesHolder, filesToCompile, outputConsumer, compilingTool, javaModulesCount > 0);
return compile(context, chunk, dirtyFilesHolder, filesToCompile, outputConsumer, compilingTool, moduleInfoFile);
}
catch (BuildDataCorruptedException | PersistentEnumeratorBase.CorruptedException | ProjectBuildException e) {
throw e;
@@ -276,7 +284,7 @@ public class JavaBuilder extends ModuleLevelBuilder {
Collection<? extends File> files,
OutputConsumer outputConsumer,
JavaCompilingTool compilingTool,
boolean hasModules) throws Exception {
File moduleInfoFile) throws Exception {
ExitCode exitCode = ExitCode.NOTHING_DONE;
final boolean hasSourcesToCompile = !files.isEmpty();
@@ -329,7 +337,7 @@ public class JavaBuilder extends ModuleLevelBuilder {
}
}
try {
compiledOk = compileJava(context, chunk, files, classpath, platformCp, srcPath, diagnosticSink, outputSink, compilingTool, hasModules);
compiledOk = compileJava(context, chunk, files, classpath, platformCp, srcPath, diagnosticSink, outputSink, compilingTool, moduleInfoFile);
}
finally {
filesWithErrors = diagnosticSink.getFilesWithErrors();
@@ -373,17 +381,19 @@ public class JavaBuilder extends ModuleLevelBuilder {
DiagnosticOutputConsumer diagnosticSink,
OutputFileConsumer outputSink,
JavaCompilingTool compilingTool,
boolean hasModules) {
File moduleInfoFile) {
final Semaphore counter = new Semaphore();
COUNTER_KEY.set(context, counter);
final Set<JpsModule> modules = chunk.getModules();
ProcessorConfigProfile profile = null;
final JpsJavaCompilerConfiguration compilerConfig = JpsJavaExtensionService.getInstance().getCompilerConfiguration(
context.getProjectDescriptor().getProject()
);
assert compilerConfig != null;
if (modules.size() == 1) {
final JpsJavaCompilerConfiguration compilerConfig =
JpsJavaExtensionService.getInstance().getCompilerConfiguration(context.getProjectDescriptor().getProject());
assert compilerConfig != null;
profile = compilerConfig.getAnnotationProcessingProfile(modules.iterator().next());
}
else {
@@ -436,12 +446,24 @@ public class JavaBuilder extends ModuleLevelBuilder {
Collection<File> modulePath = Collections.emptyList();
Collection<? extends File> upgradeModulePath = Collections.emptyList();
if (hasModules) {
// in Java 9, named modules are not allowed to read classes from the classpath
// moreover, the compiler requires all transitive dependencies to be on the module path
modulePath = ProjectPaths.getCompilationModulePath(chunk, false);
classPath = Collections.emptyList();
// modules located above the JDK make a module upgrade path
if (moduleInfoFile != null) { // has modules
final boolean useModulePathOnly = Boolean.parseBoolean(System.getProperty(USE_MODULE_PATH_ONLY_OPTION))/*compilerConfig.useModulePathOnly()*/;
if (useModulePathOnly) {
// in Java 9, named modules are not allowed to read classes from the classpath
// moreover, the compiler requires all transitive dependencies to be on the module path
modulePath = ProjectPaths.getCompilationModulePath(chunk, false);
classPath = Collections.emptyList();
}
else {
// placing only explicitly referenced modules into the module path and the rest of deps to classpath
final ModulePathSplitter splitter = MODULE_PATH_SPLITTER.get(context);
final Pair<Collection<File>, Collection<File>> pair = splitter.splitPath(
moduleInfoFile, outs.keySet(), ProjectPaths.getCompilationModulePath(chunk, false)
);
modulePath = pair.first;
classPath = pair.second;
}
// modules above the JDK in the order entry list make a module upgrade path
upgradeModulePath = platformCp;
platformCp = Collections.emptyList();
}
jps/jps-builders/src/org/jetbrains/jps/incremental/java/ModulePathSplitter.java
+147
View File
@@ -0,0 +1,147 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.jps.incremental.java;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.thoughtworks.qdox.JavaProjectBuilder;
import com.thoughtworks.qdox.library.OrderedClassLibraryBuilder;
import com.thoughtworks.qdox.model.JavaModule;
import com.thoughtworks.qdox.model.JavaModuleDescriptor;
import gnu.trove.THashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.javac.JpsJavacFileManager;
import java.io.File;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.*;
/**
* @author Eugene Zhuravlev
* Date: 26-Sep-19
*/
public class ModulePathSplitter {
private final Map<File, ModuleInfo> myCache = Collections.synchronizedMap(new THashMap<>(FileUtil.FILE_HASHING_STRATEGY));
private static final Method myModuleFinderCreateMethod;
private static final Method myFindAll;
private static final Method myGetDescriptor;
private static final Method myDescriptorName;
private static final Method myRequiresName;
private static final Method myDescriptorRequires;
static {
Method of = null;
Method findAll = null;
Method getDescriptor = null;
Method descriptorName = null;
Method requiresName = null;
Method descriptorRequires = null;
try {
final Class<?> finderClass = Class.forName("java.lang.module.ModuleFinder");
final Class<?> descriptorClass = Class.forName("java.lang.module.ModuleDescriptor");
final Class<?> referenceClass = Class.forName("java.lang.module.ModuleReference");
final Class<?> requireClass = Class.forName("java.lang.module.ModuleDescriptor$Requires");
of = finderClass.getDeclaredMethod("of", Path[].class);
findAll = finderClass.getDeclaredMethod("findAll");
getDescriptor = referenceClass.getDeclaredMethod("descriptor");
descriptorName = descriptorClass.getDeclaredMethod("name");
descriptorRequires = descriptorClass.getDeclaredMethod("requires");
requiresName = requireClass.getDeclaredMethod("name");
}
catch (Throwable ignored) {
}
myModuleFinderCreateMethod = of;
myFindAll = findAll;
myGetDescriptor = getDescriptor;
myDescriptorName = descriptorName;
myRequiresName = requiresName;
myDescriptorRequires = descriptorRequires;
}
public ModulePathSplitter() {
}
public Pair<Collection<File>, Collection<File>> splitPath(File chunkModuleInfo, Set<File> chunkOutputs, Collection<File> path) {
if (myModuleFinderCreateMethod == null) {
// the module API is not available
return Pair.create(path, Collections.emptyList());
}
final List<File> modulePath = new ArrayList<>();
final List<File> classpath = new ArrayList<>();
final Set<String> allRequired = collectRequired(chunkModuleInfo, JpsJavacFileManager.filter(path, file -> !chunkOutputs.contains(file)));
for (File file : path) {
(chunkOutputs.contains(file) || allRequired.contains(getModuleInfo(file).name) ? modulePath : classpath).add(file);
}
return Pair.create(Collections.unmodifiableList(modulePath), Collections.unmodifiableList(classpath));
}
private Set<String> collectRequired(File chunkModuleInfo, Iterable<? extends File> path) {
final Set<String> result = new HashSet<>();
// first, add all requires from chunk module-info
final JavaModuleDescriptor chunkDescr = new JavaProjectBuilder(new OrderedClassLibraryBuilder()).addSourceFolder(chunkModuleInfo.getParentFile()).getDescriptor();
for (JavaModuleDescriptor.JavaRequires require : chunkDescr.getRequires()) {
final JavaModule rm = require.getModule();
if (rm != null) {
result.add(rm.getName());
}
}
for (File file : path) {
result.addAll(getModuleInfo(file).requires);
}
return result;
}
@NotNull
private ModuleInfo getModuleInfo(File f) {
ModuleInfo info = myCache.get(f);
if (info != null) {
return info;
}
info = ModuleInfo.EMPTY;
try {
Object mf = myModuleFinderCreateMethod.invoke(null, (Object)new Path[]{f.toPath()}); // ModuleFinder.of(f.toPath());
for (Object moduleRef : (Set)myFindAll.invoke(mf)) { // mf.findAll()
final Object descriptor = myGetDescriptor.invoke(moduleRef); // moduleRef.descriptor()
final String moduleName = (String)myDescriptorName.invoke(descriptor); // descriptor.name();
final Set requires = (Set)myDescriptorRequires.invoke(descriptor); //descriptor.requires();
if (requires.isEmpty()) {
info = new ModuleInfo(moduleName);
}
else {
final Set<String> req = new HashSet<>();
for (Object require : requires) {
req.add((String)myRequiresName.invoke(require)/*require.name()*/);
}
info = new ModuleInfo(moduleName, req);
}
break;
}
}
catch (Throwable ignored) {
}
myCache.put(f, info);
return info;
}
private static final class ModuleInfo {
static final ModuleInfo EMPTY = new ModuleInfo(null, Collections.emptyList());
@Nullable
final String name;
@NotNull
final Collection<String> requires;
ModuleInfo(String name) {
this(name, Collections.emptyList());
}
ModuleInfo(@Nullable String name, @NotNull Collection<String> requires) {
this.name = name;
this.requires = Collections.unmodifiableCollection(requires);
}
}
}
jps/jps-builders/testSrc/org/jetbrains/ether/IncrementalTestCase.java
+6 -2
View File
@@ -219,8 +219,12 @@ public abstract class IncrementalTestCase extends JpsBuildTestCase {
}
protected JpsLibrary addLibrary(final String jarPath) {
JpsLibrary library = myProject.addLibrary("l", JpsJavaLibraryType.INSTANCE);
library.addRoot(new File(getAbsolutePath(jarPath)), JpsOrderRootType.COMPILED);
return addLibrary("l", new File(getAbsolutePath(jarPath)));
}
protected JpsLibrary addLibrary(final String libraryName, final File jarFile) {
JpsLibrary library = myProject.addLibrary(libraryName, JpsJavaLibraryType.INSTANCE);
library.addRoot(jarFile, JpsOrderRootType.COMPILED);
return library;
}
jps/jps-builders/testSrc/org/jetbrains/ether/Java9Test.java
+49
View File
@@ -1,9 +1,24 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.ether;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.SystemInfo;
import org.jetbrains.jps.ModuleChunk;
import org.jetbrains.jps.ProjectPaths;
import org.jetbrains.jps.builders.BuildResult;
import org.jetbrains.jps.builders.CompileScopeTestBuilder;
import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType;
import org.jetbrains.jps.incremental.ModuleBuildTarget;
import org.jetbrains.jps.incremental.java.ModulePathSplitter;
import org.jetbrains.jps.model.JpsModuleRootModificationUtil;
import org.jetbrains.jps.model.java.JpsJavaDependencyScope;
import org.jetbrains.jps.model.library.JpsLibrary;
import org.jetbrains.jps.model.module.JpsModule;
import java.io.File;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
/**
* @author Eugene Zhuravlev
@@ -101,4 +116,38 @@ public class Java9Test extends IncrementalTestCase {
modify(1);
doBuild(CompileScopeTestBuilder.make().allModules()).assertFailed();
}
public void testSplitModulePath() {
setupInitialProject();
final Map<String, JpsModule> modules = setupModules();
assertEquals(1, modules.size());
final JpsModule module = modules.values().iterator().next();
final ModuleBuildTarget target = new ModuleBuildTarget(module, JavaModuleBuildTargetType.PRODUCTION);
final File outputDir = target.getOutputDir();
final File libDir = new File(getAbsolutePath("lib"));
for (File jarFile : libDir.listFiles((dir, name) -> name.endsWith(".jar"))) {
JpsLibrary lib = addLibrary(jarFile.getName(), jarFile);
JpsModuleRootModificationUtil.addDependency(module, lib, JpsJavaDependencyScope.COMPILE, false);
}
final File moduleInfoPath = new File(getAbsolutePath("moduleA/src/module-info.java"));
final ModulePathSplitter splitter = new ModulePathSplitter();
final Collection<File> dependencies = ProjectPaths.getCompilationModulePath(new ModuleChunk(Collections.singleton(target)), false);
final Pair<Collection<File>, Collection<File>> split = splitter.splitPath(moduleInfoPath, Collections.singleton(outputDir), dependencies);
final Collection<File> modulePath = split.first;
final Collection<File> classpath = split.second;
assertEquals(4, modulePath.size());
assertTrue(modulePath.contains(outputDir));
assertTrue(modulePath.contains(new File(libDir, "module_lib_1.jar")));
assertTrue(modulePath.contains(new File(libDir, "module_lib_2.jar")));
assertTrue(modulePath.contains(new File(libDir, "module_lib_util.jar")));
assertEquals(1, classpath.size());
assertTrue(classpath.contains(new File(libDir, "module_lib_3.jar")));
}
}
platform/build-scripts/groovy/org/jetbrains/intellij/build/CommunityLibraryLicenses.groovy
+3
View File
@@ -593,6 +593,9 @@ class CommunityLibraryLicenses {
new LibraryLicense(name: "XZ for Java", libraryName: "xz", version: "1.5", license: "Public Domain",
url: "http://tukaani.org/xz/java.html"),
new LibraryLicense(name: "qdox-java-parser", libraryName: "qdox-java-parser", version: "2.0", license: "Apache 2.0",
url: "https://github.com/paul-hammant/qdox", licenseUrl: "http://www.apache.org/licenses/LICENSE-2.0"),
jetbrainsLibrary("completion-ranking-java"),
jetbrainsLibrary("completion-ranking-kotlin"),