MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

jps-bootstrap: convert jps-bootstrap to Kotlin

Browse Source 
GitOrigin-RevId: 3c7549a2452bffe1d95f5f3a8961a49cb6bbdb7a
This commit is contained in:
Leonid Shalupov
2023-01-23 13:33:56 +00:00
committed by intellij-monorepo-bot
parent 088c21cac3
commit 12f2a727a3
7 changed files with 651 additions and 813 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/jps-bootstrap/jps-bootstrap-classpath.xml
+2 -17
View File
@@ -71,9 +71,6 @@
</resolver:resolve>
<delete dir="${classes.dir}" />
<mkdir dir="${classes.dir}/java" />
<mkdir dir="${classes.dir}/kotlin" />
<java
classname="org.jetbrains.kotlin.cli.jvm.K2JVMCompiler"
classpathref="classpath.kotlin.compiler"
@@ -88,24 +85,12 @@
<arg path="${jps.bootstrap.dir}/src/main/java" />
<arg path="${community.home}/platform/build-scripts/downloader/src" />
<arg value="-d" />
<arg value="${classes.dir}/kotlin" />
<arg value="${classes.dir}" />
</java>
<javac
destdir="${classes.dir}/java"
encoding="UTF-8"
release="11"
debug="true"
includeantruntime="false"
classpathref="classpath.buildscripts">
<classpath path="${classes.dir}/kotlin" />
<src path="${jps.bootstrap.dir}/src/main/java" />
<src path="${community.home}/platform/build-scripts/downloader/src" />
</javac>
<delete file="${classes.dir}.jar" />
<zip destfile="${classes.dir}.jar">
<fileset dir="${classes.dir}/java" />
<fileset dir="${classes.dir}/kotlin" />
<fileset dir="${classes.dir}" />
</zip>
<mkdir dir="${uber.dir}" />
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/ClassesFromCompileInc.kt
+59 -83
View File
@@ -1,106 +1,82 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jpsBootstrap;
package org.jetbrains.jpsBootstrap
import com.google.common.base.StandardSystemProperty;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
import com.intellij.openapi.util.Pair;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader;
import org.jetbrains.jps.model.JpsProject;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.java.JpsJavaModuleExtension;
import org.jetbrains.jps.model.java.JpsJavaProjectExtension;
import org.jetbrains.jps.model.module.JpsModule;
import org.jetbrains.jps.util.JpsPathUtil;
import com.google.common.base.StandardSystemProperty
import com.google.gson.Gson
import com.google.gson.annotations.SerializedName
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.downloadFileToCacheLocation
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.extractFileToCacheLocation
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.verbose
import org.jetbrains.jps.model.JpsProject
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.util.JpsPathUtil
import java.io.IOException
import java.net.URI
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
import java.util.*
import java.util.concurrent.*
import java.io.BufferedReader;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
object ClassesFromCompileInc {
const val MANIFEST_JSON_URL_ENV_NAME = "JPS_BOOTSTRAP_MANIFEST_JSON_URL"
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.verbose;
public final class ClassesFromCompileInc {
public final static String MANIFEST_JSON_URL_ENV_NAME = "JPS_BOOTSTRAP_MANIFEST_JSON_URL";
public static void downloadProjectClasses(JpsProject project, BuildDependenciesCommunityRoot communityRoot, Collection<JpsModule> modules) throws IOException, InterruptedException {
String manifestUrl = System.getenv(MANIFEST_JSON_URL_ENV_NAME);
if (manifestUrl == null || manifestUrl.isBlank()) {
throw new IllegalStateException("Env variable '" + MANIFEST_JSON_URL_ENV_NAME + "' is missing or empty");
}
verbose("Got manifest json url '" + manifestUrl + "' from $" + MANIFEST_JSON_URL_ENV_NAME);
final Path manifest = BuildDependenciesDownloader.downloadFileToCacheLocation(communityRoot, URI.create(manifestUrl));
Map<JpsModule, Path> productionModuleOutputs = downloadProductionPartsFromMetadataJson(manifest, communityRoot, modules);
assignModuleOutputs(project, productionModuleOutputs);
@Throws(IOException::class, InterruptedException::class)
fun downloadProjectClasses(project: JpsProject, communityRoot: BuildDependenciesCommunityRoot, modules: Collection<JpsModule?>?) {
val manifestUrl = System.getenv(MANIFEST_JSON_URL_ENV_NAME)
check(!(manifestUrl == null || manifestUrl.isBlank())) { "Env variable '$MANIFEST_JSON_URL_ENV_NAME' is missing or empty" }
verbose("Got manifest json url '$manifestUrl' from $$MANIFEST_JSON_URL_ENV_NAME")
val manifest = downloadFileToCacheLocation(communityRoot, URI.create(manifestUrl))
val productionModuleOutputs = downloadProductionPartsFromMetadataJson(manifest, communityRoot, modules)
assignModuleOutputs(project, productionModuleOutputs)
}
private static void assignModuleOutputs(JpsProject project, Map<JpsModule, Path> productionModuleOutputs) {
Path nonExistentPath = Path.of(
private fun assignModuleOutputs(project: JpsProject, productionModuleOutputs: Map<JpsModule, Path>) {
val nonExistentPath = Path.of(
System.getProperty(StandardSystemProperty.JAVA_IO_TMPDIR.key()),
UUID.randomUUID().toString());
UUID.randomUUID().toString())
// Set it to non-existent path since we won't run build and standard built output won't be available anyway
JpsJavaProjectExtension projectExtension = JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(project);
projectExtension.setOutputUrl(JpsPathUtil.pathToUrl(nonExistentPath.toString()));
for (Map.Entry<JpsModule, Path> entry : productionModuleOutputs.entrySet()) {
JpsModule module = entry.getKey();
final JpsJavaModuleExtension javaExtension = JpsJavaExtensionService.getInstance().getOrCreateModuleExtension(module);
javaExtension.setOutputUrl(JpsPathUtil.pathToUrl(entry.getValue().toString()));
javaExtension.setInheritOutput(false);
val projectExtension = JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(project)
projectExtension.outputUrl = JpsPathUtil.pathToUrl(nonExistentPath.toString())
for ((module, value) in productionModuleOutputs) {
val javaExtension = JpsJavaExtensionService.getInstance().getOrCreateModuleExtension(module)
javaExtension.outputUrl = JpsPathUtil.pathToUrl(value.toString())
javaExtension.isInheritOutput = false
}
}
private static Map<JpsModule, Path> downloadProductionPartsFromMetadataJson(Path metadataJson, BuildDependenciesCommunityRoot communityRoot, Collection<JpsModule> modules) throws InterruptedException, IOException {
CompilationPartsMetadata partsMetadata;
try (BufferedReader manifestReader = Files.newBufferedReader(metadataJson, StandardCharsets.UTF_8)) {
partsMetadata = new Gson().fromJson(manifestReader, CompilationPartsMetadata.class);
private fun downloadProductionPartsFromMetadataJson(metadataJson: Path, communityRoot: BuildDependenciesCommunityRoot, modules: Collection<JpsModule?>?): Map<JpsModule, Path> {
var partsMetadata: CompilationPartsMetadata
Files.newBufferedReader(metadataJson, StandardCharsets.UTF_8).use { manifestReader -> partsMetadata = Gson().fromJson(manifestReader, CompilationPartsMetadata::class.java) }
check(partsMetadata.files!!.isNotEmpty()) { "partsMetadata.files is empty, check $metadataJson" }
val tasks: MutableList<Callable<Pair<JpsModule, Path>>> = ArrayList()
for (module in modules!!) {
val c = Callable {
val modulePrefix = "production/" + module!!.name
val hash = partsMetadata.files!![modulePrefix]
?: throw IllegalStateException("Unable to find module output by name '$modulePrefix' in $metadataJson")
val outputPartUri = URI.create(partsMetadata.serverUrl + "/" + partsMetadata.prefix + "/" + modulePrefix + "/" + hash + ".jar")
val outputPart = downloadFileToCacheLocation(communityRoot, outputPartUri)
val outputPartExtracted = extractFileToCacheLocation(communityRoot, outputPart)
module to outputPartExtracted
}
tasks.add(c)
}
if (partsMetadata.files.isEmpty()) {
throw new IllegalStateException("partsMetadata.files is empty, check " + metadataJson);
}
List<Callable<Pair<JpsModule, Path>>> tasks = new ArrayList<>();
for (final JpsModule module : modules) {
Callable<Pair<JpsModule, Path>> c = () -> {
String modulePrefix = "production/" + module.getName();
String hash = partsMetadata.files.get(modulePrefix);
if (hash == null) {
throw new IllegalStateException("Unable to find module output by name '" + modulePrefix + "' in " + metadataJson);
}
URI outputPartUri = URI.create(partsMetadata.serverUrl + "/" + partsMetadata.prefix + "/" + modulePrefix + "/" + hash + ".jar");
final Path outputPart = BuildDependenciesDownloader.downloadFileToCacheLocation(communityRoot, outputPartUri);
final Path outputPartExtracted = BuildDependenciesDownloader.extractFileToCacheLocation(communityRoot, outputPart);
return Pair.pair(module, outputPartExtracted);
};
tasks.add(c);
}
return JpsBootstrapUtil.executeTasksInParallel(tasks)
.stream().collect(Collectors.toUnmodifiableMap(pair -> pair.getFirst(), pair -> pair.getSecond()));
return JpsBootstrapUtil.executeTasksInParallel(tasks).associate { it.first to it.second }
}
private static final class CompilationPartsMetadata {
private class CompilationPartsMetadata {
@SerializedName("server-url")
public String serverUrl;
public String prefix;
var serverUrl: String? = null
var prefix: String? = null
/**
* Map compilation part path to a hash, for now SHA-256 is used.
* sha256(file) == hash, though that may be changed in the future.
*/
public Map<String, String> files;
var files: Map<String, String>? = null
}
}
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/JpsBootstrapMain.kt
+274 -302
View File
@@ -1,192 +1,131 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jpsBootstrap
package org.jetbrains.jpsBootstrap;
import com.google.common.hash.Hashing
import com.intellij.execution.CommandLineWrapperUtil
import com.intellij.openapi.diagnostic.IdeaLogRecordFormatter
import com.intellij.openapi.util.SystemInfo
import com.intellij.openapi.util.text.Strings
import com.intellij.util.ExceptionUtil
import jetbrains.buildServer.messages.serviceMessages.MessageWithAttributes
import jetbrains.buildServer.messages.serviceMessages.ServiceMessageTypes
import org.apache.commons.cli.*
import org.jetbrains.annotations.Contract
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.fatal
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.setVerboseEnabled
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.verbose
import org.jetbrains.intellij.build.dependencies.JdkDownloader.getJavaExecutable
import org.jetbrains.intellij.build.dependencies.JdkDownloader.getJdkHome
import org.jetbrains.intellij.build.dependencies.TeamCityHelper.isUnderTeamCity
import org.jetbrains.jps.incremental.storage.ProjectStamps
import org.jetbrains.jps.model.JpsModel
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jpsBootstrap.JpsBootstrapUtil.toBooleanChecked
import java.io.File
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.util.*
import java.util.logging.ConsoleHandler
import java.util.logging.Level
import java.util.logging.Logger
import java.util.stream.Collectors
import kotlin.io.path.readLines
import kotlin.system.exitProcess
import com.google.common.hash.Hashing;
import com.intellij.execution.CommandLineWrapperUtil;
import com.intellij.openapi.diagnostic.IdeaLogRecordFormatter;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.text.Strings;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.containers.ContainerUtil;
import jetbrains.buildServer.messages.serviceMessages.MessageWithAttributes;
import jetbrains.buildServer.messages.serviceMessages.ServiceMessageTypes;
import org.apache.commons.cli.*;
import org.jetbrains.annotations.Contract;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging;
import org.jetbrains.intellij.build.dependencies.JdkDownloader;
import org.jetbrains.intellij.build.dependencies.TeamCityHelper;
import org.jetbrains.jps.incremental.storage.ProjectStamps;
import org.jetbrains.jps.model.JpsModel;
import org.jetbrains.jps.model.module.JpsModule;
class JpsBootstrapMain(args: Array<String>?) {
private val projectHome: Path
private val communityHome: BuildDependenciesCommunityRoot
private var moduleNameToRun: String? = null
private var classNameToRun: String? = null
private val buildTargetXmx: String
private val jpsBootstrapWorkDir: Path
private var javaArgsFileTarget: Path? = null
private var mainArgsToRun: List<String>? = null
private val additionalSystemProperties: Properties
private val additionalSystemPropertiesFromPropertiesFile: Properties
private val onlyDownloadJdk: Boolean
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.logging.ConsoleHandler;
import java.util.logging.Handler;
import java.util.stream.Collectors;
init {
initLogging()
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.*;
import static org.jetbrains.jpsBootstrap.JpsBootstrapUtil.getJpsArtifactsResolutionRetryProperties;
import static org.jetbrains.jpsBootstrap.JpsBootstrapUtil.getTeamCitySystemProperties;
import static org.jetbrains.jpsBootstrap.JpsBootstrapUtil.toBooleanChecked;
@SuppressWarnings({"SameParameterValue"})
public class JpsBootstrapMain {
private static final String DEFAULT_BUILD_SCRIPT_XMX = "4g";
private static final String COMMUNITY_HOME_ENV = "JPS_BOOTSTRAP_COMMUNITY_HOME";
private static final String JPS_BOOTSTRAP_VERBOSE = "JPS_BOOTSTRAP_VERBOSE";
private static final Option OPT_HELP = Option.builder("h").longOpt("help").build();
private static final Option OPT_VERBOSE = Option.builder("v").longOpt("verbose").desc("Show more logging from jps-bootstrap and the building process").build();
private static final Option OPT_SYSTEM_PROPERTY = Option.builder("D").hasArgs().valueSeparator('=').desc("Pass system property to the build script").build();
private static final Option OPT_PROPERTIES_FILE = Option.builder().longOpt("properties-file").hasArg().desc("Pass system properties to the build script from specified properties file https://en.wikipedia.org/wiki/.properties").build();
private static final Option OPT_BUILD_TARGET_XMX = Option.builder().longOpt("build-target-xmx").hasArg().desc("Specify Xmx to run build script. default: " + DEFAULT_BUILD_SCRIPT_XMX).build();
private static final Option OPT_JAVA_ARGFILE_TARGET = Option.builder().longOpt("java-argfile-target").hasArg().desc("Write java argfile to this file").build();
private static final Option OPT_ONLY_DOWNLOAD_JDK = Option.builder().longOpt("download-jdk").desc("Download project JDK and exit").build();
private static final List<Option> ALL_OPTIONS =
Arrays.asList(OPT_HELP, OPT_VERBOSE, OPT_SYSTEM_PROPERTY, OPT_PROPERTIES_FILE, OPT_JAVA_ARGFILE_TARGET, OPT_BUILD_TARGET_XMX, OPT_ONLY_DOWNLOAD_JDK);
static final boolean underTeamCity = TeamCityHelper.INSTANCE.isUnderTeamCity();
private static Options createCliOptions() {
Options opts = new Options();
for (Option option : ALL_OPTIONS) {
opts.addOption(option);
val cmdline = try {
DefaultParser().parse(createCliOptions(), args, true)
}
catch (e: ParseException) {
e.printStackTrace()
showUsagesAndExit()
throw IllegalStateException("NOT_REACHED")
}
return opts;
}
public static void main(String[] args) {
Path jpsBootstrapWorkDir = null;
try {
JpsBootstrapMain mainInstance = new JpsBootstrapMain(args);
jpsBootstrapWorkDir = mainInstance.jpsBootstrapWorkDir;
mainInstance.main();
System.exit(0);
}
catch (Throwable t) {
fatal(ExceptionUtil.getThrowableText(t));
// Better diagnostics for local users
if (!TeamCityHelper.INSTANCE.isUnderTeamCity()) {
System.err.println("\n###### ERROR EXIT due to FATAL error: " + t.getMessage() + "\n");
String work = jpsBootstrapWorkDir == null ? "PROJECT_HOME/build/jps-bootstrap-work" : jpsBootstrapWorkDir.toString();
System.err.println("###### You may try to delete caches at " + work + " and retry");
}
System.exit(1);
}
}
private final Path projectHome;
private final BuildDependenciesCommunityRoot communityHome;
private final String moduleNameToRun;
private final String classNameToRun;
private final String buildTargetXmx;
private final Path jpsBootstrapWorkDir;
private final Path javaArgsFileTarget;
private final List<String> mainArgsToRun;
private final Properties additionalSystemProperties;
private final Properties additionalSystemPropertiesFromPropertiesFile;
private final boolean onlyDownloadJdk;
public JpsBootstrapMain(String[] args) throws IOException {
initLogging();
CommandLine cmdline;
try {
cmdline = (new DefaultParser()).parse(createCliOptions(), args, true);
}
catch (ParseException e) {
e.printStackTrace();
showUsagesAndExit();
throw new IllegalStateException("NOT_REACHED");
val freeArgs = cmdline.args.toList()
if (cmdline.hasOption(OPT_HELP) || freeArgs.isEmpty()) {
showUsagesAndExit()
}
final List<String> freeArgs = Arrays.asList(cmdline.getArgs());
if (cmdline.hasOption(OPT_HELP) || freeArgs.size() < 1) {
showUsagesAndExit();
}
projectHome = Path.of(freeArgs.get(0)).normalize();
onlyDownloadJdk = cmdline.hasOption(OPT_ONLY_DOWNLOAD_JDK);
projectHome = Path.of(freeArgs.first()).normalize()
onlyDownloadJdk = cmdline.hasOption(OPT_ONLY_DOWNLOAD_JDK)
if (onlyDownloadJdk) {
moduleNameToRun = null;
classNameToRun = null;
mainArgsToRun = Collections.emptyList();
javaArgsFileTarget = null;
moduleNameToRun = null
classNameToRun = null
mainArgsToRun = emptyList()
javaArgsFileTarget = null
}
else {
moduleNameToRun = freeArgs.get(1);
classNameToRun = freeArgs.get(2);
mainArgsToRun = freeArgs.subList(3, freeArgs.size());
javaArgsFileTarget = Path.of(cmdline.getOptionValue(OPT_JAVA_ARGFILE_TARGET));
moduleNameToRun = freeArgs[1]
classNameToRun = freeArgs[2]
mainArgsToRun = freeArgs.subList(3, freeArgs.size)
javaArgsFileTarget = Path.of(cmdline.getOptionValue(OPT_JAVA_ARGFILE_TARGET))
}
additionalSystemProperties = cmdline.getOptionProperties("D");
additionalSystemPropertiesFromPropertiesFile = new Properties();
additionalSystemProperties = cmdline.getOptionProperties("D")
additionalSystemPropertiesFromPropertiesFile = Properties()
if (cmdline.hasOption(OPT_PROPERTIES_FILE)) {
Path propertiesFile = Path.of(cmdline.getOptionValue(OPT_PROPERTIES_FILE));
try (Reader reader = Files.newBufferedReader(propertiesFile)) {
info("Loading properties from " + propertiesFile);
additionalSystemPropertiesFromPropertiesFile.load(reader);
val propertiesFile = Path.of(cmdline.getOptionValue(OPT_PROPERTIES_FILE))
Files.newBufferedReader(propertiesFile).use { reader ->
info("Loading properties from $propertiesFile")
additionalSystemPropertiesFromPropertiesFile.load(reader)
}
}
String verboseEnv = System.getenv(JPS_BOOTSTRAP_VERBOSE);
BuildDependenciesLogging.setVerboseEnabled(cmdline.hasOption(OPT_VERBOSE) || (verboseEnv != null && toBooleanChecked(verboseEnv)));
val verboseEnv = System.getenv(JPS_BOOTSTRAP_VERBOSE)
setVerboseEnabled(cmdline.hasOption(OPT_VERBOSE) || (verboseEnv != null && verboseEnv.toBooleanChecked()))
String communityHomeString = System.getenv(COMMUNITY_HOME_ENV);
if (communityHomeString == null) {
throw new IllegalStateException("Please set " + COMMUNITY_HOME_ENV + " environment variable");
}
communityHome = new BuildDependenciesCommunityRoot(Path.of(communityHomeString));
jpsBootstrapWorkDir = projectHome.resolve("build").resolve("jps-bootstrap-work");
info("Working directory: " + jpsBootstrapWorkDir);
Files.createDirectories(jpsBootstrapWorkDir);
buildTargetXmx = cmdline.hasOption(OPT_BUILD_TARGET_XMX) ? cmdline.getOptionValue(OPT_BUILD_TARGET_XMX) : DEFAULT_BUILD_SCRIPT_XMX;
val communityHomeString = System.getenv(COMMUNITY_HOME_ENV)
?: error("Please set $COMMUNITY_HOME_ENV environment variable")
communityHome = BuildDependenciesCommunityRoot(Path.of(communityHomeString))
jpsBootstrapWorkDir = projectHome.resolve("build").resolve("jps-bootstrap-work")
info("Working directory: $jpsBootstrapWorkDir")
Files.createDirectories(jpsBootstrapWorkDir)
buildTargetXmx = if (cmdline.hasOption(OPT_BUILD_TARGET_XMX)) cmdline.getOptionValue(OPT_BUILD_TARGET_XMX) else DEFAULT_BUILD_SCRIPT_XMX
}
private Path downloadJdk() {
Path jdkHome;
private fun downloadJdk(): Path {
val jdkHome: Path
if (underTeamCity) {
jdkHome = JdkDownloader.getJdkHome(communityHome);
SetParameterServiceMessage setParameterServiceMessage = new SetParameterServiceMessage(
jdkHome = getJdkHome(communityHome)
var setParameterServiceMessage = SetParameterServiceMessage(
"jps.bootstrap.java.home", jdkHome.toString()
);
System.out.println(setParameterServiceMessage.asString());
setParameterServiceMessage = new SetParameterServiceMessage(
"jps.bootstrap.java.executable", JdkDownloader.INSTANCE.getJavaExecutable(jdkHome).toString());
System.out.println(setParameterServiceMessage.asString());
)
println(setParameterServiceMessage.asString())
setParameterServiceMessage = SetParameterServiceMessage(
"jps.bootstrap.java.executable", getJavaExecutable(jdkHome).toString())
println(setParameterServiceMessage.asString())
}
else {
// On local run JDK was already downloaded via jps-bootstrap.{sh,cmd}
jdkHome = Path.of(System.getProperty("java.home"));
jdkHome = Path.of(System.getProperty("java.home"))
}
return jdkHome;
return jdkHome
}
private void main() throws Throwable {
Path jdkHome = downloadJdk();
@Throws(Throwable::class)
private fun main() {
val jdkHome = downloadJdk()
if (onlyDownloadJdk) {
return;
return
}
/*
@@ -194,133 +133,123 @@ public class JpsBootstrapMain {
* Don't override settings properties if they're already present in System.properties(), in additionalSystemProperties or
* in additionalSystemPropertiesFromPropertiesFile.
*/
Properties resolverRetrySettingsProperties = getJpsArtifactsResolutionRetryProperties(
val resolverRetrySettingsProperties = JpsBootstrapUtil.getJpsArtifactsResolutionRetryProperties(
additionalSystemPropertiesFromPropertiesFile,
additionalSystemProperties,
System.getProperties()
);
resolverRetrySettingsProperties.forEach((k, v) -> System.setProperty((String) k, (String) v));
)
resolverRetrySettingsProperties.forEach { k, v -> System.setProperty(k as String, v as String) }
Path kotlincHome = KotlinCompiler.downloadAndExtractKotlinCompiler(communityHome);
JpsModel model = JpsProjectUtils.loadJpsProject(projectHome, jdkHome, kotlincHome);
JpsModule module = JpsProjectUtils.getModuleByName(model, moduleNameToRun);
downloadOrBuildClasses(module, model, kotlincHome);
List<File> moduleRuntimeClasspath = JpsProjectUtils.getModuleRuntimeClasspath(module);
verbose("Module " + module.getName() + " classpath:\n " + moduleRuntimeClasspath.stream().map(JpsBootstrapMain::fileDebugInfo).collect(Collectors.joining("\n ")));
writeJavaArgfile(moduleRuntimeClasspath);
val kotlincHome = KotlinCompiler.downloadAndExtractKotlinCompiler(communityHome)
val model = JpsProjectUtils.loadJpsProject(projectHome, jdkHome, kotlincHome)
val module = JpsProjectUtils.getModuleByName(model, moduleNameToRun!!)
downloadOrBuildClasses(module, model, kotlincHome)
val moduleRuntimeClasspath = JpsProjectUtils.getModuleRuntimeClasspath(module)
verbose("""Module ${module.name} classpath:
${moduleRuntimeClasspath.stream().map { file: File? -> fileDebugInfo(file) }.collect(Collectors.joining("\n "))}""")
writeJavaArgfile(moduleRuntimeClasspath)
}
private void removeOpenedPackage(List<String> openedPackages, String openedPackage, List<String> unknownPackages) {
private fun removeOpenedPackage(openedPackages: MutableList<String>, openedPackage: String, unknownPackages: MutableList<String>) {
if (!openedPackages.remove(openedPackage)) {
unknownPackages.add(openedPackage);
unknownPackages.add(openedPackage)
}
}
private List<String> getOpenedPackages() throws Exception {
Path openedPackagesPath = communityHome.communityRoot.resolve("plugins/devkit/devkit-core/src/run/OpenedPackages.txt");
List<String> openedPackages = ContainerUtil.filter(Files.readAllLines(openedPackagesPath), it -> !it.isBlank());
List<String> unknownPackages = new ArrayList<>();
if (!SystemInfo.isWindows) {
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/sun.awt.windows=ALL-UNNAMED", unknownPackages);
}
if (!SystemInfo.isMac) {
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/com.apple.eawt=ALL-UNNAMED", unknownPackages);
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/com.apple.eawt.event=ALL-UNNAMED", unknownPackages);
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/com.apple.laf=ALL-UNNAMED", unknownPackages);
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/sun.lwawt.macosx=ALL-UNNAMED", unknownPackages);
}
if (!SystemInfo.isLinux) {
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/com.sun.java.swing.plaf.gtk=ALL-UNNAMED", unknownPackages);
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/sun.awt.X11=ALL-UNNAMED", unknownPackages);
removeOpenedPackage(openedPackages,"--add-opens=java.desktop/sun.lwawt=ALL-UNNAMED", unknownPackages);
}
if (!unknownPackages.isEmpty()) {
throw new Exception(String.format("OS specific opened packages: ['%s'] not found in '%s'. " +
@get:Throws(Exception::class)
private val openedPackages: List<String>
get() {
val openedPackagesPath = communityHome.communityRoot.resolve("plugins/devkit/devkit-core/src/run/OpenedPackages.txt")
val openedPackages = openedPackagesPath.readLines().filter { it.isNotBlank() }.toMutableList()
val unknownPackages = mutableListOf<String>()
if (!SystemInfo.isWindows) {
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/sun.awt.windows=ALL-UNNAMED", unknownPackages)
}
if (!SystemInfo.isMac) {
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/com.apple.eawt=ALL-UNNAMED", unknownPackages)
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/com.apple.eawt.event=ALL-UNNAMED", unknownPackages)
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/com.apple.laf=ALL-UNNAMED", unknownPackages)
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/sun.lwawt.macosx=ALL-UNNAMED", unknownPackages)
}
if (!SystemInfo.isLinux) {
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/com.sun.java.swing.plaf.gtk=ALL-UNNAMED", unknownPackages)
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/sun.awt.X11=ALL-UNNAMED", unknownPackages)
removeOpenedPackage(openedPackages, "--add-opens=java.desktop/sun.lwawt=ALL-UNNAMED", unknownPackages)
}
if (unknownPackages.isNotEmpty()) {
throw Exception(String.format("OS specific opened packages: ['%s'] not found in '%s'. " +
"Probably you need to clean up OS-specific package names in org.jetbrains.jpsBootstrap.JpsBootstrapMain",
String.join("','", unknownPackages), openedPackagesPath));
java.lang.String.join("','", unknownPackages), openedPackagesPath))
}
return openedPackages
}
return openedPackages;
}
private void writeJavaArgfile(List<File> moduleRuntimeClasspath) throws Exception {
Properties systemProperties = new Properties();
@Throws(Exception::class)
private fun writeJavaArgfile(moduleRuntimeClasspath: List<File?>?) {
val systemProperties = Properties()
if (underTeamCity) {
systemProperties.putAll(getTeamCitySystemProperties());
systemProperties.putAll(JpsBootstrapUtil.teamCitySystemProperties)
}
systemProperties.putAll(additionalSystemPropertiesFromPropertiesFile);
systemProperties.putAll(additionalSystemProperties);
systemProperties.putIfAbsent("file.encoding", "UTF-8"); // just in case
systemProperties.putIfAbsent("java.awt.headless", "true");
systemProperties.putAll(additionalSystemPropertiesFromPropertiesFile)
systemProperties.putAll(additionalSystemProperties)
systemProperties.putIfAbsent("file.encoding", "UTF-8") // just in case
systemProperties.putIfAbsent("java.awt.headless", "true")
/*
* Add dependencies resolution retries properties to argfile.
* Don't override them if they're already present in additionalSystemProperties or in additionalSystemPropertiesFromPropertiesFile.
*/
Properties resolverRetrySettingsProperties = getJpsArtifactsResolutionRetryProperties(
* Add dependencies resolution retries properties to argfile.
* Don't override them if they're already present in additionalSystemProperties or in additionalSystemPropertiesFromPropertiesFile.
*/
val resolverRetrySettingsProperties = JpsBootstrapUtil.getJpsArtifactsResolutionRetryProperties(
additionalSystemPropertiesFromPropertiesFile,
additionalSystemProperties
);
systemProperties.putAll(resolverRetrySettingsProperties);
List<String> args = new ArrayList<>();
args.add("-ea");
args.add("-Xmx" + buildTargetXmx);
args.addAll(getOpenedPackages());
args.addAll(convertPropertiesToCommandLineArgs(systemProperties));
args.add("-classpath");
args.add(Strings.join(moduleRuntimeClasspath, File.pathSeparator));
args.add("-Dbuild.script.launcher.main.class=" + classNameToRun);
args.add("org.jetbrains.intellij.build.impl.BuildScriptLauncher");
args.addAll(mainArgsToRun);
)
systemProperties.putAll(resolverRetrySettingsProperties)
val args: MutableList<String> = ArrayList()
args.add("-ea")
args.add("-Xmx$buildTargetXmx")
args.addAll(openedPackages)
args.addAll(convertPropertiesToCommandLineArgs(systemProperties))
args.add("-classpath")
args.add(Strings.join(moduleRuntimeClasspath!!, File.pathSeparator))
args.add("-Dbuild.script.launcher.main.class=$classNameToRun")
args.add("org.jetbrains.intellij.build.impl.BuildScriptLauncher")
args.addAll(mainArgsToRun!!)
CommandLineWrapperUtil.writeArgumentsFile(
javaArgsFileTarget.toFile(),
javaArgsFileTarget!!.toFile(),
args,
StandardCharsets.UTF_8
);
info("java argfile:\n" + Files.readString(javaArgsFileTarget));
)
info("""
java argfile:
${Files.readString(javaArgsFileTarget)}
""".trimIndent())
}
private void downloadOrBuildClasses(JpsModule module, JpsModel model, Path kotlincHome) throws Throwable {
String fromJpsBuildEnvValue = System.getenv(JpsBuild.CLASSES_FROM_JPS_BUILD_ENV_NAME);
boolean runJpsBuild = fromJpsBuildEnvValue != null && JpsBootstrapUtil.toBooleanChecked(fromJpsBuildEnvValue) || ProjectStamps.PORTABLE_CACHES;
@Throws(Throwable::class)
private fun downloadOrBuildClasses(module: JpsModule, model: JpsModel, kotlincHome: Path) {
val fromJpsBuildEnvValue = System.getenv(JpsBuild.CLASSES_FROM_JPS_BUILD_ENV_NAME)
val runJpsBuild = (fromJpsBuildEnvValue != null && fromJpsBuildEnvValue.toBooleanChecked()) || ProjectStamps.PORTABLE_CACHES
String manifestJsonUrl = System.getenv(ClassesFromCompileInc.MANIFEST_JSON_URL_ENV_NAME);
var manifestJsonUrl = System.getenv(ClassesFromCompileInc.MANIFEST_JSON_URL_ENV_NAME)
if (manifestJsonUrl != null && manifestJsonUrl.isBlank()) {
manifestJsonUrl = null;
manifestJsonUrl = null
}
if (runJpsBuild && manifestJsonUrl != null) {
throw new IllegalStateException("Both env. variables are set, choose only one: " +
check(!(runJpsBuild && manifestJsonUrl != null)) {
"Both env. variables are set, choose only one: " +
JpsBuild.CLASSES_FROM_JPS_BUILD_ENV_NAME + " " +
ClassesFromCompileInc.MANIFEST_JSON_URL_ENV_NAME);
ClassesFromCompileInc.MANIFEST_JSON_URL_ENV_NAME
}
if (!runJpsBuild && manifestJsonUrl == null) {
// Nothing specified. It's ok locally, but on buildserver we must be sure
if (underTeamCity) {
throw new IllegalStateException("On buildserver one of the following env. variables must be set: " +
check(!underTeamCity) {
"On buildserver one of the following env. variables must be set: " +
JpsBuild.CLASSES_FROM_JPS_BUILD_ENV_NAME + " " +
ClassesFromCompileInc.MANIFEST_JSON_URL_ENV_NAME);
ClassesFromCompileInc.MANIFEST_JSON_URL_ENV_NAME
}
}
Set<JpsModule> modulesSubset = JpsProjectUtils.getRuntimeModulesClasspath(module);
JpsBuild jpsBuild = new JpsBuild(communityHome, model, jpsBootstrapWorkDir, kotlincHome);
val modulesSubset = JpsProjectUtils.getRuntimeModulesClasspath(module)
val jpsBuild = JpsBuild(communityHome, model, jpsBootstrapWorkDir, kotlincHome)
// Some workarounds like 'kotlinx.kotlinx-serialization-compiler-plugin-for-compilation' library (used as Kotlin compiler plugin)
// require that the corresponding library was downloaded. It's unclear from modules structure which libraries exactly required
@@ -328,71 +257,114 @@ public class JpsBootstrapMain {
//
// In case of running from read-to-use classes we need all dependent libraries as well
// Instead of calculating what libraries are exactly required, download them all
jpsBuild.resolveProjectDependencies();
jpsBuild.resolveProjectDependencies()
if (manifestJsonUrl != null) {
info("Downloading project classes from " + manifestJsonUrl);
ClassesFromCompileInc.downloadProjectClasses(model.getProject(), communityHome, modulesSubset);
} else {
jpsBuild.buildModules(modulesSubset);
info("Downloading project classes from $manifestJsonUrl")
ClassesFromCompileInc.downloadProjectClasses(model.project, communityHome, modulesSubset)
}
else {
jpsBuild.buildModules(modulesSubset)
}
}
private static String fileDebugInfo(File file) {
try {
if (file.exists()) {
BasicFileAttributes attributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
if (attributes.isDirectory()) {
return file + " directory";
private class SetParameterServiceMessage(name: String, value: String)
: MessageWithAttributes(ServiceMessageTypes.BUILD_SET_PARAMETER, mapOf("name" to name, "value" to value))
companion object {
private const val DEFAULT_BUILD_SCRIPT_XMX = "4g"
private const val COMMUNITY_HOME_ENV = "JPS_BOOTSTRAP_COMMUNITY_HOME"
private const val JPS_BOOTSTRAP_VERBOSE = "JPS_BOOTSTRAP_VERBOSE"
private val OPT_HELP = Option.builder("h").longOpt("help").build()
private val OPT_VERBOSE = Option.builder("v").longOpt("verbose").desc("Show more logging from jps-bootstrap and the building process").build()
private val OPT_SYSTEM_PROPERTY = Option.builder("D").hasArgs().valueSeparator('=').desc("Pass system property to the build script").build()
private val OPT_PROPERTIES_FILE = Option.builder().longOpt("properties-file").hasArg().desc("Pass system properties to the build script from specified properties file https://en.wikipedia.org/wiki/.properties").build()
private val OPT_BUILD_TARGET_XMX = Option.builder().longOpt("build-target-xmx").hasArg().desc("Specify Xmx to run build script. default: $DEFAULT_BUILD_SCRIPT_XMX").build()
private val OPT_JAVA_ARGFILE_TARGET = Option.builder().longOpt("java-argfile-target").hasArg().desc("Write java argfile to this file").build()
private val OPT_ONLY_DOWNLOAD_JDK = Option.builder().longOpt("download-jdk").desc("Download project JDK and exit").build()
private val ALL_OPTIONS = listOf(OPT_HELP, OPT_VERBOSE, OPT_SYSTEM_PROPERTY, OPT_PROPERTIES_FILE, OPT_JAVA_ARGFILE_TARGET, OPT_BUILD_TARGET_XMX, OPT_ONLY_DOWNLOAD_JDK)
val underTeamCity = isUnderTeamCity
private fun createCliOptions(): Options {
val opts = Options()
for (option in ALL_OPTIONS) {
opts.addOption(option)
}
return opts
}
@JvmStatic
fun main(args: Array<String>) {
var jpsBootstrapWorkDir: Path? = null
try {
val mainInstance = JpsBootstrapMain(args)
@Suppress("UNUSED_VALUE")
jpsBootstrapWorkDir = mainInstance.jpsBootstrapWorkDir
mainInstance.main()
exitProcess(0)
}
catch (t: Throwable) {
fatal(ExceptionUtil.getThrowableText(t))
// Better diagnostics for local users
if (!isUnderTeamCity) {
System.err.println("""
###### ERROR EXIT due to FATAL error: ${t.message}
""".trimIndent())
val work = jpsBootstrapWorkDir?.toString() ?: "PROJECT_HOME/build/jps-bootstrap-work"
System.err.println("###### You may try to delete caches at $work and retry")
}
exitProcess(1)
}
}
private fun fileDebugInfo(file: File?): String {
return try {
if (file!!.exists()) {
val attributes = Files.readAttributes(file.toPath(), BasicFileAttributes::class.java)
if (attributes.isDirectory) {
"$file directory"
}
else {
val length = attributes.size()
val sha256 = Hashing.sha256().hashBytes(Files.readAllBytes(file.toPath())).toString()
"$file file length $length sha256 $sha256"
}
}
else {
long length = attributes.size();
String sha256 = Hashing.sha256().hashBytes(Files.readAllBytes(file.toPath())).toString();
return file + " file length " + length + " sha256 " + sha256;
"$file missing file"
}
}
else {
return file + " missing file";
catch (e: Exception) {
throw RuntimeException(e)
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static List<String> convertPropertiesToCommandLineArgs(Properties properties) {
List<String> result = new ArrayList<>();
for (String propertyName : properties.stringPropertyNames().stream().sorted().collect(Collectors.toList())) {
String value = properties.getProperty(propertyName);
result.add("-D" + propertyName + "=" + value);
private fun convertPropertiesToCommandLineArgs(properties: Properties): List<String> {
return properties
.map { (it.key as String) to (it.value as String) }
.sortedBy { it.first }
.map { "-D${it.first}=${it.second}" }
}
return result;
}
@Contract("->fail")
private static void showUsagesAndExit() {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(1000);
formatter.printHelp("./jps-bootstrap.sh [jps-bootstrap options] MODULE_NAME CLASS_NAME [arguments_passed_to_CLASS_NAME's_main]", createCliOptions());
System.exit(1);
}
private static void initLogging() {
java.util.logging.Logger rootLogger = java.util.logging.Logger.getLogger("");
for (Handler handler : rootLogger.getHandlers()) {
rootLogger.removeHandler(handler);
@Contract("->fail")
private fun showUsagesAndExit() {
val formatter = HelpFormatter()
formatter.width = 1000
formatter.printHelp("./jps-bootstrap.sh [jps-bootstrap options] MODULE_NAME CLASS_NAME [arguments_passed_to_CLASS_NAME's_main]", createCliOptions())
exitProcess(1)
}
IdeaLogRecordFormatter layout = new IdeaLogRecordFormatter();
ConsoleHandler consoleHandler = new ConsoleHandler();
consoleHandler.setFormatter(new IdeaLogRecordFormatter(false, layout));
consoleHandler.setLevel(java.util.logging.Level.WARNING);
rootLogger.addHandler(consoleHandler);
}
private static class SetParameterServiceMessage extends MessageWithAttributes {
public SetParameterServiceMessage(@NotNull String name, @NotNull String value) {
super(ServiceMessageTypes.BUILD_SET_PARAMETER, Map.of("name", name, "value", value));
private fun initLogging() {
val rootLogger = Logger.getLogger("")
for (handler in rootLogger.handlers) {
rootLogger.removeHandler(handler)
}
val layout = IdeaLogRecordFormatter()
val consoleHandler = ConsoleHandler()
consoleHandler.formatter = IdeaLogRecordFormatter(false, layout)
consoleHandler.level = Level.WARNING
rootLogger.addHandler(consoleHandler)
}
}
}
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/JpsBootstrapUtil.kt
+88 -125
View File
@@ -1,159 +1,122 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jpsBootstrap;
package org.jetbrains.jpsBootstrap
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.*;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.util.*
import java.util.concurrent.*
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info;
import static org.jetbrains.jpsBootstrap.JpsBootstrapMain.underTeamCity;
object JpsBootstrapUtil {
const val TEAMCITY_BUILD_PROPERTIES_FILE_ENV = "TEAMCITY_BUILD_PROPERTIES_FILE"
const val TEAMCITY_CONFIGURATION_PROPERTIES_SYSTEM_PROPERTY = "teamcity.configuration.properties.file"
const val JPS_RESOLUTION_RETRY_ENABLED_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.enabled"
const val JPS_RESOLUTION_RETRY_MAX_ATTEMPTS_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.max.attempts"
const val JPS_RESOLUTION_RETRY_DELAY_MS_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.delay.ms"
const val JPS_RESOLUTION_RETRY_BACKOFF_LIMIT_MS_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.backoff.limit.ms"
public final class JpsBootstrapUtil {
public static final String TEAMCITY_BUILD_PROPERTIES_FILE_ENV = "TEAMCITY_BUILD_PROPERTIES_FILE";
public static final String TEAMCITY_CONFIGURATION_PROPERTIES_SYSTEM_PROPERTY = "teamcity.configuration.properties.file";
public static final String JPS_RESOLUTION_RETRY_ENABLED_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.enabled";
public static final String JPS_RESOLUTION_RETRY_MAX_ATTEMPTS_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.max.attempts";
public static final String JPS_RESOLUTION_RETRY_DELAY_MS_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.delay.ms";
public static final String JPS_RESOLUTION_RETRY_BACKOFF_LIMIT_MS_PROPERTY = "org.jetbrains.jps.incremental.dependencies.resolution.retry.backoff.limit.ms";
public static boolean toBooleanChecked(String s) {
switch (s) {
case "true": return true;
case "false": return false;
default:
throw new IllegalArgumentException("Could not convert '" + s + "' to boolean. Only 'true' or 'false' values are accepted");
fun String.toBooleanChecked(): Boolean {
return when (this) {
"true" -> true
"false" -> false
else -> error("Could not convert '$this' to boolean. Only 'true' or 'false' values are accepted")
}
}
public static Properties getTeamCitySystemProperties() throws IOException {
if (!underTeamCity) {
throw new IllegalStateException("Not under TeamCity");
@get:Throws(IOException::class)
val teamCitySystemProperties: Properties
get() {
check(JpsBootstrapMain.underTeamCity) { "Not under TeamCity" }
val buildPropertiesFile = System.getenv(TEAMCITY_BUILD_PROPERTIES_FILE_ENV)
check(!(buildPropertiesFile == null || buildPropertiesFile.length == 0)) { "'TEAMCITY_BUILD_PROPERTIES_FILE_ENV' env. variable is missing or empty under TeamCity build" }
val properties = Properties()
Files.newBufferedReader(Path.of(buildPropertiesFile)).use { reader -> properties.load(reader) }
return properties
}
final String buildPropertiesFile = System.getenv(TEAMCITY_BUILD_PROPERTIES_FILE_ENV);
if (buildPropertiesFile == null || buildPropertiesFile.length() == 0) {
throw new IllegalStateException("'TEAMCITY_BUILD_PROPERTIES_FILE_ENV' env. variable is missing or empty under TeamCity build");
val teamCityConfigProperties: Properties
get() {
val systemProperties = teamCitySystemProperties
val configPropertiesFile = systemProperties.getProperty(TEAMCITY_CONFIGURATION_PROPERTIES_SYSTEM_PROPERTY)
check(!(configPropertiesFile == null || configPropertiesFile.length == 0)) { "TeamCity system property '$TEAMCITY_CONFIGURATION_PROPERTIES_SYSTEM_PROPERTY' is missing under TeamCity build" }
val properties = Properties()
Files.newBufferedReader(Path.of(configPropertiesFile)).use { reader -> properties.load(reader) }
return properties
}
Properties properties = new Properties();
try (BufferedReader reader = Files.newBufferedReader(Path.of(buildPropertiesFile))) {
properties.load(reader);
}
return properties;
}
public static Properties getTeamCityConfigProperties() throws IOException {
Properties systemProperties = getTeamCitySystemProperties();
final String configPropertiesFile = systemProperties.getProperty(TEAMCITY_CONFIGURATION_PROPERTIES_SYSTEM_PROPERTY);
if (configPropertiesFile == null || configPropertiesFile.length() == 0) {
throw new IllegalStateException("TeamCity system property '" + TEAMCITY_CONFIGURATION_PROPERTIES_SYSTEM_PROPERTY + "' is missing under TeamCity build");
}
Properties properties = new Properties();
try (BufferedReader reader = Files.newBufferedReader(Path.of(configPropertiesFile))) {
properties.load(reader);
}
return properties;
}
public static String getTeamCityConfigPropertyOrThrow(String configProperty) throws IOException {
final Properties properties = getTeamCityConfigProperties();
final String value = properties.getProperty(configProperty);
if (value == null) {
throw new IllegalStateException("TeamCity config property " + configProperty + " was not found");
}
return value;
@Throws(IOException::class)
fun getTeamCityConfigPropertyOrThrow(configProperty: String): String {
val properties = teamCityConfigProperties
return properties.getProperty(configProperty)
?: throw IllegalStateException("TeamCity config property $configProperty was not found")
}
/**
* Create properties to enable artifacts resolution retries in org.jetbrains.jps.incremental.dependencies.DependencyResolvingBuilder
* if ones absent in {@code existingProperties}. Latest of {@code existingProperties} has the highest priority.
* if ones absent in `existingProperties`. Latest of `existingProperties` has the highest priority.
*
* @param existingProperties Existing properties to check whether required values already present.
* @return Properties to enable artifacts resolution retries while build.
*/
public static Properties getJpsArtifactsResolutionRetryProperties(final Properties... existingProperties) {
final Properties properties = new Properties();
final Properties existingPropertiesMerged = new Properties();
for (Properties it : existingProperties) {
existingPropertiesMerged.putAll(it);
fun getJpsArtifactsResolutionRetryProperties(vararg existingProperties: Properties?): Properties {
val properties = Properties()
val existingPropertiesMerged = Properties()
for (it in existingProperties) {
existingPropertiesMerged.putAll(it!!)
}
String enabled = existingPropertiesMerged.getProperty(JPS_RESOLUTION_RETRY_ENABLED_PROPERTY, "true");
properties.put(JPS_RESOLUTION_RETRY_ENABLED_PROPERTY, enabled);
String maxAttempts = existingPropertiesMerged.getProperty(JPS_RESOLUTION_RETRY_MAX_ATTEMPTS_PROPERTY, "3");
properties.put(JPS_RESOLUTION_RETRY_MAX_ATTEMPTS_PROPERTY, maxAttempts);
String initialDelayMs = existingPropertiesMerged.getProperty(JPS_RESOLUTION_RETRY_DELAY_MS_PROPERTY, "1000");
properties.put(JPS_RESOLUTION_RETRY_DELAY_MS_PROPERTY, initialDelayMs);
String backoffLimitMs = existingPropertiesMerged.getProperty(
val enabled = existingPropertiesMerged.getProperty(JPS_RESOLUTION_RETRY_ENABLED_PROPERTY, "true")
properties[JPS_RESOLUTION_RETRY_ENABLED_PROPERTY] = enabled
val maxAttempts = existingPropertiesMerged.getProperty(JPS_RESOLUTION_RETRY_MAX_ATTEMPTS_PROPERTY, "3")
properties[JPS_RESOLUTION_RETRY_MAX_ATTEMPTS_PROPERTY] = maxAttempts
val initialDelayMs = existingPropertiesMerged.getProperty(JPS_RESOLUTION_RETRY_DELAY_MS_PROPERTY, "1000")
properties[JPS_RESOLUTION_RETRY_DELAY_MS_PROPERTY] = initialDelayMs
val backoffLimitMs = existingPropertiesMerged.getProperty(
JPS_RESOLUTION_RETRY_BACKOFF_LIMIT_MS_PROPERTY,
Long.toString(TimeUnit.MINUTES.toMillis(5))
);
properties.put(JPS_RESOLUTION_RETRY_BACKOFF_LIMIT_MS_PROPERTY, backoffLimitMs);
return properties;
java.lang.Long.toString(TimeUnit.MINUTES.toMillis(5))
)
properties[JPS_RESOLUTION_RETRY_BACKOFF_LIMIT_MS_PROPERTY] = backoffLimitMs
return properties
}
static <T> List<T> executeTasksInParallel(List<Callable<T>> tasks) throws InterruptedException {
ExecutorService executorService = Executors.newFixedThreadPool(5);
long start = System.currentTimeMillis();
try {
info("Executing " + tasks.size() + " in parallel");
List<Future<T>> futures = executorService.invokeAll(tasks);
List<Throwable> errors = new ArrayList<>();
List<T> results = new ArrayList<>();
for (Future<T> future : futures) {
fun <T> executeTasksInParallel(tasks: List<Callable<T>>): List<T> {
val executorService = Executors.newFixedThreadPool(5)
val start = System.currentTimeMillis()
return try {
info("Executing " + tasks.size + " in parallel")
val futures = executorService.invokeAll(tasks)
val errors: MutableList<Throwable?> = ArrayList()
val results: MutableList<T> = ArrayList()
for (future in futures) {
try {
T r = future.get(10, TimeUnit.MINUTES);
results.add(r);
val r = future[10, TimeUnit.MINUTES]
results.add(r)
}
catch (ExecutionException e) {
errors.add(e.getCause());
if (errors.size() > 4) {
executorService.shutdownNow();
break;
catch (e: ExecutionException) {
errors.add(e.cause)
if (errors.size > 4) {
executorService.shutdownNow()
break
}
}
catch (TimeoutException e) {
throw new IllegalStateException("Timeout waiting for results, exiting");
catch (e: TimeoutException) {
throw IllegalStateException("Timeout waiting for results, exiting")
}
}
if (errors.size() > 0) {
RuntimeException t = new RuntimeException("Unable to execute all targets, " + errors.size() + " error(s)");
for (Throwable err : errors) {
t.addSuppressed(err);
if (errors.size > 0) {
val t = RuntimeException("Unable to execute all targets, " + errors.size + " error(s)")
for (err in errors) {
t.addSuppressed(err)
}
throw t;
throw t
}
if (results.size() != tasks.size()) {
throw new IllegalStateException("received results size != tasks size (" + results.size() + " != " + tasks.size() + ")");
}
return results;
} finally {
info("Finished all tasks in " + (System.currentTimeMillis() - start) + " ms");
if (!executorService.isShutdown()) {
executorService.shutdownNow();
check(results.size == tasks.size) { "received results size != tasks size (" + results.size + " != " + tasks.size + ")" }
results
}
finally {
info("Finished all tasks in " + (System.currentTimeMillis() - start) + " ms")
if (!executorService.isShutdown) {
executorService.shutdownNow()
}
}
}
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/JpsBuild.kt
+130 -160
View File
@@ -1,224 +1,194 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jpsBootstrap;
package org.jetbrains.jpsBootstrap
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.util.io.FileUtilRt;
import jetbrains.buildServer.messages.serviceMessages.PublishArtifacts;
import org.jetbrains.groovy.compiler.rt.GroovyRtConstants;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesConstants;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil;
import org.jetbrains.intellij.build.dependencies.DotNetPackagesCredentials;
import org.jetbrains.jps.api.CmdlineRemoteProto;
import org.jetbrains.jps.api.GlobalOptions;
import org.jetbrains.jps.build.Standalone;
import org.jetbrains.jps.incremental.MessageHandler;
import org.jetbrains.jps.incremental.groovy.JpsGroovycRunner;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.model.JpsModel;
import org.jetbrains.jps.model.JpsNamedElement;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.module.JpsModule;
import com.intellij.openapi.application.PathManager
import com.intellij.openapi.util.io.FileUtilRt
import jetbrains.buildServer.messages.serviceMessages.PublishArtifacts
import org.jetbrains.groovy.compiler.rt.GroovyRtConstants
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.dependencies.BuildDependenciesConstants
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.error
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.verbose
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.warn
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.cleanDirectory
import org.jetbrains.intellij.build.dependencies.DotNetPackagesCredentials.setupSystemCredentials
import org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.TargetTypeBuildScope
import org.jetbrains.jps.api.GlobalOptions
import org.jetbrains.jps.build.Standalone
import org.jetbrains.jps.incremental.MessageHandler
import org.jetbrains.jps.incremental.groovy.JpsGroovycRunner
import org.jetbrains.jps.incremental.messages.BuildMessage
import org.jetbrains.jps.model.JpsModel
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.module.JpsModule
import java.nio.file.Path
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.atomic.AtomicReference
import java.util.stream.Collectors
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
class JpsBuild(communityRoot: BuildDependenciesCommunityRoot, private val myModel: JpsModel?, jpsBootstrapWorkDir: Path, kotlincHome: Path?) {
private val myModuleNames: Set<String>
private val myDataStorageRoot: Path
private val myJpsLogDir: Path
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.*;
import static org.jetbrains.jpsBootstrap.JpsBootstrapMain.underTeamCity;
public final class JpsBuild {
public static final String CLASSES_FROM_JPS_BUILD_ENV_NAME = "JPS_BOOTSTRAP_CLASSES_FROM_JPS_BUILD";
private final JpsModel myModel;
private final Set<String> myModuleNames;
private final Path myDataStorageRoot;
private final Path myJpsLogDir;
public JpsBuild(BuildDependenciesCommunityRoot communityRoot, JpsModel model, Path jpsBootstrapWorkDir, Path kotlincHome) throws Exception {
myModel = model;
myModuleNames = myModel.getProject().getModules().stream().map(JpsNamedElement::getName).collect(Collectors.toUnmodifiableSet());
myDataStorageRoot = jpsBootstrapWorkDir.resolve("jps-build-data");
System.setProperty("aether.connector.resumeDownloads", "false");
System.setProperty("jps.kotlin.home", kotlincHome.toString());
init {
myModuleNames = myModel!!.project.modules.stream().map { obj: JpsModule -> obj.name }.collect(Collectors.toUnmodifiableSet())
myDataStorageRoot = jpsBootstrapWorkDir.resolve("jps-build-data")
System.setProperty("aether.connector.resumeDownloads", "false")
System.setProperty("jps.kotlin.home", kotlincHome.toString())
// Set IDEA home path to something or JPS can't instantiate ClasspathBoostrap.java for Groovy JPS
// which calls PathManager.getLibPath() (it should not)
System.setProperty(PathManager.PROPERTY_HOME_PATH, communityRoot.communityRoot.toString());
System.setProperty("kotlin.incremental.compilation", "true");
System.setProperty(GlobalOptions.COMPILE_PARALLEL_OPTION, "true");
if (underTeamCity && System.getProperty(GlobalOptions.COMPILE_PARALLEL_MAX_THREADS_OPTION) == null) {
System.setProperty(PathManager.PROPERTY_HOME_PATH, communityRoot.communityRoot.toString())
System.setProperty("kotlin.incremental.compilation", "true")
System.setProperty(GlobalOptions.COMPILE_PARALLEL_OPTION, "true")
if (JpsBootstrapMain.Companion.underTeamCity && System.getProperty(GlobalOptions.COMPILE_PARALLEL_MAX_THREADS_OPTION) == null) {
// Under TeamCity agents try to utilize all available cpu resources
int cpuCount = Integer.parseInt(JpsBootstrapUtil.getTeamCityConfigPropertyOrThrow("teamcity.agent.hardware.cpuCount"));
System.setProperty(GlobalOptions.COMPILE_PARALLEL_MAX_THREADS_OPTION, Integer.toString(cpuCount + 1));
val cpuCount = JpsBootstrapUtil.getTeamCityConfigPropertyOrThrow("teamcity.agent.hardware.cpuCount").toInt()
System.setProperty(GlobalOptions.COMPILE_PARALLEL_MAX_THREADS_OPTION, Integer.toString(cpuCount + 1))
}
System.setProperty(JpsGroovycRunner.GROOVYC_IN_PROCESS, "true");
System.setProperty(GroovyRtConstants.GROOVYC_ASM_RESOLVING_ONLY, "false");
System.setProperty(GlobalOptions.USE_DEFAULT_FILE_LOGGING_OPTION, "true");
myJpsLogDir = jpsBootstrapWorkDir.resolve("log");
System.setProperty(GlobalOptions.LOG_DIR_OPTION, myJpsLogDir.toString());
String url = "file://" + FileUtilRt.toSystemIndependentName(jpsBootstrapWorkDir.resolve("out").toString());
JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(model.getProject()).setOutputUrl(url);
info("Compilation log directory: " + System.getProperty(GlobalOptions.LOG_DIR_OPTION));
System.setProperty(JpsGroovycRunner.GROOVYC_IN_PROCESS, "true")
System.setProperty(GroovyRtConstants.GROOVYC_ASM_RESOLVING_ONLY, "false")
System.setProperty(GlobalOptions.USE_DEFAULT_FILE_LOGGING_OPTION, "true")
myJpsLogDir = jpsBootstrapWorkDir.resolve("log")
System.setProperty(GlobalOptions.LOG_DIR_OPTION, myJpsLogDir.toString())
val url = "file://" + FileUtilRt.toSystemIndependentName(jpsBootstrapWorkDir.resolve("out").toString())
JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(myModel.project).outputUrl = url
info("Compilation log directory: " + System.getProperty(GlobalOptions.LOG_DIR_OPTION))
}
public void buildModules(Set<JpsModule> modules) throws Exception {
runBuild(modules.stream().map(JpsNamedElement::getName).collect(Collectors.toSet()), false);
@Throws(Exception::class)
fun buildModules(modules: Set<JpsModule?>?) {
runBuild(modules!!.stream().map { obj: JpsModule? -> obj!!.name }.collect(Collectors.toSet()), false)
}
/**
* @see com.intellij.space.java.jps.SpaceDependencyAuthenticationDataProvider
*/
public void resolveProjectDependencies() throws Exception {
info("Resolving project dependencies...");
var spaceUsername = System.getProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME);
var spacePassword = System.getProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_PASSWORD);
@Throws(Exception::class)
fun resolveProjectDependencies() {
info("Resolving project dependencies...")
val spaceUsername = System.getProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME)
val spacePassword = System.getProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_PASSWORD)
if (spaceUsername == null || spaceUsername.isBlank() || spacePassword == null || spacePassword.isBlank()) {
if (!DotNetPackagesCredentials.setupSystemCredentials()) {
if (!setupSystemCredentials()) {
warn("Space credentials are not provided via -D" + BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME
+ " and -D" + BuildDependenciesConstants.JPS_AUTH_SPACE_PASSWORD
+ ". Private Space Maven dependencies, if not available locally, will fail to be resolved.");
+ ". Private Space Maven dependencies, if not available locally, will fail to be resolved.")
}
}
final long buildStart = System.currentTimeMillis();
List<CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.TargetTypeBuildScope> scopes = new ArrayList<>();
CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.TargetTypeBuildScope.Builder builder = CmdlineRemoteProto.Message.ControllerMessage.ParametersMessage.TargetTypeBuildScope.newBuilder();
scopes.add(builder.setTypeId("project-dependencies-resolving").setForceBuild(false).setAllTargets(true).build());
JpsMessageHandler messageHandler = new JpsMessageHandler();
if (!underTeamCity) {
val buildStart = System.currentTimeMillis()
val scopes: MutableList<TargetTypeBuildScope> = ArrayList()
val builder = TargetTypeBuildScope.newBuilder()
scopes.add(builder.setTypeId("project-dependencies-resolving").setForceBuild(false).setAllTargets(true).build())
val messageHandler = JpsMessageHandler()
if (!JpsBootstrapMain.Companion.underTeamCity) {
// Show downloading process on local run, very handy
messageHandler.setExplicitlyVerbose();
messageHandler.setExplicitlyVerbose()
}
Standalone.runBuild(
() -> myModel,
{ myModel },
myDataStorageRoot.toFile(),
messageHandler,
scopes,
false
);
info("Finished resolving project dependencies in " + (System.currentTimeMillis() - buildStart) + " ms");
messageHandler.assertNoErrors();
)
info("Finished resolving project dependencies in " + (System.currentTimeMillis() - buildStart) + " ms")
messageHandler.assertNoErrors()
}
private void runBuild(Set<String> modules, boolean rebuild) throws Exception {
final long buildStart = System.currentTimeMillis();
JpsMessageHandler messageHandler = new JpsMessageHandler();
for (String moduleName : modules) {
if (!myModuleNames.contains(moduleName)) {
throw new IllegalStateException("Module '" + moduleName + "' was not found");
}
@Throws(Exception::class)
private fun runBuild(modules: Set<String>, rebuild: Boolean) {
val buildStart = System.currentTimeMillis()
val messageHandler = JpsMessageHandler()
for (moduleName in modules) {
check(myModuleNames.contains(moduleName)) { "Module '$moduleName' was not found" }
}
Standalone.runBuild(
() -> myModel,
{ myModel },
myDataStorageRoot.toFile(),
rebuild,
modules,
false,
Collections.emptyList(),
false, emptyList(),
false,
messageHandler
);
System.out.println("Finished building '" + String.join(" ", modules) + "' in " + (System.currentTimeMillis() - buildStart) + " ms");
List<String> errors = new ArrayList<>(messageHandler.myErrors);
)
println("Finished building '" + java.lang.String.join(" ", modules) + "' in " + (System.currentTimeMillis() - buildStart) + " ms")
val errors: List<String> = ArrayList(messageHandler.myErrors)
if (!errors.isEmpty() && !rebuild) {
warn("Incremental build finished with errors. Forcing rebuild. Compilation errors:\n" + String.join("\n", errors));
BuildDependenciesUtil.cleanDirectory(myDataStorageRoot);
runBuild(modules, true);
warn("""
Incremental build finished with errors. Forcing rebuild. Compilation errors:
${java.lang.String.join("\n", errors)}
""".trimIndent())
cleanDirectory(myDataStorageRoot)
runBuild(modules, true)
}
else {
messageHandler.assertNoErrors();
messageHandler.assertNoErrors()
}
}
private class JpsMessageHandler implements MessageHandler {
private boolean myExplicitlyVerbose;
private final List<String> myErrors = new CopyOnWriteArrayList<>();
private final AtomicReference<String> myLastMessage = new AtomicReference<>();
public void setExplicitlyVerbose() {
myExplicitlyVerbose = true;
private inner class JpsMessageHandler : MessageHandler {
private var myExplicitlyVerbose = false
val myErrors: MutableList<String> = CopyOnWriteArrayList()
private val myLastMessage = AtomicReference<String>()
fun setExplicitlyVerbose() {
myExplicitlyVerbose = true
}
@Override
public void processMessage(BuildMessage msg) {
BuildMessage.Kind kind = msg.getKind();
String text = msg.toString();
switch (kind) {
case PROGRESS:
case WARNING:
String lastMessage = myLastMessage.get();
if (text.equals(lastMessage)) {
override fun processMessage(msg: BuildMessage) {
val kind = msg.kind
val text = msg.toString()
when (kind) {
BuildMessage.Kind.PROGRESS, BuildMessage.Kind.WARNING -> {
val lastMessage = myLastMessage.get()
if (text == lastMessage) {
// Quick and dirty way to remove duplicate verbose messages
return;
return
}
else {
myLastMessage.set(text);
myLastMessage.set(text)
}
if (myExplicitlyVerbose) {
info(text);
info(text)
}
else {
// Warnings mean little for bootstrapping
verbose(text);
verbose(text)
}
break;
case ERROR:
case INTERNAL_BUILDER_ERROR:
// Do not log since we may call rebuild later and teamcity will fail on the first error
myErrors.add(text);
break;
default:
if (!msg.getMessageText().isBlank()) {
if (myModuleNames.contains(msg.getMessageText())) {
verbose(text);
}
else {
info(text);
}
}
BuildMessage.Kind.ERROR, BuildMessage.Kind.INTERNAL_BUILDER_ERROR -> // Do not log since we may call rebuild later and teamcity will fail on the first error
myErrors.add(text)
else -> if (!msg.messageText.isBlank()) {
if (myModuleNames.contains(msg.messageText)) {
verbose(text)
}
break;
else {
info(text)
}
}
}
}
public void assertNoErrors() {
List<String> errors = new ArrayList<>(myErrors);
fun assertNoErrors() {
val errors: List<String> = ArrayList(myErrors)
if (!errors.isEmpty()) {
System.out.println(new PublishArtifacts(myJpsLogDir + "=>jps-bootstrap-jps-logs.zip").asString());
for (String error : errors) {
error(error);
println(PublishArtifacts("$myJpsLogDir=>jps-bootstrap-jps-logs.zip").asString())
for (error in errors) {
error(error)
}
throw new IllegalStateException("Build finished with errors. See TC artifacts for build log. First error:\n" + errors.get(0));
throw IllegalStateException("""
Build finished with errors. See TC artifacts for build log. First error:
${errors[0]}
""".trimIndent())
}
}
}
companion object {
const val CLASSES_FROM_JPS_BUILD_ENV_NAME = "JPS_BOOTSTRAP_CLASSES_FROM_JPS_BUILD"
}
}
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/JpsProjectUtils.kt
+74 -97
View File
@@ -1,125 +1,102 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jpsBootstrap;
package org.jetbrains.jpsBootstrap
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.io.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.model.JpsElementFactory;
import org.jetbrains.jps.model.JpsModel;
import org.jetbrains.jps.model.java.JpsJavaDependenciesEnumerator;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.java.JpsJavaSdkType;
import org.jetbrains.jps.model.library.JpsLibrary;
import org.jetbrains.jps.model.library.JpsOrderRootType;
import org.jetbrains.jps.model.module.JpsModule;
import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService;
import org.jetbrains.jps.model.serialization.JpsPathVariablesConfiguration;
import org.jetbrains.jps.model.serialization.JpsProjectLoader;
import org.jetbrains.jps.model.serialization.library.JpsSdkTableSerializer;
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.openapi.util.text.StringUtil
import com.intellij.util.containers.ContainerUtil
import com.intellij.util.io.URLUtil
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info
import org.jetbrains.jps.model.JpsElementFactory
import org.jetbrains.jps.model.JpsModel
import org.jetbrains.jps.model.java.JpsJavaDependenciesEnumerator
import org.jetbrains.jps.model.java.JpsJavaExtensionService
import org.jetbrains.jps.model.java.JpsJavaSdkType
import org.jetbrains.jps.model.library.JpsOrderRootType
import org.jetbrains.jps.model.module.JpsModule
import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService
import org.jetbrains.jps.model.serialization.JpsProjectLoader
import org.jetbrains.jps.model.serialization.library.JpsSdkTableSerializer
import java.io.File
import java.nio.file.Path
import java.util.*
import kotlin.io.path.inputStream
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
object JpsProjectUtils {
fun loadJpsProject(projectHome: Path, jdkHome: Path, kotlincHome: Path): JpsModel {
val startTime = System.currentTimeMillis()
val m2LocalRepository = Path.of(System.getProperty("user.home"), ".m2", "repository")
val model = JpsElementFactory.getInstance().createModel()
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info;
@SuppressWarnings("SameParameterValue")
public final class JpsProjectUtils {
public static JpsModel loadJpsProject(Path projectHome, Path jdkHome, Path kotlincHome) throws Exception {
long startTime = System.currentTimeMillis();
Path m2LocalRepository = Path.of(System.getProperty("user.home"), ".m2", "repository");
JpsModel model = JpsElementFactory.getInstance().createModel();
JpsPathVariablesConfiguration pathVariablesConfiguration =
JpsModelSerializationDataService.getOrCreatePathVariablesConfiguration(model.getGlobal());
val pathVariablesConfiguration = JpsModelSerializationDataService.getOrCreatePathVariablesConfiguration(model.global)
pathVariablesConfiguration.addPathVariable(
"MAVEN_REPOSITORY", FileUtilRt.toSystemIndependentName(m2LocalRepository.toAbsolutePath().toString()));
"MAVEN_REPOSITORY", FileUtilRt.toSystemIndependentName(m2LocalRepository.toAbsolutePath().toString()))
// Required for various Kotlin compiler plugins
pathVariablesConfiguration.addPathVariable("KOTLIN_BUNDLED", kotlincHome.toString());
pathVariablesConfiguration.addPathVariable("KOTLIN_BUNDLED", kotlincHome.toString())
Map<String, String> pathVariables = JpsModelSerializationDataService.computeAllPathVariables(model.getGlobal());
JpsProjectLoader.loadProject(model.getProject(), pathVariables, projectHome);
System.out.println(
"Loaded project " + projectHome + ": " +
model.getProject().getModules().size() + " modules, " +
model.getProject().getLibraryCollection().getLibraries().size() + " libraries in " +
(System.currentTimeMillis() - startTime) + " ms");
String sdkName = "jdk-home";
addSdk(model, sdkName, jdkHome);
JpsSdkTableSerializer.setSdkReference(model.getProject().getSdkReferencesTable(), sdkName, JpsJavaSdkType.INSTANCE);
return model;
val pathVariables = JpsModelSerializationDataService.computeAllPathVariables(model.global)
JpsProjectLoader.loadProject(model.project, pathVariables, projectHome)
println(
"Loaded project $projectHome: " +
"${model.project.modules.size} modules, " +
"${model.project.libraryCollection.libraries.size} libraries " +
"in ${System.currentTimeMillis() - startTime} ms")
val sdkName = "jdk-home"
addSdk(model, sdkName, jdkHome)
JpsSdkTableSerializer.setSdkReference(model.project.sdkReferencesTable, sdkName, JpsJavaSdkType.INSTANCE)
return model
}
public static JpsModule getModuleByName(JpsModel model, String moduleName) {
return model.getProject().getModules()
.stream()
.filter(m -> moduleName.equals(m.getName()))
.findFirst().orElseThrow(() -> new IllegalStateException("Module " + moduleName + " is not found"));
fun getModuleByName(model: JpsModel, moduleName: String): JpsModule {
return model.project.modules
.firstOrNull { it.name == moduleName }
?: error("Module '$moduleName' was not found")
}
public static List<File> getModuleRuntimeClasspath(JpsModule module) {
JpsJavaDependenciesEnumerator enumerator = getModuleRuntimeClasspathEnumerator(module);
List<File> roots = new ArrayList<>(enumerator.classes().getRoots());
roots.sort(Comparator.comparing(File::toString));
for (File root : roots) {
if (!root.exists()) {
throw new IllegalStateException("Classpath element does not exist: " + root);
}
fun getModuleRuntimeClasspath(module: JpsModule): List<File> {
val enumerator = getModuleRuntimeClasspathEnumerator(module)
val roots = enumerator.classes().roots.sortedBy { it.path }
for (root in roots) {
check(root.exists()) { "Classpath element does not exist: $root" }
}
return roots;
return roots
}
@NotNull
private static JpsJavaDependenciesEnumerator getModuleRuntimeClasspathEnumerator(JpsModule module) {
private fun getModuleRuntimeClasspathEnumerator(module: JpsModule): JpsJavaDependenciesEnumerator {
return JpsJavaExtensionService
.dependencies(module)
.runtimeOnly()
.productionOnly()
.recursively()
.withoutSdk();
.withoutSdk()
}
public static Set<JpsModule> getRuntimeModulesClasspath(JpsModule module) {
JpsJavaDependenciesEnumerator enumerator = getModuleRuntimeClasspathEnumerator(module);
return enumerator.getModules();
fun getRuntimeModulesClasspath(module: JpsModule): Set<JpsModule> {
val enumerator = getModuleRuntimeClasspathEnumerator(module)
return enumerator.modules
}
private static void addSdk(JpsModel model, String sdkName, Path sdkHome) throws IOException {
info("Adding SDK '" + sdkName + "' at " + sdkHome);
JpsJavaExtensionService.getInstance().addJavaSdk(model.getGlobal(), sdkName, sdkHome.toString());
JpsLibrary additionalSdk = model.getGlobal().getLibraryCollection().findLibrary(sdkName);
if (additionalSdk == null) {
throw new IllegalStateException("SDK " + sdkHome + " was not found");
}
for (String moduleUrl : readModulesFromReleaseFile(sdkHome)) {
additionalSdk.addRoot(moduleUrl, JpsOrderRootType.COMPILED);
private fun addSdk(model: JpsModel, sdkName: String, sdkHome: Path) {
info("Adding SDK '$sdkName' at $sdkHome")
JpsJavaExtensionService.getInstance().addJavaSdk(model.global, sdkName, sdkHome.toString())
val additionalSdk = model.global.libraryCollection.findLibrary(sdkName)
?: throw IllegalStateException("SDK $sdkHome was not found")
for (moduleUrl in readModulesFromReleaseFile(sdkHome)) {
additionalSdk.addRoot(moduleUrl, JpsOrderRootType.COMPILED)
}
}
private static List<String> readModulesFromReleaseFile(Path jdkDir) throws IOException {
Path releaseFile = jdkDir.resolve("release");
Properties p = new Properties();
try (InputStream is = Files.newInputStream(releaseFile)) {
p.load(is);
}
String jbrBaseUrl = URLUtil.JRT_PROTOCOL + URLUtil.SCHEME_SEPARATOR +
FileUtil.toSystemIndependentName(jdkDir.toFile().getAbsolutePath()) +
URLUtil.JAR_SEPARATOR;
String modules = p.getProperty("MODULES");
return ContainerUtil.map(StringUtil.split(StringUtil.unquoteString(modules), " "), s -> jbrBaseUrl + s);
private fun readModulesFromReleaseFile(jdkDir: Path): List<String> {
val releaseFile = jdkDir.resolve("release")
val p = Properties()
releaseFile.inputStream().use { p.load(it) }
val modules = p.getProperty("MODULES")
val jbrBaseUrl = URLUtil.JRT_PROTOCOL + URLUtil.SCHEME_SEPARATOR +
FileUtil.toSystemIndependentName(jdkDir.toFile().absolutePath) +
URLUtil.JAR_SEPARATOR
return ContainerUtil.map(StringUtil.split(StringUtil.unquoteString(modules), " ")) { s: String -> jbrBaseUrl + s }
}
}
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/KotlinCompiler.kt
+24 -29
View File
@@ -1,38 +1,33 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jpsBootstrap;
package org.jetbrains.jpsBootstrap
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.downloadFileToCacheLocation
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.extractFileToCacheLocation
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.getUriForMavenArtifact
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.verbose
import java.nio.file.Path
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Objects;
object KotlinCompiler {
private const val KOTLIN_IDE_MAVEN_REPOSITORY_URL =
"https://cache-redirector.jetbrains.com/maven.pkg.jetbrains.space/kotlin/p/kotlin/kotlin-ide-plugin-dependencies"
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.verbose;
public final class KotlinCompiler {
private static final String KOTLIN_IDE_MAVEN_REPOSITORY_URL = "https://cache-redirector.jetbrains.com/maven.pkg.jetbrains.space/kotlin/p/kotlin/kotlin-ide-plugin-dependencies";
public static Path downloadAndExtractKotlinCompiler(BuildDependenciesCommunityRoot communityRoot) throws Exception {
fun downloadAndExtractKotlinCompiler(communityRoot: BuildDependenciesCommunityRoot): Path {
// We already have kotlin JPS in the classpath, fetch version from it
String kotlincVersion;
try (InputStream inputStream = KotlinCompiler.class.getClassLoader().getResourceAsStream("META-INF/compiler.version")) {
kotlincVersion = new String(Objects.requireNonNull(inputStream).readAllBytes(), StandardCharsets.UTF_8);
}
val kotlincVersion = javaClass.classLoader.getResourceAsStream("META-INF/compiler.version")
.use { inputStream -> inputStream!!.readAllBytes().decodeToString() }
info("Kotlin compiler version is $kotlincVersion")
info("Kotlin compiler version is " + kotlincVersion);
URI kotlincUrl = BuildDependenciesDownloader.getUriForMavenArtifact(
val kotlincUrl = getUriForMavenArtifact(
KOTLIN_IDE_MAVEN_REPOSITORY_URL,
"org.jetbrains.kotlin", "kotlin-dist-for-ide", kotlincVersion, "jar");
Path kotlincDist = BuildDependenciesDownloader.downloadFileToCacheLocation(communityRoot, kotlincUrl);
Path kotlinc = BuildDependenciesDownloader.extractFileToCacheLocation(communityRoot, kotlincDist);
verbose("Kotlin compiler is at " + kotlinc);
return kotlinc;
"org.jetbrains.kotlin",
"kotlin-dist-for-ide",
kotlincVersion,
"jar")
val kotlincDist = downloadFileToCacheLocation(communityRoot, kotlincUrl)
val kotlinc = extractFileToCacheLocation(communityRoot, kotlincDist)
verbose("Kotlin compiler is at $kotlinc")
return kotlinc
}
}