MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-01-05 01:50:56 +07:00
Code Issues Packages Projects Releases Wiki Activity

jps-bootstrap: convert buildScripts.downloader to Kotlin

Browse Source 
GitOrigin-RevId: c61793c70af072ca89e7df67562d6d2b673e7e6c
This commit is contained in:
Leonid Shalupov
2023-01-22 20:10:16 +01:00
committed by intellij-monorepo-bot
parent ae7a3b04c9
commit 3911397732
23 changed files with 1031 additions and 1323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
build/deps/src/org/jetbrains/intellij/build/impl/BundledMavenDownloader.kt
View File

@@ -7,6 +7,8 @@ import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.security.MessageDigest
import kotlin.io.path.listDirectoryEntries
import kotlin.io.path.readBytes
object BundledMavenDownloader {
private val mavenCommonLibs: List<String> = listOf(
@@ -21,7 +23,7 @@ object BundledMavenDownloader {
@JvmStatic
fun main(args: Array<String>) {
val communityRoot = BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory()
val communityRoot = BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory
val distRoot = downloadMavenDistribution(communityRoot)
val commonLibs = downloadMavenCommonLibs(communityRoot)
@@ -29,16 +31,11 @@ object BundledMavenDownloader {
println("Maven common libs at $commonLibs")
}
private fun fileChecksum(path: Path): String? {
return try {
val md5 = MessageDigest.getInstance("MD5")
md5.update(Files.readAllBytes(path))
val digest = md5.digest()
BigInteger(1, digest).toString(32)
}
catch (e: Exception) {
null
}
private fun fileChecksum(path: Path): String {
val md5 = MessageDigest.getInstance("MD5")
md5.update(path.readBytes())
val digest = md5.digest()
return BigInteger(1, digest).toString(32)
}
fun downloadMavenCommonLibs(communityRoot: BuildDependenciesCommunityRoot): Path {
@@ -60,13 +57,12 @@ object BundledMavenDownloader {
targetToSourceFiles[targetFile] = sourceFile
}
Files.list(root).use { stream ->
stream.forEach { file: Path? ->
if (!targetToSourceFiles.containsKey(file)) {
BuildDependenciesUtil.deleteFileOrFolder(file)
}
root.listDirectoryEntries().forEach { file ->
if (!targetToSourceFiles.containsKey(file)) {
BuildDependenciesUtil.deleteFileOrFolder(file)
}
}
synchronized(this) {
for (targetFile in targetToSourceFiles.keys) {
val sourceFile = targetToSourceFiles[targetFile]!!
@@ -76,7 +72,7 @@ object BundledMavenDownloader {
else {
val sourceCheckSum = fileChecksum(sourceFile)
val targetCheckSum = fileChecksum(targetFile)
if (!sourceCheckSum.equals(targetCheckSum)) {
if (sourceCheckSum != targetCheckSum) {
Files.copy(sourceFile, targetFile, StandardCopyOption.REPLACE_EXISTING)
}
}

4
build/launch/src/com/intellij/tools/launch/PathsProvider.kt
View File

@@ -1,9 +1,9 @@
package com.intellij.tools.launch
import java.io.File
import com.intellij.openapi.util.SystemInfo
import com.intellij.util.SystemProperties
import org.jetbrains.intellij.build.dependencies.TeamCityHelper
import java.io.File
interface PathsProvider {
val productId: String
@@ -12,7 +12,7 @@ interface PathsProvider {
val outputRootFolder: File
val tempFolder: File
get() = TeamCityHelper.getTempDirectory()?.toFile() ?: sourcesRootFolder.resolve("out").resolve("tmp")
get() = TeamCityHelper.tempDirectory?.toFile() ?: sourcesRootFolder.resolve("out").resolve("tmp")
val launcherFolder: File
get() = tempFolder.resolve("launcher").resolve(productId)

36
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesCommunityRoot.kt
View File

@@ -1,37 +1,27 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import java.nio.file.Files;
import java.nio.file.Path;
import org.jetbrains.annotations.ApiStatus
import java.nio.file.Files
import java.nio.file.Path
/**
* Special wrapper not to mix community root with other parameters
*/
@ApiStatus.Internal
public final class BuildDependenciesCommunityRoot {
@NotNull
private final Path communityRoot;
class BuildDependenciesCommunityRoot(communityRoot: Path) {
@JvmField
val communityRoot: Path
public @NotNull Path getCommunityRoot() {
return communityRoot;
init {
val probeFile = communityRoot.resolve("intellij.idea.community.main.iml")
check(!Files.notExists(probeFile)) { "community root was not found at $communityRoot" }
this.communityRoot = communityRoot
}
public BuildDependenciesCommunityRoot(@NotNull Path communityRoot) {
Path probeFile = communityRoot.resolve("intellij.idea.community.main.iml");
if (Files.notExists(probeFile)) {
throw new IllegalStateException("community root was not found at " + communityRoot);
}
this.communityRoot = communityRoot;
}
@Override
public String toString() {
override fun toString(): String {
return "BuildDependenciesCommunityRoot{" +
"communityRoot=" + communityRoot +
'}';
'}'
}
}

14
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesConstants.kt
View File

@@ -1,12 +1,12 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.ApiStatus
@ApiStatus.Internal
public final class BuildDependenciesConstants {
public static final String INTELLIJ_DEPENDENCIES_URL = "https://cache-redirector.jetbrains.com/intellij-dependencies";
public static final String MAVEN_CENTRAL_URL = "https://cache-redirector.jetbrains.com/repo.maven.apache.org/maven2";
public static final String JPS_AUTH_SPACE_USERNAME = "jps.auth.spaceUsername";
public static final String JPS_AUTH_SPACE_PASSWORD = "jps.auth.spacePassword";
object BuildDependenciesConstants {
const val INTELLIJ_DEPENDENCIES_URL = "https://cache-redirector.jetbrains.com/intellij-dependencies"
const val MAVEN_CENTRAL_URL = "https://cache-redirector.jetbrains.com/repo.maven.apache.org/maven2"
const val JPS_AUTH_SPACE_USERNAME = "jps.auth.spaceUsername"
const val JPS_AUTH_SPACE_PASSWORD = "jps.auth.spacePassword"
}

476
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesDownloader.kt
View File

@@ -1,365 +1,313 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.Striped;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.api.trace.TracerProvider;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.TestOnly;
import org.jetbrains.intellij.build.KtorKt;
import com.github.luben.zstd.ZstdInputStreamNoFinalizer
import com.google.common.hash.Hashing
import com.google.common.util.concurrent.Striped
import io.opentelemetry.api.trace.Tracer
import io.opentelemetry.api.trace.TracerProvider
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.TestOnly
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.cleanDirectory
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.extractTarBz2
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.extractTarGz
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.extractZip
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.listDirectory
import org.jetbrains.intellij.build.dependencies.TeamCityHelper.isUnderTeamCity
import org.jetbrains.intellij.build.dependencies.TeamCityHelper.systemProperties
import org.jetbrains.intellij.build.downloadFileToCacheLocationSync
import java.io.IOException
import java.io.PrintWriter
import java.io.StringWriter
import java.math.BigInteger
import java.net.URI
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.channels.FileChannel
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.LinkOption
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.attribute.FileTime
import java.time.Instant
import java.util.*
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
import java.util.logging.Logger
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.math.BigInteger;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@SuppressWarnings({"SSBasedInspection", "UnstableApiUsage"})
@ApiStatus.Internal
public final class BuildDependenciesDownloader {
private static final Logger LOG = Logger.getLogger(BuildDependenciesDownloader.class.getName());
private static final Striped<Lock> fileLocks = Striped.lock(1024);
private static final AtomicBoolean cleanupFlag = new AtomicBoolean(false);
object BuildDependenciesDownloader {
private val LOG = Logger.getLogger(BuildDependenciesDownloader::class.java.name)
private val fileLocks = Striped.lock(1024)
private val cleanupFlag = AtomicBoolean(false)
// increment on semantic changes in extract code to invalidate all current caches
private static final int EXTRACT_CODE_VERSION = 3;
private const val EXTRACT_CODE_VERSION = 4
// increment on semantic changes in download code to invalidate all current caches
// e.g. when some issues in extraction code were fixed
private static final int DOWNLOAD_CODE_VERSION = 2;
private const val DOWNLOAD_CODE_VERSION = 3
/**
* Set tracer to get telemetry. e.g. it's set for build scripts to get opentelemetry events
*/
@SuppressWarnings("StaticNonFinalField")
public static volatile @NotNull Tracer TRACER = TracerProvider.noop().get("noop-build-dependencies");
@Volatile
var TRACER: Tracer = TracerProvider.noop()["noop-build-dependencies"]
public static DependenciesProperties getDependenciesProperties(BuildDependenciesCommunityRoot communityRoot) {
try {
return new DependenciesProperties(communityRoot);
}
catch (IOException e) {
throw new RuntimeException(e);
}
fun getDependenciesProperties(communityRoot: BuildDependenciesCommunityRoot): DependenciesProperties =
DependenciesProperties(communityRoot)
@JvmStatic
fun getUriForMavenArtifact(mavenRepository: String, groupId: String, artifactId: String, version: String, packaging: String): URI {
return getUriForMavenArtifact(mavenRepository, groupId, artifactId, version, null, packaging)
}
public static URI getUriForMavenArtifact(String mavenRepository, String groupId, String artifactId, String version, String packaging) {
return getUriForMavenArtifact(mavenRepository, groupId, artifactId, version, null, packaging);
}
public static URI getUriForMavenArtifact(String mavenRepository,
String groupId,
String artifactId,
String version,
String classifier,
String packaging) {
String result = mavenRepository;
@JvmStatic
fun getUriForMavenArtifact(mavenRepository: String,
groupId: String,
artifactId: String,
version: String,
classifier: String?,
packaging: String): URI {
var result = mavenRepository
if (!result.endsWith("/")) {
result += "/";
result += "/"
}
result += groupId.replace('.', '/') + "/" + artifactId + "/" + version + "/" + artifactId + "-" + version +
(classifier != null ? ("-" + classifier) : "") +
"." + packaging;
return URI.create(result);
(if (classifier != null) "-$classifier" else "") +
"." + packaging
return URI.create(result)
}
private static Path getProjectLocalDownloadCache(BuildDependenciesCommunityRoot communityRoot) {
Path projectLocalDownloadCache = communityRoot.getCommunityRoot().resolve("build").resolve("download");
private fun getProjectLocalDownloadCache(communityRoot: BuildDependenciesCommunityRoot): Path {
val projectLocalDownloadCache = communityRoot.communityRoot.resolve("build").resolve("download")
try {
Files.createDirectories(projectLocalDownloadCache);
Files.createDirectories(projectLocalDownloadCache)
}
catch (IOException e) {
throw new RuntimeException(e);
catch (e: IOException) {
throw RuntimeException(e)
}
return projectLocalDownloadCache;
return projectLocalDownloadCache
}
private static Path getDownloadCachePath(BuildDependenciesCommunityRoot communityRoot) throws IOException {
Path path;
if (TeamCityHelper.isUnderTeamCity) {
String persistentCachePath = TeamCityHelper.getSystemProperties().get("agent.persistent.cache");
if (persistentCachePath == null || persistentCachePath.isBlank()) {
throw new IllegalStateException("'agent.persistent.cache' system property is required under TeamCity");
@Throws(IOException::class)
private fun getDownloadCachePath(communityRoot: BuildDependenciesCommunityRoot): Path {
val path: Path = if (isUnderTeamCity) {
val persistentCachePath = systemProperties["agent.persistent.cache"]
check(!persistentCachePath.isNullOrBlank()) {
"'agent.persistent.cache' system property is required under TeamCity"
}
path = Paths.get(persistentCachePath);
Paths.get(persistentCachePath)
}
else {
path = getProjectLocalDownloadCache(communityRoot);
getProjectLocalDownloadCache(communityRoot)
}
Files.createDirectories(path);
return path;
Files.createDirectories(path)
return path
}
public static Path downloadFileToCacheLocation(@NotNull BuildDependenciesCommunityRoot communityRoot, @NotNull URI uri) {
return KtorKt.downloadFileToCacheLocationSync(uri.toString(), communityRoot);
@JvmStatic
fun downloadFileToCacheLocation(communityRoot: BuildDependenciesCommunityRoot, uri: URI): Path {
return downloadFileToCacheLocationSync(uri.toString(), communityRoot)
}
public static @NotNull Path getTargetFile(@NotNull BuildDependenciesCommunityRoot communityRoot, @NotNull String uriString) throws IOException {
String lastNameFromUri = uriString.substring(uriString.lastIndexOf('/') + 1);
String fileName = hashString(uriString + "V" + DOWNLOAD_CODE_VERSION).substring(0, 10) + "-" + lastNameFromUri;
return getDownloadCachePath(communityRoot).resolve(fileName);
fun getTargetFile(communityRoot: BuildDependenciesCommunityRoot, uriString: String): Path {
val lastNameFromUri = uriString.substring(uriString.lastIndexOf('/') + 1)
val fileName = hashString(uriString + "V" + DOWNLOAD_CODE_VERSION).substring(0, 10) + "-" + lastNameFromUri
return getDownloadCachePath(communityRoot).resolve(fileName)
}
public static synchronized Path extractFileToCacheLocation(BuildDependenciesCommunityRoot communityRoot,
Path archiveFile,
BuildDependenciesExtractOptions... options) {
cleanUpIfRequired(communityRoot);
try {
Path cachePath = getDownloadCachePath(communityRoot);
String toHash = archiveFile.toString() + getExtractOptionsShortString(options);
String directoryName = archiveFile.getFileName().toString() + "." + hashString(toHash).substring(0, 6) + ".d";
Path targetDirectory = cachePath.resolve(directoryName);
Path flagFile = cachePath.resolve(directoryName + ".flag");
extractFileWithFlagFileLocation(archiveFile, targetDirectory, flagFile, options);
return targetDirectory;
@JvmStatic
@Synchronized
fun extractFileToCacheLocation(communityRoot: BuildDependenciesCommunityRoot,
archiveFile: Path,
vararg options: BuildDependenciesExtractOptions): Path {
cleanUpIfRequired(communityRoot)
return try {
val cachePath = getDownloadCachePath(communityRoot)
val toHash = archiveFile.toString() + getExtractOptionsShortString(options)
val directoryName = archiveFile.fileName.toString() + "." + hashString(toHash).substring(0, 6) + ".d"
val targetDirectory = cachePath.resolve(directoryName)
val flagFile = cachePath.resolve("$directoryName.flag")
extractFileWithFlagFileLocation(archiveFile, targetDirectory, flagFile, options)
targetDirectory
}
catch (RuntimeException e) {
throw e;
catch (e: RuntimeException) {
throw e
}
catch (Exception e) {
throw new RuntimeException(e);
catch (e: Exception) {
throw RuntimeException(e)
}
}
private static @NotNull String hashString(@NotNull String s) {
return new BigInteger(1, Hashing.sha256().hashString(s, StandardCharsets.UTF_8).asBytes()).toString(36);
private fun hashString(s: String): String {
return BigInteger(1, Hashing.sha256().hashString(s, StandardCharsets.UTF_8).asBytes()).toString(36)
}
private static byte[] getExpectedFlagFileContent(Path archiveFile, Path targetDirectory, BuildDependenciesExtractOptions[] options)
throws IOException {
long numberOfTopLevelEntries;
try (Stream<Path> stream = Files.list(targetDirectory)) {
numberOfTopLevelEntries = stream.count();
}
return (EXTRACT_CODE_VERSION + "\n" + archiveFile.toRealPath(LinkOption.NOFOLLOW_LINKS) + "\n" +
"topLevelEntries:" + numberOfTopLevelEntries + "\n" +
"options:" + getExtractOptionsShortString(options) + "\n").getBytes(StandardCharsets.UTF_8);
private fun getExpectedFlagFileContent(archiveFile: Path,
targetDirectory: Path,
options: Array<out BuildDependenciesExtractOptions>): ByteArray {
var numberOfTopLevelEntries: Long
Files.list(targetDirectory).use { stream -> numberOfTopLevelEntries = stream.count() }
return """$EXTRACT_CODE_VERSION
${archiveFile.toRealPath(LinkOption.NOFOLLOW_LINKS)}
topLevelEntries:$numberOfTopLevelEntries
options:${getExtractOptionsShortString(options)}
""".toByteArray(StandardCharsets.UTF_8)
}
private static boolean checkFlagFile(Path archiveFile, Path flagFile, Path targetDirectory, BuildDependenciesExtractOptions[] options)
throws IOException {
private fun checkFlagFile(archiveFile: Path,
flagFile: Path,
targetDirectory: Path,
options: Array<out BuildDependenciesExtractOptions>): Boolean {
if (!Files.isRegularFile(flagFile) || !Files.isDirectory(targetDirectory)) {
return false;
return false
}
byte[] existingContent = Files.readAllBytes(flagFile);
return Arrays.equals(existingContent, getExpectedFlagFileContent(archiveFile, targetDirectory, options));
val existingContent = Files.readAllBytes(flagFile)
return Arrays.equals(existingContent, getExpectedFlagFileContent(archiveFile, targetDirectory, options))
}
// assumes file at `archiveFile` is immutable
private static void extractFileWithFlagFileLocation(Path archiveFile,
Path targetDirectory,
Path flagFile,
BuildDependenciesExtractOptions[] options)
throws Exception {
private fun extractFileWithFlagFileLocation(archiveFile: Path,
targetDirectory: Path,
flagFile: Path,
options: Array<out BuildDependenciesExtractOptions>) {
if (checkFlagFile(archiveFile, flagFile, targetDirectory, options)) {
LOG.fine("Skipping extract to " + targetDirectory + " since flag file " + flagFile + " is correct");
LOG.fine("Skipping extract to $targetDirectory since flag file $flagFile is correct")
// Update file modification time to maintain FIFO caches i.e.
// in persistent cache folder on TeamCity agent
FileTime now = FileTime.from(Instant.now());
Files.setLastModifiedTime(targetDirectory, now);
Files.setLastModifiedTime(flagFile, now);
return;
val now = FileTime.from(Instant.now())
Files.setLastModifiedTime(targetDirectory, now)
Files.setLastModifiedTime(flagFile, now)
return
}
if (Files.exists(targetDirectory)) {
if (!Files.isDirectory(targetDirectory)) {
throw new IllegalStateException("Target '" + targetDirectory + "' exists, but it's not a directory. Please delete it manually");
}
BuildDependenciesUtil.cleanDirectory(targetDirectory);
check(Files.isDirectory(targetDirectory)) { "Target '$targetDirectory' exists, but it's not a directory. Please delete it manually" }
cleanDirectory(targetDirectory)
}
LOG.info(" * Extracting " + archiveFile + " to " + targetDirectory);
extractCount.incrementAndGet();
Files.createDirectories(targetDirectory);
List<Path> filesAfterCleaning = BuildDependenciesUtil.listDirectory(targetDirectory);
if (!filesAfterCleaning.isEmpty()) {
throw new IllegalStateException("Target directory " + targetDirectory + " is not empty after cleaning: " +
filesAfterCleaning.stream().map(Path::toString).collect(Collectors.joining(" ")));
LOG.info(" * Extracting $archiveFile to $targetDirectory")
extractCount.incrementAndGet()
Files.createDirectories(targetDirectory)
val filesAfterCleaning = listDirectory(targetDirectory)
check(filesAfterCleaning.isEmpty()) {
"Target directory $targetDirectory is not empty after cleaning: " +
filesAfterCleaning.joinToString(" ")
}
ByteBuffer start = ByteBuffer.allocate(4);
try (FileChannel channel = FileChannel.open(archiveFile)) {
channel.read(start, 0);
}
start.flip();
if (start.remaining() != 4) {
throw new IllegalStateException("File " + archiveFile + " is smaller than 4 bytes, could not be extracted");
}
boolean stripRoot = Arrays.stream(options).anyMatch(opt -> opt == BuildDependenciesExtractOptions.STRIP_ROOT);
int magicNumber = start.order(ByteOrder.LITTLE_ENDIAN).getInt(0);
if (magicNumber == 0xFD2FB528) {
Path unwrappedArchiveFile = archiveFile.getParent().resolve(archiveFile.getFileName() + ".unwrapped");
val start = ByteBuffer.allocate(4)
FileChannel.open(archiveFile).use { channel -> channel.read(start, 0) }
start.flip()
check(start.remaining() == 4) { "File $archiveFile is smaller than 4 bytes, could not be extracted" }
val stripRoot = options.any { it == BuildDependenciesExtractOptions.STRIP_ROOT }
val magicNumber = start.order(ByteOrder.LITTLE_ENDIAN).getInt(0)
if (magicNumber == -0x2d04ad8) {
val unwrappedArchiveFile = archiveFile.parent.resolve(archiveFile.fileName.toString() + ".unwrapped")
try {
try (OutputStream out = Files.newOutputStream(unwrappedArchiveFile)) {
try (ZstdInputStreamNoFinalizer input = new ZstdInputStreamNoFinalizer(Files.newInputStream(archiveFile))) {
input.transferTo(out);
}
Files.newOutputStream(unwrappedArchiveFile).use { out ->
ZstdInputStreamNoFinalizer(
Files.newInputStream(archiveFile)).use { input -> input.transferTo(out) }
}
BuildDependenciesUtil.extractZip(unwrappedArchiveFile, targetDirectory, stripRoot);
extractZip(unwrappedArchiveFile, targetDirectory, stripRoot)
}
finally {
Files.deleteIfExists(unwrappedArchiveFile);
Files.deleteIfExists(unwrappedArchiveFile)
}
}
else if (start.get(0) == (byte)0x50 && start.get(1) == (byte)0x4B) {
BuildDependenciesUtil.extractZip(archiveFile, targetDirectory, stripRoot);
else if (start[0] == 0x50.toByte() && start[1] == 0x4B.toByte()) {
extractZip(archiveFile, targetDirectory, stripRoot)
}
else if (start.get(0) == (byte)0x1F && start.get(1) == (byte)0x8B) {
BuildDependenciesUtil.extractTarGz(archiveFile, targetDirectory, stripRoot);
else if (start[0] == 0x1F.toByte() && start[1] == 0x8B.toByte()) {
extractTarGz(archiveFile, targetDirectory, stripRoot)
}
else if (start.get(0) == (byte)0x42 && start.get(1) == (byte)0x5A) {
BuildDependenciesUtil.extractTarBz2(archiveFile, targetDirectory, stripRoot);
else if (start[0] == 0x42.toByte() && start[1] == 0x5A.toByte()) {
extractTarBz2(archiveFile, targetDirectory, stripRoot)
}
else {
throw new IllegalStateException("Unknown archive format at " + archiveFile + "." +
" Magic number (little endian hex): " + Integer.toHexString(magicNumber) + "." +
" Currently only .tar.gz or .zip are supported");
throw IllegalStateException("Unknown archive format at " + archiveFile + "." +
" Magic number (little endian hex): " + Integer.toHexString(magicNumber) + "." +
" Currently only .tar.gz or .zip are supported")
}
Files.write(flagFile, getExpectedFlagFileContent(archiveFile, targetDirectory, options));
if (!checkFlagFile(archiveFile, flagFile, targetDirectory, options)) {
throw new IllegalStateException("checkFlagFile must be true right after extracting the archive. flagFile:" +
flagFile +
" archiveFile:" +
archiveFile +
" target:" +
targetDirectory);
Files.write(flagFile, getExpectedFlagFileContent(archiveFile, targetDirectory, options))
check(checkFlagFile(archiveFile, flagFile, targetDirectory, options)) {
"checkFlagFile must be true right after extracting the archive. flagFile:" +
flagFile +
" archiveFile:" +
archiveFile +
" target:" +
targetDirectory
}
}
public static void extractFile(Path archiveFile,
Path target,
BuildDependenciesCommunityRoot communityRoot,
BuildDependenciesExtractOptions... options) {
cleanUpIfRequired(communityRoot);
final Lock lock = fileLocks.get(target);
lock.lock();
fun extractFile(archiveFile: Path,
target: Path,
communityRoot: BuildDependenciesCommunityRoot,
vararg options: BuildDependenciesExtractOptions) {
cleanUpIfRequired(communityRoot)
val lock = fileLocks[target]
lock.lock()
try {
// Extracting different archive files into the same target should overwrite target each time
// That's why flagFile should be dependent only on target location
Path flagFile = getProjectLocalDownloadCache(communityRoot)
.resolve(hashString(target.toString()).substring(0, 6) + "-" + target.getFileName().toString() + ".flag.txt");
extractFileWithFlagFileLocation(archiveFile, target, flagFile, options);
val flagFile = getProjectLocalDownloadCache(communityRoot)
.resolve(hashString(target.toString()).substring(0, 6) + "-" + target.fileName.toString() + ".flag.txt")
extractFileWithFlagFileLocation(archiveFile, target, flagFile, options)
}
catch (Exception e) {
throw new RuntimeException(e);
catch (e: Exception) {
throw RuntimeException(e)
}
finally {
lock.unlock();
lock.unlock()
}
}
public static final class HttpStatusException extends IllegalStateException {
private final int statusCode;
private final String url;
public HttpStatusException(@NotNull String message, int statusCode, @NotNull String url) {
super(message);
this.statusCode = statusCode;
this.url = url;
}
public int getStatusCode() {
return statusCode;
}
public @NotNull String getUrl() {
return url;
}
@Override
public String toString() {
return "HttpStatusException(status=" + statusCode + ", url=" + url + ", message=" + getMessage() + ")";
}
}
public static void cleanUpIfRequired(BuildDependenciesCommunityRoot communityRoot) {
fun cleanUpIfRequired(communityRoot: BuildDependenciesCommunityRoot) {
if (!cleanupFlag.getAndSet(true)) {
// run only once per process
return;
return
}
if (TeamCityHelper.isUnderTeamCity) {
if (isUnderTeamCity) {
// Cleanup on TeamCity is handled by TeamCity
return;
return
}
Path cacheDir = getProjectLocalDownloadCache(communityRoot);
val cacheDir = getProjectLocalDownloadCache(communityRoot)
try {
new BuildDependenciesDownloaderCleanup(cacheDir).runCleanupIfRequired();
BuildDependenciesDownloaderCleanup(cacheDir).runCleanupIfRequired()
}
catch (Throwable t) {
StringWriter writer = new StringWriter();
t.printStackTrace(new PrintWriter(writer));
LOG.warning("Cleaning up failed for the directory '" + cacheDir + "'\n" + writer);
catch (t: Throwable) {
val writer = StringWriter()
t.printStackTrace(PrintWriter(writer))
LOG.warning("Cleaning up failed for the directory '$cacheDir'\n$writer")
}
}
private static String getExtractOptionsShortString(BuildDependenciesExtractOptions[] options) {
if (options.length == 0) {
return "";
private fun getExtractOptionsShortString(options: Array<out BuildDependenciesExtractOptions>): String {
if (options.isEmpty()) {
return ""
}
StringBuilder sb = new StringBuilder();
for (BuildDependenciesExtractOptions option : options) {
if (option == BuildDependenciesExtractOptions.STRIP_ROOT) {
sb.append("s");
val sb = StringBuilder()
for (option in options) {
if (option === BuildDependenciesExtractOptions.STRIP_ROOT) {
sb.append("s")
}
else {
throw new IllegalStateException("Unhandled case: " + option);
throw IllegalStateException("Unhandled case: $option")
}
}
return sb.toString();
return sb.toString()
}
private static final AtomicInteger extractCount = new AtomicInteger();
private val extractCount = AtomicInteger()
@TestOnly
public static int getExtractCount() {
return extractCount.get();
fun getExtractCount(): Int {
return extractCount.get()
}
class HttpStatusException(message: String, private val statusCode: Int, val url: String) : IllegalStateException(message) {
override fun toString(): String {
return "HttpStatusException(status=$statusCode, url=$url, message=$message)"
}
}
}

170
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesDownloaderCleanup.kt
View File

@@ -1,139 +1,121 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import com.google.common.io.MoreFiles;
import com.google.common.io.RecursiveDeleteOption;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
import java.util.logging.Logger;
import com.google.common.io.MoreFiles
import com.google.common.io.RecursiveDeleteOption
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.listDirectory
import java.io.IOException
import java.io.PrintWriter
import java.io.StringWriter
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.time.Duration
import java.time.LocalDateTime
import java.util.*
import java.util.logging.Logger
/**
* Clean-up local download cache in two stages:
* 1) mark old files by writing near them a marker file (.marked.for.cleanup)
* 2) on the second run remove both marked and original old files.
* <p>
*
*
* Why two stage removing is required: suppose you're running some build scripts after a month of vacation.
* Older downloaded files will be marked for deletion and then some of them will be used again.
* Without the marking they would be removed and re-downloaded again, which we do not want.
*/
public final class BuildDependenciesDownloaderCleanup {
private static final Logger LOG = Logger.getLogger(BuildDependenciesDownloaderCleanup.class.getName());
class BuildDependenciesDownloaderCleanup(private val myCachesDir: Path) {
private val myLastCleanupMarkerFile: Path = myCachesDir.resolve(LAST_CLEANUP_MARKER_FILE_NAME)
private static final Duration MAXIMUM_ACCESS_TIME_AGE = Duration.ofDays(22);
private static final Duration CLEANUP_EVERY_DURATION = Duration.ofDays(1);
static final String LAST_CLEANUP_MARKER_FILE_NAME = ".last.cleanup.marker";
private static final String MARKED_FOR_CLEANUP_SUFFIX = ".marked.for.cleanup";
private final Path myCachesDir;
private final Path myLastCleanupMarkerFile;
public BuildDependenciesDownloaderCleanup(Path cachesDir) {
myCachesDir = cachesDir;
myLastCleanupMarkerFile = cachesDir.resolve(LAST_CLEANUP_MARKER_FILE_NAME);
}
public boolean runCleanupIfRequired() throws IOException {
if (!isTimeForCleanup()) {
return false;
@Throws(IOException::class)
fun runCleanupIfRequired(): Boolean {
if (!isTimeForCleanup) {
return false
}
// Update file timestamp mostly
Files.writeString(myLastCleanupMarkerFile, LocalDateTime.now().toString());
cleanupCachesDir();
return true;
Files.writeString(myLastCleanupMarkerFile, LocalDateTime.now().toString())
cleanupCachesDir()
return true
}
private boolean isTimeForCleanup() {
try {
return !Files.exists(myLastCleanupMarkerFile) ||
Files.getLastModifiedTime(myLastCleanupMarkerFile).toMillis() < System.currentTimeMillis() - CLEANUP_EVERY_DURATION.toMillis();
private val isTimeForCleanup: Boolean
get() = try {
!Files.exists(myLastCleanupMarkerFile) ||
Files.getLastModifiedTime(myLastCleanupMarkerFile).toMillis() < System.currentTimeMillis() - CLEANUP_EVERY_DURATION.toMillis()
}
catch (IOException e) {
throw new RuntimeException(e);
catch (e: IOException) {
throw RuntimeException(e)
}
}
private void cleanupCachesDir() throws IOException {
@Throws(IOException::class)
private fun cleanupCachesDir() {
if (!Files.exists(myCachesDir)) {
LOG.fine("Caches directory '" + myCachesDir + "' is missing, skipping cleanup");
return;
LOG.fine("Caches directory '$myCachesDir' is missing, skipping cleanup")
return
}
if (!Files.isDirectory(myCachesDir)) {
throw new IllegalStateException("Caches directory '" + myCachesDir + "' is not a directory");
}
Set<Path> cacheFiles = new HashSet<>(BuildDependenciesUtil.listDirectory(myCachesDir));
long maxTimeMs = MAXIMUM_ACCESS_TIME_AGE.toMillis();
long currentTime = System.currentTimeMillis();
for (Path file : cacheFiles) {
if (file.equals(myLastCleanupMarkerFile)) {
continue;
check(Files.isDirectory(myCachesDir)) { "Caches directory '$myCachesDir' is not a directory" }
val cacheFiles: Set<Path> = HashSet(listDirectory(myCachesDir))
val maxTimeMs = MAXIMUM_ACCESS_TIME_AGE.toMillis()
val currentTime = System.currentTimeMillis()
for (file in cacheFiles) {
if (file == myLastCleanupMarkerFile) {
continue
}
String fileName = file.getFileName().toString();
val fileName = file.fileName.toString()
if (fileName.endsWith(MARKED_FOR_CLEANUP_SUFFIX)) {
Path realFile = myCachesDir.resolve(fileName.substring(0, fileName.length() - MARKED_FOR_CLEANUP_SUFFIX.length()));
val realFile = myCachesDir.resolve(fileName.substring(0, fileName.length - MARKED_FOR_CLEANUP_SUFFIX.length))
if (!cacheFiles.contains(realFile)) {
LOG.info("CACHE-CLEANUP: Removing orphan marker: " + file);
Files.delete(file);
LOG.info("CACHE-CLEANUP: Removing orphan marker: $file")
Files.delete(file)
}
continue;
continue
}
Path markFile = myCachesDir.resolve(fileName + MARKED_FOR_CLEANUP_SUFFIX);
BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class);
FileTime lastAccessTime = attrs.lastModifiedTime();
val markFile = myCachesDir.resolve(fileName + MARKED_FOR_CLEANUP_SUFFIX)
val attrs = Files.readAttributes(file, BasicFileAttributes::class.java)
val lastAccessTime = attrs.lastModifiedTime()
if (lastAccessTime.toMillis() > currentTime - maxTimeMs) {
if (cacheFiles.contains(markFile)) {
// File was recently updated, un-mark it
Files.delete(markFile);
Files.delete(markFile)
}
continue;
continue
}
if (Files.exists(markFile)) {
// File is old AND already marked for cleanup - delete
LOG.info("CACHE-CLEANUP: Deleting file/directory '" + file + "': it's too old and marked for cleanup");
LOG.info(
"CACHE-CLEANUP: Deleting file/directory '$file': it's too old and marked for cleanup")
// Renaming file to a temporary name to prevent deletion of currently opened files, just in case
Path toRemove = myCachesDir.resolve(fileName + ".toRemove." + UUID.randomUUID());
val toRemove = myCachesDir.resolve(fileName + ".toRemove." + UUID.randomUUID())
try {
Files.move(file, toRemove);
MoreFiles.deleteRecursively(toRemove, RecursiveDeleteOption.ALLOW_INSECURE);
Files.move(file, toRemove)
MoreFiles.deleteRecursively(toRemove, RecursiveDeleteOption.ALLOW_INSECURE)
}
catch (Throwable t) {
StringWriter writer = new StringWriter();
t.printStackTrace(new PrintWriter(writer));
LOG.warning("Unable to delete file '" + file + "': " + t.getMessage() + "\n" + writer);
catch (t: Throwable) {
val writer = StringWriter()
t.printStackTrace(PrintWriter(writer))
LOG.warning("""
Unable to delete file '$file': ${t.message}
$writer
""".trimIndent())
}
Files.delete(markFile);
Files.delete(markFile)
}
else {
LOG.info("CACHE-CLEANUP: Marking File '" + file + "' for deletion, it'll be removed on the next cleanup run");
Files.writeString(markFile, "");
LOG.info(
"CACHE-CLEANUP: Marking File '$file' for deletion, it'll be removed on the next cleanup run")
Files.writeString(markFile, "")
}
}
}
companion object {
private val LOG = Logger.getLogger(BuildDependenciesDownloaderCleanup::class.java.name)
private val MAXIMUM_ACCESS_TIME_AGE = Duration.ofDays(22)
private val CLEANUP_EVERY_DURATION = Duration.ofDays(1)
const val LAST_CLEANUP_MARKER_FILE_NAME = ".last.cleanup.marker"
private const val MARKED_FOR_CLEANUP_SUFFIX = ".marked.for.cleanup"
}
}

8
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesExtractOptions.kt
View File

@@ -1,13 +1,13 @@
// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.ApiStatus
@ApiStatus.Internal
public enum BuildDependenciesExtractOptions {
enum class BuildDependenciesExtractOptions {
/**
* Strip leading component from file names on extraction.
* Asserts that the leading component is the same for every file
*/
STRIP_ROOT,
STRIP_ROOT
}

90
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesLogging.kt
View File

@@ -1,64 +1,70 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import jetbrains.buildServer.messages.serviceMessages.Message;
import jetbrains.buildServer.messages.serviceMessages.MessageWithAttributes;
import jetbrains.buildServer.messages.serviceMessages.ServiceMessageTypes;
import jetbrains.buildServer.messages.serviceMessages.Message
import jetbrains.buildServer.messages.serviceMessages.MessageWithAttributes
import jetbrains.buildServer.messages.serviceMessages.ServiceMessageTypes
import java.util.HashMap;
import java.util.Map;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.underTeamCity;
public final class BuildDependenciesLogging {
public static void setVerboseEnabled(boolean verboseEnabled) {
BuildDependenciesLogging.verboseEnabled = verboseEnabled;
object BuildDependenciesLogging {
@JvmStatic
fun setVerboseEnabled(verboseEnabled: Boolean) {
BuildDependenciesLogging.verboseEnabled = verboseEnabled
}
private static boolean verboseEnabled = false;
public static void warn(String message) {
if (underTeamCity) {
System.out.println(new Message(message, "WARNING", null).asString());
} else {
System.out.println(message);
private var verboseEnabled = false
@JvmStatic
fun warn(message: String?) {
if (TeamCityHelper.isUnderTeamCity) {
println(Message(message!!, "WARNING", null).asString())
}
else {
println(message)
}
}
public static void info(String message) {
if (underTeamCity) {
System.out.println(new Message(message, "NORMAL", null).asString());
} else {
System.out.println(message);
@JvmStatic
fun info(message: String?) {
if (TeamCityHelper.isUnderTeamCity) {
println(Message(message!!, "NORMAL", null).asString())
}
else {
println(message)
}
}
public static void verbose(String message) {
if (underTeamCity) {
Map<String, String> attributes = new HashMap<>();
attributes.put("text", message);
attributes.put("status", "NORMAL");
attributes.put("tc:tags", "tc:internal");
System.out.println(new MessageWithAttributes(ServiceMessageTypes.MESSAGE, attributes) {}.asString());
} else {
@JvmStatic
fun verbose(message: String) {
if (TeamCityHelper.isUnderTeamCity) {
val attributes: MutableMap<String, String> = HashMap()
attributes["text"] = message
attributes["status"] = "NORMAL"
attributes["tc:tags"] = "tc:internal"
println(object : MessageWithAttributes(ServiceMessageTypes.MESSAGE, attributes) {}.asString())
}
else {
if (verboseEnabled) {
System.out.println(message);
println(message)
}
}
}
public static void error(String message) {
if (underTeamCity) {
System.out.println(new Message(message, "ERROR", null).asString());
} else {
System.out.println("ERROR: " + message);
@JvmStatic
fun error(message: String) {
if (TeamCityHelper.isUnderTeamCity) {
println(Message(message, "ERROR", null).asString())
}
else {
println("ERROR: $message")
}
}
public static void fatal(String message) {
if (underTeamCity) {
System.out.println(new Message(message, "FAILURE", null).asString());
} else {
System.err.println("\nFATAL: " + message);
@JvmStatic
fun fatal(message: String) {
if (TeamCityHelper.isUnderTeamCity) {
println(Message(message, "FAILURE", null).asString())
}
else {
System.err.println("\nFATAL: $message")
}
}
}

47
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesManualRunOnly.kt
View File

@@ -1,36 +1,27 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import org.jetbrains.annotations.ApiStatus
import java.nio.file.Files
import java.nio.file.Paths
@ApiStatus.Internal
public final class BuildDependenciesManualRunOnly {
public static @NotNull BuildDependenciesCommunityRoot getCommunityRootFromWorkingDirectory() {
// This method assumes the current working directory is inside intellij-based product checkout root
Path workingDirectory = Paths.get(System.getProperty("user.dir"));
Path current = workingDirectory;
while (current.getParent() != null) {
for (String pathCandidate : Arrays.asList(".", "community", "ultimate/community")) {
Path probeFile = current.resolve(pathCandidate).resolve("intellij.idea.community.main.iml").normalize();
if (Files.exists(probeFile)) {
return new BuildDependenciesCommunityRoot(probeFile.getParent());
object BuildDependenciesManualRunOnly {
@JvmStatic
val communityRootFromWorkingDirectory: BuildDependenciesCommunityRoot
get() {
// This method assumes the current working directory is inside intellij-based product checkout root
val workingDirectory = Paths.get(System.getProperty("user.dir"))
var current = workingDirectory
while (current.parent != null) {
for (pathCandidate in mutableListOf(".", "community", "ultimate/community")) {
val probeFile = current.resolve(pathCandidate).resolve("intellij.idea.community.main.iml").normalize()
if (Files.exists(probeFile)) {
return BuildDependenciesCommunityRoot(probeFile.parent)
}
}
current = current.parent
}
current = current.getParent();
throw IllegalStateException("IDEA Community root was not found from current working directory $workingDirectory")
}
throw new IllegalStateException("IDEA Community root was not found from current working directory " + workingDirectory);
}
public static DependenciesProperties getDependenciesPropertiesFromWorkingDirectory() {
return BuildDependenciesDownloader.getDependenciesProperties(getCommunityRootFromWorkingDirectory());
}
}

752
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesUtil.kt
View File

@@ -1,538 +1,420 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import com.google.common.io.MoreFiles;
import com.google.common.io.RecursiveDeleteOption;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import com.google.common.io.MoreFiles
import com.google.common.io.RecursiveDeleteOption
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.archivers.zip.ZipFile
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
import org.apache.commons.io.input.CloseShieldInputStream
import org.jetbrains.annotations.ApiStatus
import org.w3c.dom.Element
import java.io.BufferedInputStream
import java.io.IOException
import java.io.InputStream
import java.nio.channels.FileChannel
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.nio.file.attribute.PosixFilePermissions
import java.util.*
import java.util.function.Function
import java.util.logging.Logger
import java.util.stream.Collectors
import javax.xml.parsers.DocumentBuilder
import javax.xml.parsers.DocumentBuilderFactory
import kotlin.io.path.listDirectoryEntries
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.*;
import java.util.function.Function;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@SuppressWarnings("UnstableApiUsage")
@ApiStatus.Internal
public final class BuildDependenciesUtil {
private static final Logger LOG = Logger.getLogger(BuildDependenciesUtil.class.getName());
object BuildDependenciesUtil {
private val LOG = Logger.getLogger(BuildDependenciesUtil::class.java.name)
private val isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains("posix")
private val octal_0111 = "111".toInt(8)
private static final boolean isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains("posix");
private static final int octal_0111 = Integer.parseInt("111", 8);
public static final boolean isWindows = System.getProperty("os.name").toLowerCase(Locale.ENGLISH).startsWith("windows");
val isWindows = System.getProperty("os.name").lowercase().startsWith("windows")
public static final boolean underTeamCity = System.getenv("TEAMCITY_VERSION") != null;
@SuppressWarnings("HttpUrlsUsage")
public static DocumentBuilder createDocumentBuilder() {
@Suppress("HttpUrlsUsage")
fun createDocumentBuilder(): DocumentBuilder {
// from https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html#jaxp-documentbuilderfactory-saxparserfactory-and-dom4j
DocumentBuilderFactory dbf = DocumentBuilderFactory.newDefaultInstance();
try {
String FEATURE;
val dbf = DocumentBuilderFactory.newDefaultInstance()
return try {
// This is the PRIMARY defense. If DTDs (doctype) are disallowed, almost all
// XML entity attacks are prevented
// Xerces 2 only - http://xerces.apache.org/xerces2-j/features.html#disallow-doctype-decl
FEATURE = "http://apache.org/xml/features/disallow-doctype-decl";
dbf.setFeature(FEATURE, true);
var feature = "http://apache.org/xml/features/disallow-doctype-decl"
dbf.setFeature(feature, true)
// If you can't completely disable DTDs, then at least do the following:
// Xerces 1 - http://xerces.apache.org/xerces-j/features.html#external-general-entities
// Xerces 2 - http://xerces.apache.org/xerces2-j/features.html#external-general-entities
// JDK7+ - http://xml.org/sax/features/external-general-entities
//This feature has to be used together with the following one, otherwise it will not protect you from XXE for sure
FEATURE = "http://xml.org/sax/features/external-general-entities";
dbf.setFeature(FEATURE, false);
feature = "http://xml.org/sax/features/external-general-entities"
dbf.setFeature(feature, false)
// Xerces 1 - http://xerces.apache.org/xerces-j/features.html#external-parameter-entities
// Xerces 2 - http://xerces.apache.org/xerces2-j/features.html#external-parameter-entities
// JDK7+ - http://xml.org/sax/features/external-parameter-entities
//This feature has to be used together with the previous one, otherwise it will not protect you from XXE for sure
FEATURE = "http://xml.org/sax/features/external-parameter-entities";
dbf.setFeature(FEATURE, false);
feature = "http://xml.org/sax/features/external-parameter-entities"
dbf.setFeature(feature, false)
// Disable external DTDs as well
FEATURE = "http://apache.org/xml/features/nonvalidating/load-external-dtd";
dbf.setFeature(FEATURE, false);
feature = "http://apache.org/xml/features/nonvalidating/load-external-dtd"
dbf.setFeature(feature, false)
// and these as well, per Timothy Morgan's 2014 paper: "XML Schema, DTD, and Entity Attacks"
dbf.setXIncludeAware(false);
dbf.setExpandEntityReferences(false);
dbf.isXIncludeAware = false
dbf.isExpandEntityReferences = false
// And, per Timothy Morgan: "If for some reason support for inline DOCTYPE is a requirement, then
// ensure the entity settings are disabled (as shown above) and beware that SSRF attacks
// (http://cwe.mitre.org/data/definitions/918.html) and denial
// of service attacks (such as a billion laughs or decompression bombs via "jar:") are a risk."
return dbf.newDocumentBuilder();
dbf.newDocumentBuilder()
}
catch (Throwable throwable) {
throw new IllegalStateException("Unable to create DOM parser", throwable);
catch (throwable: Throwable) {
throw IllegalStateException("Unable to create DOM parser", throwable)
}
}
public static List<Element> getChildElements(Element parent, String tagName) {
NodeList childNodes = parent.getChildNodes();
ArrayList<Element> result = new ArrayList<>();
for (int i = 0; i < childNodes.getLength(); i++) {
Node node = childNodes.item(i);
if (node instanceof Element) {
Element element = (Element)node;
if (tagName.equals(element.getTagName())) {
result.add(element);
fun getChildElements(parent: Element, tagName: String): List<Element> {
val childNodes = parent.childNodes
val result = ArrayList<Element>()
for (i in 0 until childNodes.length) {
val node = childNodes.item(i)
if (node is Element) {
if (tagName == node.tagName) {
result.add(node)
}
}
}
return result;
return result
}
public static Element getComponentElement(Element root, String componentName) {
//noinspection SSBasedInspection
List<Element> elements = getChildElements(root, "component")
.stream()
.filter(x -> componentName.equals(x.getAttribute("name")))
.collect(Collectors.toList());
if (elements.size() != 1) {
throw new IllegalStateException("Expected one and only one component with name '" + componentName + "'");
}
return elements.get(0);
fun getComponentElement(root: Element, componentName: String): Element {
val elements = getChildElements(root, "component")
.filter { x -> componentName == x.getAttribute("name") }
check(elements.size == 1) { "Expected one and only one component with name '$componentName'" }
return elements[0]
}
public static Element getSingleChildElement(Element parent, String tagName) {
List<Element> result = getChildElements(parent, tagName);
if (result.size() != 1) {
throw new IllegalStateException("Expected one and only one element by tag '" + tagName + "'");
}
return result.get(0);
fun getSingleChildElement(parent: Element, tagName: String): Element {
val result = getChildElements(parent, tagName)
check(result.size == 1) { "Expected one and only one element by tag '$tagName'" }
return result[0]
}
public static Element tryGetSingleChildElement(Element parent, String tagName) {
List<Element> result = getChildElements(parent, tagName);
if (result.size() == 1) {
return result.get(0);
fun tryGetSingleChildElement(parent: Element, tagName: String): Element? {
val result = getChildElements(parent, tagName)
return if (result.size == 1) {
result[0]
}
return null;
else null
}
public static String getLibraryMavenId(Path libraryXml) {
try {
DocumentBuilder documentBuilder = createDocumentBuilder();
Document document = documentBuilder.parse(libraryXml.toFile());
Element libraryElement = getSingleChildElement(document.getDocumentElement(), "library");
Element propertiesElement = getSingleChildElement(libraryElement, "properties");
String mavenId = propertiesElement.getAttribute("maven-id");
if (mavenId.isBlank()) {
throw new IllegalStateException("Invalid maven-id");
}
return mavenId;
fun getLibraryMavenId(libraryXml: Path): String {
return try {
val documentBuilder = createDocumentBuilder()
val document = documentBuilder.parse(libraryXml.toFile())
val libraryElement = getSingleChildElement(document.documentElement, "library")
val propertiesElement = getSingleChildElement(libraryElement, "properties")
val mavenId = propertiesElement.getAttribute("maven-id")
check(!mavenId.isBlank()) { "Invalid maven-id" }
mavenId
}
catch (Throwable t) {
throw new IllegalStateException("Unable to load maven-id from " + libraryXml + ": " + t.getMessage(), t);
catch (t: Throwable) {
throw IllegalStateException("Unable to load maven-id from " + libraryXml + ": " + t.message, t)
}
}
public static void extractZip(Path archiveFile, Path target, boolean stripRoot) throws Exception {
try (ZipFile zipFile = new ZipFile(FileChannel.open(archiveFile))) {
Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
genericExtract(archiveFile, new ArchiveContent() {
@Override
public @Nullable Entry getNextEntry() {
if (!entries.hasMoreElements()) {
return null;
@Throws(Exception::class)
fun extractZip(archiveFile: Path, target: Path, stripRoot: Boolean) {
ZipFile(FileChannel.open(archiveFile)).use { zipFile ->
val entries = zipFile.entries
genericExtract(archiveFile, object : ArchiveContent {
override val nextEntry: Entry?
get() {
if (!entries.hasMoreElements()) {
return null
}
val entry = entries.nextElement() ?: return null
return object : Entry {
override val type: Entry.Type
get() {
if (entry.isUnixSymlink) {
return Entry.Type.SYMLINK
}
return if (entry.isDirectory) Entry.Type.DIR else Entry.Type.FILE
}
override val name: String
get() = entry.name
override val isExecutable: Boolean
get() = entry.unixMode and octal_0111 != 0
@get:Throws(IOException::class)
override val linkTarget: String?
get() = zipFile.getUnixSymlink(entry)
@get:Throws(IOException::class)
override val inputStream: InputStream
get() = zipFile.getInputStream(entry)
}
}
ZipArchiveEntry entry = entries.nextElement();
if (entry == null) {
return null;
}
return new Entry() {
@Override
public @NotNull Type getType() {
if (entry.isUnixSymlink()) {
return Type.SYMLINK;
}
return entry.isDirectory() ? Type.DIR : Type.FILE;
}
@Override
public @NotNull String getName() {
return entry.getName();
}
@Override
public boolean isExecutable() {
return (entry.getUnixMode() & octal_0111) != 0;
}
@Override
public @Nullable String getLinkTarget() throws IOException {
return zipFile.getUnixSymlink(entry);
}
@Override
public @NotNull InputStream getInputStream() throws IOException {
return zipFile.getInputStream(entry);
}
};
}
}, target, stripRoot);
}, target, stripRoot)
}
}
static void extractTarBz2(Path archiveFile, Path target, boolean stripRoot) throws Exception {
extractTarBasedArchive(archiveFile, target, stripRoot, is -> {
@Throws(Exception::class)
fun extractTarBz2(archiveFile: Path, target: Path, stripRoot: Boolean) {
extractTarBasedArchive(archiveFile, target, stripRoot) { `is`: InputStream? ->
try {
return new BZip2CompressorInputStream(is);
return@extractTarBasedArchive BZip2CompressorInputStream(`is`)
}
catch (IOException e) {
throw new RuntimeException(e);
catch (e: IOException) {
throw RuntimeException(e)
}
});
}
public static void extractTarGz(Path archiveFile, Path target, boolean stripRoot) throws Exception {
extractTarBasedArchive(archiveFile, target, stripRoot, is -> {
try {
return new GzipCompressorInputStream(is);
}
catch (IOException e) {
throw new RuntimeException(e);
}
});
}
private static void extractTarBasedArchive(Path archiveFile, Path target, boolean stripRoot, Function<? super InputStream, ? extends InputStream> unpacker)
throws Exception {
try (TarArchiveInputStream archive = new TarArchiveInputStream(
unpacker.apply(new BufferedInputStream(Files.newInputStream(archiveFile))))) {
genericExtract(archiveFile, new ArchiveContent() {
@Override
public @Nullable Entry getNextEntry() throws IOException {
TarArchiveEntry entry = archive.getNextTarEntry();
if (entry == null) {
return null;
}
return new Entry() {
@Override
public @NotNull Type getType() {
if (entry.isSymbolicLink()) {
return Type.SYMLINK;
}
if (entry.isDirectory()) {
return Type.DIR;
}
if (entry.isFile()) {
return Type.FILE;
}
throw new IllegalStateException(archiveFile + ": unknown entry type at '" + entry.getName() + "'");
}
@Override
public @NotNull String getName() {
return entry.getName();
}
@Override
public boolean isExecutable() {
return (entry.getMode() & octal_0111) != 0;
}
@Override
public @Nullable String getLinkTarget() {
return entry.getLinkName();
}
@Override
public @NotNull InputStream getInputStream() {
return CloseShieldInputStream.wrap(archive);
}
};
}
}, target, stripRoot);
}
}
private static void genericExtract(Path archiveFile, ArchiveContent archive, Path target, boolean stripRoot) throws IOException {
@Throws(Exception::class)
fun extractTarGz(archiveFile: Path, target: Path, stripRoot: Boolean) {
extractTarBasedArchive(archiveFile, target, stripRoot) { `is`: InputStream? ->
try {
return@extractTarBasedArchive GzipCompressorInputStream(`is`)
}
catch (e: IOException) {
throw RuntimeException(e)
}
}
}
@Throws(Exception::class)
private fun extractTarBasedArchive(archiveFile: Path,
target: Path,
stripRoot: Boolean,
unpacker: Function<in InputStream, out InputStream>) {
TarArchiveInputStream(
unpacker.apply(BufferedInputStream(Files.newInputStream(archiveFile)))).use { archive ->
genericExtract(archiveFile, object : ArchiveContent {
@get:Throws(IOException::class)
override val nextEntry: Entry?
get() {
val entry = archive.nextTarEntry ?: return null
return object : Entry {
override val type: Entry.Type
get() {
if (entry.isSymbolicLink) {
return Entry.Type.SYMLINK
}
if (entry.isDirectory) {
return Entry.Type.DIR
}
if (entry.isFile) {
return Entry.Type.FILE
}
throw IllegalStateException(archiveFile.toString() + ": unknown entry type at '" + entry.name + "'")
}
override val name: String
get() = entry.name
override val isExecutable: Boolean
get() = entry.mode and octal_0111 != 0
override val linkTarget: String?
get() = entry.linkName
override val inputStream: InputStream
get() = CloseShieldInputStream.wrap(archive)
}
}
}, target, stripRoot)
}
}
private fun genericExtract(archiveFile: Path, archive: ArchiveContent, target: Path, stripRoot: Boolean) {
// avoid extra createDirectories calls
Set<Path> createdDirs = new HashSet<>();
EntryNameConverter converter = new EntryNameConverter(archiveFile, target, stripRoot);
Path canonicalTarget = target.normalize();
val createdDirs: MutableSet<Path> = HashSet()
val converter = EntryNameConverter(archiveFile, target, stripRoot)
val canonicalTarget = target.normalize()
while (true) {
Entry entry = archive.getNextEntry();
if (entry == null) {
break;
}
Entry.Type type = entry.getType();
Path entryPath = converter.getOutputPath(entry.getName(), type == Entry.Type.DIR);
if (entryPath == null) {
continue;
}
val entry: Entry = archive.nextEntry ?: break
val type: Entry.Type = entry.type
val entryPath = converter.getOutputPath(entry.name, type == Entry.Type.DIR) ?: continue
if (type == Entry.Type.DIR) {
Files.createDirectories(entryPath);
createdDirs.add(entryPath);
Files.createDirectories(entryPath)
createdDirs.add(entryPath)
}
else {
Path parent = entryPath.getParent();
val parent = entryPath.parent
if (createdDirs.add(parent)) {
Files.createDirectories(parent);
Files.createDirectories(parent)
}
if (type == Entry.Type.SYMLINK) {
Path relativeSymlinkTarget = Path.of(Objects.requireNonNull(entry.getLinkTarget()));
Path resolvedTarget = entryPath.resolveSibling(relativeSymlinkTarget).normalize();
if (!resolvedTarget.startsWith(canonicalTarget) || resolvedTarget.equals(canonicalTarget)) {
LOG.fine(archiveFile + ": skipping symlink entry '" + entry.getName() +
"' which points outside of archive extraction directory, which is forbidden.\n" +
"resolved target = " + resolvedTarget + "\n" +
"root = " + canonicalTarget + "\n");
continue;
val relativeSymlinkTarget = Path.of(entry.linkTarget!!)
val resolvedTarget = entryPath.resolveSibling(relativeSymlinkTarget).normalize()
if (!resolvedTarget.startsWith(canonicalTarget) || resolvedTarget == canonicalTarget) {
LOG.fine("""
$archiveFile: skipping symlink entry '${entry.name}' which points outside of archive extraction directory, which is forbidden.
resolved target = $resolvedTarget
root = $canonicalTarget
""".trimIndent())
continue
}
if (isWindows) {
// On Windows symlink creation is still gated by various registry keys
if (Files.isRegularFile(resolvedTarget)) {
Files.copy(resolvedTarget, entryPath, StandardCopyOption.REPLACE_EXISTING);
Files.copy(resolvedTarget, entryPath, StandardCopyOption.REPLACE_EXISTING)
}
}
else {
Files.createSymbolicLink(entryPath, relativeSymlinkTarget);
Files.createSymbolicLink(entryPath, relativeSymlinkTarget)
}
}
else if (type == Entry.Type.FILE) {
try (InputStream is = entry.getInputStream()) {
Files.copy(is, entryPath, StandardCopyOption.REPLACE_EXISTING);
}
if (isPosix && entry.isExecutable()) {
//noinspection SpellCheckingInspection
Files.setPosixFilePermissions(entryPath, PosixFilePermissions.fromString("rwxr-xr-x"));
entry.inputStream.use { fs -> Files.copy(fs, entryPath, StandardCopyOption.REPLACE_EXISTING) }
if (isPosix && entry.isExecutable) {
@Suppress("SpellCheckingInspection")
Files.setPosixFilePermissions(entryPath, PosixFilePermissions.fromString("rwxr-xr-x"))
}
}
else {
throw new IllegalStateException("Unknown entry type: " + type);
throw IllegalStateException("Unknown entry type: $type")
}
}
}
}
fun normalizeEntryName(name: String): String {
val withForwardSlashes = name.replace(backwardSlash, forwardSlash)
val trimmed = trim(withForwardSlashes, forwardSlash)
assertValidEntryName(trimmed)
return trimmed
}
private const val backwardSlash = '\\'
private const val backwardSlashString = backwardSlash.toString()
private const val forwardSlash = '/'
private const val forwardSlashString = forwardSlash.toString()
private const val doubleForwardSlashString = forwardSlashString + forwardSlashString
private fun assertValidEntryName(normalizedEntryName: String) {
check(!normalizedEntryName.isBlank()) { "Entry names should not be blank: '$normalizedEntryName'" }
check(!normalizedEntryName.contains(backwardSlashString)) { "Normalized entry names should not contain '$backwardSlashString'" }
check(!normalizedEntryName.startsWith(
forwardSlashString)) { "Normalized entry names should not start with forward slash: $normalizedEntryName" }
check(!normalizedEntryName.endsWith(
forwardSlashString)) { "Normalized entry names should not end with forward slash: $normalizedEntryName" }
check(!normalizedEntryName.contains(
doubleForwardSlashString)) { "Normalized entry name should not contain '$doubleForwardSlashString': $normalizedEntryName" }
check(!(normalizedEntryName.contains("..") &&
listOf(
*normalizedEntryName.split(
forwardSlashString.toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()).contains(
".."))) { "Invalid entry name: $normalizedEntryName" }
}
fun trim(s: String, charToTrim: Char): String {
val len = s.length
var start = 0
while (start < len && s[start] == charToTrim) start++
var end = len
while (end > 0 && start < end && s[end - 1] == charToTrim) end--
return s.substring(start, end)
}
fun deleteFileOrFolder(file: Path) {
@Suppress("UnstableApiUsage")
MoreFiles.deleteRecursively(file, RecursiveDeleteOption.ALLOW_INSECURE)
}
@JvmStatic
fun cleanDirectory(directory: Path) {
Files.createDirectories(directory)
directory.listDirectoryEntries().forEach { deleteFileOrFolder(it) }
}
@JvmStatic
fun loadPropertiesFile(file: Path?): Map<String, String> {
val result = HashMap<String, String>()
val properties = Properties()
try {
Files.newBufferedReader(file).use { reader -> properties.load(reader) }
}
catch (e: IOException) {
throw RuntimeException(e)
}
for ((key, value) in properties) {
result[key as String] = value as String
}
return result
}
fun listDirectory(directory: Path?): List<Path> {
try {
Files.list(directory).use { stream -> return stream.collect(Collectors.toList()) }
}
catch (e: IOException) {
throw RuntimeException(e)
}
}
fun directoryContentToString(directory: Path, humanReadableName: String?): String {
val contents = listDirectory(directory)
val sb = StringBuilder()
sb.append("Directory contents of ")
sb.append(directory.toAbsolutePath())
sb.append(" (")
sb.append(humanReadableName)
sb.append("), ")
sb.append(contents.size)
sb.append(" entries:")
for (p in contents) {
sb.append(if (Files.isDirectory(p)) "\nD " else "\nF ")
sb.append(p.fileName.toString())
}
return sb.toString()
}
private interface ArchiveContent {
@Nullable Entry getNextEntry() throws IOException;
val nextEntry: Entry?
}
private interface Entry {
enum Type {FILE, DIR, SYMLINK}
@NotNull
Type getType();
@NotNull
String getName();
boolean isExecutable();
@Nullable
String getLinkTarget() throws IOException;
// entry should be the current entry
@NotNull InputStream getInputStream() throws IOException;
}
private static final class EntryNameConverter {
private final Path target;
private final Path archiveFile;
private final boolean stripRoot;
private String leadingComponentPrefix = null;
private EntryNameConverter(Path archiveFile, Path target, boolean stripRoot) {
this.archiveFile = archiveFile;
this.stripRoot = stripRoot;
this.target = target;
enum class Type {
FILE,
DIR,
SYMLINK
}
private @Nullable Path getOutputPath(String entryName, boolean isDirectory) {
String normalizedName = normalizeEntryName(entryName);
val type: Type
val name: String
val isExecutable: Boolean
val linkTarget: String?
val inputStream: InputStream
}
private class EntryNameConverter(private val archiveFile: Path, private val target: Path, private val stripRoot: Boolean) {
private var leadingComponentPrefix: String? = null
fun getOutputPath(entryName: String, isDirectory: Boolean): Path? {
val normalizedName = normalizeEntryName(entryName)
if (!stripRoot) {
return target.resolve(normalizedName);
return target.resolve(normalizedName)
}
if (leadingComponentPrefix == null) {
String[] split = normalizedName.split(Character.toString(forwardSlash), 2);
leadingComponentPrefix = split[0] + forwardSlash;
if (split.length < 2) {
if (!isDirectory) {
throw new IllegalStateException(archiveFile + ": first top-level entry must be a directory if strip root is enabled");
}
return null;
val split = normalizedName.split(forwardSlash.toString().toRegex(), limit = 2).toTypedArray()
leadingComponentPrefix = split[0] + forwardSlash
return if (split.size < 2) {
check(isDirectory) { "$archiveFile: first top-level entry must be a directory if strip root is enabled" }
null
}
else {
return target.resolve(split[1]);
target.resolve(split[1])
}
}
if (!normalizedName.startsWith(leadingComponentPrefix)) {
throw new IllegalStateException(
archiveFile + ": entry name '" + normalizedName + "' should start with previously found prefix '" + leadingComponentPrefix + "'");
}
return target.resolve(normalizedName.substring(leadingComponentPrefix.length()));
check(normalizedName.startsWith(
leadingComponentPrefix!!)) { "$archiveFile: entry name '$normalizedName' should start with previously found prefix '$leadingComponentPrefix'" }
return target.resolve(normalizedName.substring(leadingComponentPrefix!!.length))
}
}
static String normalizeEntryName(String name) {
String withForwardSlashes = name.replace(backwardSlash, forwardSlash);
String trimmed = trim(withForwardSlashes, forwardSlash);
assertValidEntryName(trimmed);
return trimmed;
}
private static final char backwardSlash = '\\';
private static final String backwardSlashString = Character.toString(backwardSlash);
private static final char forwardSlash = '/';
private static final String forwardSlashString = Character.toString(forwardSlash);
private static final String doubleForwardSlashString = forwardSlashString + forwardSlashString;
private static void assertValidEntryName(String normalizedEntryName) {
if (normalizedEntryName.isBlank()) {
throw new IllegalStateException("Entry names should not be blank: '" + normalizedEntryName + "'");
}
if (normalizedEntryName.contains(backwardSlashString)) {
throw new IllegalStateException("Normalized entry names should not contain '" + backwardSlashString + "'");
}
if (normalizedEntryName.startsWith(forwardSlashString)) {
throw new IllegalStateException("Normalized entry names should not start with forward slash: " + normalizedEntryName);
}
if (normalizedEntryName.endsWith(forwardSlashString)) {
throw new IllegalStateException("Normalized entry names should not end with forward slash: " + normalizedEntryName);
}
if (normalizedEntryName.contains(doubleForwardSlashString)) {
throw new IllegalStateException(
"Normalized entry name should not contain '" + doubleForwardSlashString + "': " + normalizedEntryName);
}
if (normalizedEntryName.contains("..") &&
Arrays.asList(normalizedEntryName.split(forwardSlashString)).contains("..")) {
throw new IllegalStateException("Invalid entry name: " + normalizedEntryName);
}
}
static String trim(String s, char charToTrim) {
int len = s.length();
int start = 0;
while (start < len && s.charAt(start) == charToTrim) start++;
int end = len;
while (end > 0 && start < end && s.charAt(end - 1) == charToTrim) end--;
return s.substring(start, end);
}
public static void deleteFileOrFolder(Path file) {
try {
MoreFiles.deleteRecursively(file, RecursiveDeleteOption.ALLOW_INSECURE);
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void cleanDirectory(Path directory) throws IOException {
Files.createDirectories(directory);
try (Stream<Path> stream = Files.list(directory)) {
stream.forEach(BuildDependenciesUtil::deleteFileOrFolder);
}
}
public static Map<String, String> loadPropertiesFile(Path file) {
HashMap<String, String> result = new HashMap<>();
Properties properties = new Properties();
try (BufferedReader reader = Files.newBufferedReader(file)) {
properties.load(reader);
}
catch (IOException e) {
throw new RuntimeException(e);
}
for (Map.Entry<Object, Object> entry : properties.entrySet()) {
result.put((String)entry.getKey(), (String)entry.getValue());
}
return result;
}
static List<Path> listDirectory(Path directory) {
try (Stream<Path> stream = Files.list(directory)) {
return stream.collect(Collectors.toList());
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
public static String directoryContentToString(Path directory, String humanReadableName) {
List<Path> contents = listDirectory(directory);
StringBuilder sb = new StringBuilder();
sb.append("Directory contents of ");
sb.append(directory.toAbsolutePath());
sb.append(" (");
sb.append(humanReadableName);
sb.append("), ");
sb.append(contents.size());
sb.append(" entries:");
for (Path p : contents) {
sb.append(Files.isDirectory(p) ? "\nD " : "\nF ");
sb.append(p.getFileName().toString());
}
return sb.toString();
}
}

100
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/DependenciesProperties.kt
View File

@@ -1,77 +1,55 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.jetbrains.annotations.ApiStatus
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import java.util.*
import java.util.stream.Collectors
import java.util.stream.Stream
@ApiStatus.Internal
public final class DependenciesProperties {
private final Map<String, String> dependencies = new TreeMap<>();
class DependenciesProperties(communityRoot: BuildDependenciesCommunityRoot, vararg customPropertyFiles: Path?) {
private val dependencies: MutableMap<String, String?> = TreeMap()
public DependenciesProperties(BuildDependenciesCommunityRoot communityRoot, Path... customPropertyFiles) throws IOException {
var communityPropertiesFile = communityRoot.getCommunityRoot()
init {
val communityPropertiesFile = communityRoot.communityRoot
.resolve("build")
.resolve("dependencies")
.resolve("dependencies.properties");
var ultimatePropertiesFile = communityRoot.getCommunityRoot()
.getParent()
.resolve("dependencies.properties")
val ultimatePropertiesFile = communityRoot.communityRoot
.parent
.resolve("build")
.resolve("dependencies.properties");
//noinspection SimplifyStreamApiCallChains
var propertyFiles = Stream.concat(
Stream.of(customPropertyFiles),
Stream.of(communityPropertiesFile, ultimatePropertiesFile).filter(Files::exists)
).distinct().collect(Collectors.toList());
for (Path propertyFile : propertyFiles) {
try (var file = Files.newInputStream(propertyFile)) {
var properties = new Properties();
properties.load(file);
properties.forEach((key, value) -> {
if (dependencies.containsKey(key)) {
throw new IllegalStateException("Key '" + key + "' from " + propertyFile + " is already defined");
}
dependencies.put(key.toString(), value.toString());
});
.resolve("dependencies.properties")
val propertyFiles = Stream.concat(
Stream.of(*customPropertyFiles),
Stream.of(communityPropertiesFile, ultimatePropertiesFile).filter { path: Path? -> Files.exists(path) }
).distinct().collect(Collectors.toList())
for (propertyFile in propertyFiles) {
Files.newInputStream(propertyFile).use { file ->
val properties = Properties()
properties.load(file)
properties.forEach { key: Any, value: Any ->
check(!dependencies.containsKey(key)) { "Key '$key' from $propertyFile is already defined" }
dependencies[key.toString()] = value.toString()
}
}
}
if (dependencies.isEmpty()) {
throw new IllegalStateException("No dependencies are defined");
}
check(!dependencies.isEmpty()) { "No dependencies are defined" }
}
@Override
public String toString() {
//noinspection SimplifyStreamApiCallChains,SSBasedInspection
return String.join("\n", dependencies.entrySet().stream()
.map(it -> it.getKey() + "=" + it.getValue())
.collect(Collectors.toList()));
override fun toString(): String =
dependencies.entries.joinToString("\n") { "${it.key}=${it.value}"}
fun property(name: String): String {
return dependencies[name] ?: throw IllegalArgumentException("'$name' is unknown key: $this")
}
@NotNull
public String property(String name) {
var property = dependencies.get(name);
if (property == null) {
throw new IllegalArgumentException("'" + name + "' is unknown key: " + this);
}
return property;
}
public void copy(Path copy) throws IOException {
try (var file = Files.newBufferedWriter(copy, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
file.write(toString());
}
if (!dependencies.containsKey("jdkBuild")) {
throw new IllegalStateException("'jdkBuild' key is required for backward compatibility with gradle-intellij-plugin");
}
@Throws(IOException::class)
fun copy(copy: Path?) {
Files.newBufferedWriter(copy, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING).use { file -> file.write(toString()) }
check(dependencies.containsKey("jdkBuild")) { "'jdkBuild' key is required for backward compatibility with gradle-intellij-plugin" }
}
}

133
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/DotNetPackagesCredentials.kt
View File

@@ -1,101 +1,90 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;
import org.w3c.dom.Element
import org.xml.sax.SAXException
import java.io.File
import java.io.IOException
import java.io.PrintWriter
import java.io.StringWriter
import java.nio.file.Path
import javax.xml.parsers.DocumentBuilder;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Path;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.error;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.*;
@SuppressWarnings("unused")
public final class DotNetPackagesCredentials {
public static boolean setupSystemCredentials() {
@Suppress("unused")
object DotNetPackagesCredentials {
@JvmStatic
fun setupSystemCredentials(): Boolean {
try {
if (loadFromEnvVars()) {
return true;
return true
}
if (loadFromNuGetConfig()) {
return true;
return true
}
}
catch (Throwable t) {
StringWriter writer = new StringWriter();
t.printStackTrace(new PrintWriter(writer));
error(writer.getBuffer().toString());
catch (t: Throwable) {
val writer = StringWriter()
t.printStackTrace(PrintWriter(writer))
error(writer.buffer.toString())
}
return false;
return false
}
private static boolean loadFromEnvVars() {
String credentialsFromEnvVar = System.getenv("NuGetPackageSourceCredentials_dotnet_build_space");
if (credentialsFromEnvVar == null) {
return false;
}
String[] parts = credentialsFromEnvVar.split(";");
boolean isUsernameSet = false;
boolean isPasswordSet = false;
for (String part : parts) {
String[] subParts = part.split("=");
if ("Username".equals(subParts[0])) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME, subParts[1]);
isUsernameSet = true;
private fun loadFromEnvVars(): Boolean {
val credentialsFromEnvVar = System.getenv("NuGetPackageSourceCredentials_dotnet_build_space") ?: return false
val parts = credentialsFromEnvVar.split(";".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
var isUsernameSet = false
var isPasswordSet = false
for (part in parts) {
val subParts = part.split("=".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
if ("Username" == subParts[0]) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME, subParts[1])
isUsernameSet = true
}
else if ("Password".equals(subParts[0])) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_PASSWORD, subParts[1]);
isPasswordSet = true;
else if ("Password" == subParts[0]) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_PASSWORD, subParts[1])
isPasswordSet = true
}
}
return isUsernameSet && isPasswordSet;
return isUsernameSet && isPasswordSet
}
private static boolean loadFromNuGetConfig() throws IOException, SAXException {
File nuGetConfig;
if (isWindows) {
nuGetConfig = Path.of(System.getenv("APPDATA"), "NuGet", "NuGet.Config").toFile();
@Throws(IOException::class, SAXException::class)
private fun loadFromNuGetConfig(): Boolean {
val nuGetConfig: File = if (BuildDependenciesUtil.isWindows) {
Path.of(System.getenv("APPDATA"), "NuGet", "NuGet.Config").toFile()
}
else {
nuGetConfig = Path.of(System.getProperty("user.home"), ".nuget", "NuGet", "NuGet.Config").toFile();
Path.of(System.getProperty("user.home"), ".nuget", "NuGet", "NuGet.Config").toFile()
}
if (!nuGetConfig.exists()) {
return false;
return false
}
DocumentBuilder documentBuilder = createDocumentBuilder();
Document document = documentBuilder.parse(nuGetConfig);
Element packageSourceCredentialsElement = tryGetSingleChildElement(document.getDocumentElement(), "packageSourceCredentials");
if (packageSourceCredentialsElement == null) {
return false;
}
Element dotNetSpaceBuild = tryGetSingleChildElement(packageSourceCredentialsElement, "dotnet_build_space");
if (dotNetSpaceBuild == null) {
return false;
}
boolean isUsernameSet = false;
boolean isPasswordSet = false;
for (int i = 0; i < dotNetSpaceBuild.getChildNodes().getLength(); i++) {
if (dotNetSpaceBuild.getChildNodes().item(i) instanceof Element) {
Element childElement = (Element) dotNetSpaceBuild.getChildNodes().item(i);
if ("add".equals(childElement.getTagName())) {
String key = childElement.getAttribute("key");
String value = childElement.getAttribute("value");
if ("Username".equals(key)) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME, value);
isUsernameSet = true;
val documentBuilder = BuildDependenciesUtil.createDocumentBuilder()
val document = documentBuilder.parse(nuGetConfig)
val packageSourceCredentialsElement = BuildDependenciesUtil.tryGetSingleChildElement(document.documentElement,
"packageSourceCredentials")
?: return false
val dotNetSpaceBuild = BuildDependenciesUtil.tryGetSingleChildElement(packageSourceCredentialsElement, "dotnet_build_space")
?: return false
var isUsernameSet = false
var isPasswordSet = false
for (i in 0 until dotNetSpaceBuild.childNodes.length) {
if (dotNetSpaceBuild.childNodes.item(i) is Element) {
val childElement = dotNetSpaceBuild.childNodes.item(i) as Element
if ("add" == childElement.tagName) {
val key = childElement.getAttribute("key")
val value = childElement.getAttribute("value")
if ("Username" == key) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME, value)
isUsernameSet = true
}
else if ("ClearTextPassword".equals(key)) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_PASSWORD, value);
isPasswordSet = true;
else if ("ClearTextPassword" == key) {
System.setProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_PASSWORD, value)
isPasswordSet = true
}
}
}
}
return isUsernameSet && isPasswordSet;
return isUsernameSet && isPasswordSet
}
}

180
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/JdkDownloader.kt
View File

@@ -1,140 +1,114 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Locale;
import java.util.function.Consumer;
import java.util.logging.Logger;
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import java.util.logging.Logger
/**
* Provides a reasonable stable version of JDK for current platform
* <p>
*
* JDK is used for compiling and running build scripts, compiling intellij project
* It's currently fixed here to be the same on all build agents and also in Docker images
*/
public final class JdkDownloader {
public static Path getJdkHome(BuildDependenciesCommunityRoot communityRoot, Consumer<? super String> infoLog) {
OS os = OS.getCurrent();
Arch arch = Arch.getCurrent();
return getJdkHome(communityRoot, os, arch, infoLog);
object JdkDownloader {
@JvmStatic
fun getJdkHome(communityRoot: BuildDependenciesCommunityRoot, infoLog: (String) -> Unit): Path {
val os = OS.current
val arch = Arch.current
return getJdkHome(communityRoot, os, arch, infoLog)
}
public static Path getJdkHome(BuildDependenciesCommunityRoot communityRoot) {
return getJdkHome(communityRoot, Logger.getLogger(JdkDownloader.class.getName())::info);
@JvmStatic
fun getJdkHome(communityRoot: BuildDependenciesCommunityRoot): Path {
return getJdkHome(communityRoot) { msg: String? ->
Logger.getLogger(JdkDownloader::class.java.name).info(msg)
}
}
static Path getJdkHome(BuildDependenciesCommunityRoot communityRoot, OS os, Arch arch, Consumer<? super String> infoLog) {
URI jdkUrl = getUrl(communityRoot, os, arch);
fun getJdkHome(communityRoot: BuildDependenciesCommunityRoot, os: OS, arch: Arch, infoLog: (String) -> Unit): Path {
val jdkUrl = getUrl(communityRoot, os, arch)
val jdkArchive = BuildDependenciesDownloader.downloadFileToCacheLocation(communityRoot, jdkUrl)
val jdkExtracted = BuildDependenciesDownloader.extractFileToCacheLocation(
communityRoot, jdkArchive, BuildDependenciesExtractOptions.STRIP_ROOT)
infoLog("jps-bootstrap JDK is at $jdkExtracted")
Path jdkArchive = BuildDependenciesDownloader.downloadFileToCacheLocation(communityRoot, jdkUrl);
Path jdkExtracted = BuildDependenciesDownloader.extractFileToCacheLocation(communityRoot, jdkArchive, BuildDependenciesExtractOptions.STRIP_ROOT);
infoLog.accept("jps-bootstrap JDK is at " + jdkExtracted);
Path jdkHome;
if (os == OS.MACOSX) {
jdkHome = jdkExtracted.resolve("Contents").resolve("Home");
val jdkHome: Path = if (os == OS.MACOSX) {
jdkExtracted.resolve("Contents").resolve("Home")
}
else {
jdkHome = jdkExtracted;
jdkExtracted
}
Path executable = getJavaExecutable(jdkHome);
infoLog.accept("JDK home is at " + jdkHome + ", executable at " + executable);
return jdkHome;
val executable = getJavaExecutable(jdkHome)
infoLog("JDK home is at $jdkHome, executable at $executable")
return jdkHome
}
public static Path getJavaExecutable(Path jdkHome) {
for (String candidateRelative : Arrays.asList("bin/java", "bin/java.exe")) {
Path candidate = jdkHome.resolve(candidateRelative);
fun getJavaExecutable(jdkHome: Path): Path {
for (candidateRelative in mutableListOf("bin/java", "bin/java.exe")) {
val candidate = jdkHome.resolve(candidateRelative)
if (Files.exists(candidate)) {
return candidate;
return candidate
}
}
throw new IllegalStateException("No java executables were found under " + jdkHome);
throw IllegalStateException("No java executables were found under $jdkHome")
}
// TODO: convert to enhanced switch when build level is fixed
@SuppressWarnings("EnhancedSwitchMigration")
private static URI getUrl(BuildDependenciesCommunityRoot communityRoot, OS os, Arch arch) {
String archString;
String osString;
String version;
String build;
String ext = ".tar.gz";
switch (os) {
case WINDOWS:
osString = "windows";
break;
case MACOSX:
osString = "osx";
break;
case LINUX:
osString = "linux";
break;
default:
throw new IllegalStateException("Unsupported OS: " + os);
private fun getUrl(communityRoot: BuildDependenciesCommunityRoot, os: OS, arch: Arch): URI {
val ext = ".tar.gz"
val osString: String = when (os) {
OS.WINDOWS -> "windows"
OS.MACOSX -> "osx"
OS.LINUX -> "linux"
}
val archString: String = when (arch) {
Arch.X86_64 -> "x64"
Arch.ARM64 -> "aarch64"
}
switch (arch) {
case X86_64:
archString = "x64";
break;
case ARM64:
archString = "aarch64";
break;
default:
throw new IllegalStateException("Unsupported arch: " + arch);
}
var dependenciesProperties = BuildDependenciesDownloader.getDependenciesProperties(communityRoot);
var jdkBuild = dependenciesProperties.property("jdkBuild");
var jdkBuildSplit = jdkBuild.split("b");
if (jdkBuildSplit.length != 2) {
throw new IllegalStateException("Malformed jdkBuild property: " + jdkBuild);
}
version = jdkBuildSplit[0];
build = "b" + jdkBuildSplit[1];
val dependenciesProperties = BuildDependenciesDownloader.getDependenciesProperties(communityRoot)
val jdkBuild = dependenciesProperties.property("jdkBuild")
val jdkBuildSplit = jdkBuild.split("b".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
check(jdkBuildSplit.size == 2) { "Malformed jdkBuild property: $jdkBuild" }
val version = jdkBuildSplit[0]
val build = "b" + jdkBuildSplit[1]
return URI.create("https://cache-redirector.jetbrains.com/intellij-jbr/jbrsdk-" +
version + "-" + osString + "-" +
archString + "-" + build + ext);
archString + "-" + build + ext)
}
enum OS {
WINDOWS, MACOSX, LINUX;
enum class OS {
WINDOWS,
MACOSX,
LINUX;
public static OS getCurrent() {
String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
if (osName.startsWith("mac")) {
return MACOSX;
}
else if (osName.startsWith("linux")) {
return LINUX;
}
else if (osName.startsWith("windows")) {
return WINDOWS;
}
else {
throw new IllegalStateException("Only Mac/Linux/Windows are supported now, current os: " + osName);
}
companion object {
val current: OS
get() {
val osName = System.getProperty("os.name").lowercase()
return when {
osName.startsWith("mac") -> MACOSX
osName.startsWith("linux") -> LINUX
osName.startsWith("windows") -> WINDOWS
else -> throw IllegalStateException("Only Mac/Linux/Windows are supported now, current os: $osName")
}
}
}
}
enum Arch {
X86_64, ARM64;
enum class Arch {
X86_64,
ARM64;
public static Arch getCurrent() {
String arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH);
if ("x86_64".equals(arch) || "amd64".equals(arch)) return X86_64;
if ("aarch64".equals(arch) || "arm64".equals(arch)) return ARM64;
throw new IllegalStateException("Only X86_64 and ARM64 are supported, current arch: " + arch);
companion object {
val current: Arch
get() {
val arch = System.getProperty("os.arch").lowercase()
if ("x86_64" == arch || "amd64" == arch) return X86_64
if ("aarch64" == arch || "arm64" == arch) return ARM64
throw IllegalStateException("Only X86_64 and ARM64 are supported, current arch: $arch")
}
}
}
}

145
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/TeamCityHelper.kt
View File

@@ -1,103 +1,76 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import com.google.common.base.Suppliers;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import com.google.common.base.Suppliers
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.loadPropertiesFile
import java.nio.file.Files
import java.nio.file.Path
import java.util.function.Supplier
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;
@SuppressWarnings("unused")
@Suppress("unused")
@ApiStatus.Internal
public final class TeamCityHelper {
public static final boolean isUnderTeamCity = System.getenv("TEAMCITY_VERSION") != null;
public static @Nullable Path getCheckoutDirectory() {
object TeamCityHelper {
val isUnderTeamCity = System.getenv("TEAMCITY_VERSION") != null
val checkoutDirectory: Path?
get() {
if (!isUnderTeamCity) {
return null
}
val name = "teamcity.build.checkoutDir"
val value = systemProperties[name]
if (value == null || value.isEmpty()) {
throw RuntimeException("TeamCity system property " + name + "was not found while running under TeamCity")
}
val file = Path.of(value)
if (!Files.isDirectory(file)) {
throw RuntimeException("TeamCity system property $name contains non existent directory: $file")
}
return file
}
val systemProperties: Map<String, String>
get() = systemPropertiesValue.get()
val allProperties: Map<String, String>
get() = allPropertiesValue.get()
val tempDirectory: Path?
get() {
val systemProperties = systemProperties
if (systemProperties.isEmpty()) {
return null
}
val propertyName = "teamcity.build.tempDir"
val tempPath = systemProperties[propertyName]
?: throw IllegalStateException("TeamCity must provide system property $propertyName")
return Path.of(tempPath)
}
private val systemPropertiesValue: Supplier<Map<String, String>> = Suppliers.memoize {
if (!isUnderTeamCity) {
return null;
return@memoize HashMap<String, String>()
}
String name = "teamcity.build.checkoutDir";
String value = getSystemProperties().get(name);
if (value == null || value.isEmpty()) {
throw new RuntimeException("TeamCity system property " + name + "was not found while running under TeamCity");
}
Path file = Path.of(value);
if (!Files.isDirectory(file)) {
throw new RuntimeException("TeamCity system property " + name + " contains non existent directory: " + file);
}
return file;
}
static @NotNull Map<String, String> getSystemProperties() {
return systemPropertiesValue.get();
}
public static @NotNull Map<String, String> getAllProperties() {
return allPropertiesValue.get();
}
public static @Nullable Path getTempDirectory() {
Map<String, String> systemProperties = getSystemProperties();
if (systemProperties.isEmpty()) {
return null;
}
String propertyName = "teamcity.build.tempDir";
String tempPath = systemProperties.get(propertyName);
if (tempPath == null) {
throw new IllegalStateException("TeamCity must provide system property " + propertyName);
}
return Path.of(tempPath);
}
private static final Supplier<Map<String, String>> systemPropertiesValue = Suppliers.memoize(() -> {
if (!isUnderTeamCity) {
return new HashMap<>();
}
String systemPropertiesEnvName = "TEAMCITY_BUILD_PROPERTIES_FILE";
String systemPropertiesFile = System.getenv(systemPropertiesEnvName);
val systemPropertiesEnvName = "TEAMCITY_BUILD_PROPERTIES_FILE"
val systemPropertiesFile = System.getenv(systemPropertiesEnvName)
if (systemPropertiesFile == null || systemPropertiesFile.isEmpty()) {
throw new RuntimeException("TeamCity environment variable " + systemPropertiesEnvName + "was not found while running under TeamCity");
throw RuntimeException("TeamCity environment variable " + systemPropertiesEnvName + "was not found while running under TeamCity")
}
Path file = Path.of(systemPropertiesFile);
val file = Path.of(systemPropertiesFile)
if (!Files.exists(file)) {
throw new RuntimeException("TeamCity system properties file is not found: " + file);
throw RuntimeException("TeamCity system properties file is not found: $file")
}
return BuildDependenciesUtil.loadPropertiesFile(file);
});
private static final Supplier<Map<String, String>> allPropertiesValue = Suppliers.memoize(() -> {
loadPropertiesFile(file)
}
private val allPropertiesValue: Supplier<Map<String, String>> = Suppliers.memoize {
if (!isUnderTeamCity) {
return new HashMap<>();
return@memoize HashMap<String, String>()
}
String propertyName = "teamcity.configuration.properties.file";
String value = getSystemProperties().get(propertyName);
val propertyName = "teamcity.configuration.properties.file"
val value = systemProperties[propertyName]
if (value == null || value.isEmpty()) {
throw new RuntimeException("TeamCity system property '" + propertyName + " is not found");
throw RuntimeException("TeamCity system property '$propertyName is not found")
}
Path file = Path.of(value);
val file = Path.of(value)
if (!Files.exists(file)) {
throw new RuntimeException("TeamCity configuration properties file was not found: " + file);
throw RuntimeException("TeamCity configuration properties file was not found: $file")
}
return BuildDependenciesUtil.loadPropertiesFile(file);
});
loadPropertiesFile(file)
}
}

60
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/impl/BuildScriptLauncher.kt
View File

@@ -1,55 +1,49 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.impl;
package org.jetbrains.intellij.build.impl
import jetbrains.buildServer.messages.serviceMessages.Message;
import jetbrains.buildServer.messages.serviceMessages.ServiceMessage;
import org.jetbrains.intellij.build.dependencies.TeamCityHelper;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.util.Map;
import jetbrains.buildServer.messages.serviceMessages.Message
import jetbrains.buildServer.messages.serviceMessages.ServiceMessage
import org.jetbrains.intellij.build.dependencies.TeamCityHelper
import java.io.PrintWriter
import java.io.StringWriter
import java.lang.invoke.MethodHandles
import java.lang.invoke.MethodType
import kotlin.system.exitProcess
/**
* jps-bootstrap launchers main classes via this wrapper to correctly log exceptions
* please do not add any more logic here as it won't be run if you start your target
* from IDE
*/
public final class BuildScriptLauncher {
private static final String MAIN_CLASS_PROPERTY = "build.script.launcher.main.class";
object BuildScriptLauncher {
private const val MAIN_CLASS_PROPERTY = "build.script.launcher.main.class"
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public static void main(String[] args) throws Exception {
@JvmStatic
fun main(args: Array<String>) {
try {
String mainClassName = System.getProperty(MAIN_CLASS_PROPERTY);
Class<?> mainClass = BuildScriptLauncher.class.getClassLoader().loadClass(mainClassName);
//noinspection ConfusingArgumentToVarargsMethod
val mainClassName = System.getProperty(MAIN_CLASS_PROPERTY)
val mainClass = BuildScriptLauncher::class.java.classLoader.loadClass(mainClassName)
MethodHandles.lookup()
.findStatic(mainClass, "main", MethodType.methodType(Void.TYPE, String[].class))
.invokeExact(args);
System.exit(0);
.findStatic(mainClass, "main", MethodType.methodType(Void.TYPE, Array<String>::class.java))
.invokeExact(args)
exitProcess(0)
}
catch (Throwable t) {
StringWriter sw = new StringWriter();
try (PrintWriter printWriter = new PrintWriter(sw)) {
t.printStackTrace(printWriter);
}
String message = sw.toString();
catch (t: Throwable) {
val sw = StringWriter()
PrintWriter(sw).use { printWriter -> t.printStackTrace(printWriter) }
val message = sw.toString()
if (TeamCityHelper.isUnderTeamCity) {
// Under TeamCity non-zero exit code will be displayed as a separate build error
System.out.println(new Message(message, "FAILURE", null).asString());
println(Message(message, "FAILURE", null).asString())
// Make sure it fails the build, see
// https://www.jetbrains.com/help/teamcity/service-messages.html#Reporting+Build+Problems
System.out.println(new ServiceMessage("buildProblem", Map.of("description", message)) {}.asString());
System.exit(0);
println(object : ServiceMessage("buildProblem", mapOf("description" to message)) {}.asString())
exitProcess(0)
}
else {
System.err.println("\nFATAL: " + message);
System.exit(1);
System.err.println("\nFATAL: $message")
exitProcess(1)
}
}
}

2
platform/build-scripts/src/org/jetbrains/intellij/build/impl/ToolboxLiteGen.kt
View File

@@ -46,7 +46,7 @@ internal object ToolboxLiteGen {
@JvmStatic
fun main(args: Array<String>) {
val path = downloadToolboxLiteGen(BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), "1.2.1553")
val path = downloadToolboxLiteGen(BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, "1.2.1553")
println("litegen is at $path")
}
}

5
platform/build-scripts/src/org/jetbrains/intellij/build/kotlin/KotlinCompilerDependencyDownloader.kt
View File

@@ -7,7 +7,10 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.apache.commons.io.FileUtils
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.*
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.downloadFileToCacheLocation
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.extractFileToCacheLocation
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.getTargetFile
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader.getUriForMavenArtifact
import org.jetbrains.intellij.build.suspendingRetryWithExponentialBackOff
import java.nio.file.Files
import java.nio.file.Path

49
platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/dependencies/BuildDependenciesExtractTest.kt
View File

@@ -36,6 +36,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
private val isWindows = System.getProperty("os.name").lowercase().startsWith("windows")
}
@Suppress("DEPRECATION")
@Rule
@JvmField
val thrown: ExpectedException = ExpectedException.none()
@@ -49,11 +50,11 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
val testArchive = createTestFile(archiveType, listOf(TestFile("top-level/test.txt")))
val root1 = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
Assert.assertEquals("top-level/test.txt", Files.readString(root1.resolve("top-level/test.txt")))
val root2 = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive,
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive,
BuildDependenciesExtractOptions.STRIP_ROOT)
Assert.assertEquals("top-level/test.txt", Files.readString(root2.resolve("test.txt")))
@@ -61,13 +62,13 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
assertUpToDate {
val root1_copy = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
Assert.assertEquals(root1.toString(), root1_copy.toString())
}
assertUpToDate {
val root2_copy = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive, BuildDependenciesExtractOptions.STRIP_ROOT)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive, BuildDependenciesExtractOptions.STRIP_ROOT)
Assert.assertEquals(root2.toString(), root2_copy.toString())
}
}
@@ -80,7 +81,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
))
val root = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
val symlinkFile = root.resolve("top-level/dir/test.symlink")
if (SystemInfo.isWindows) {
@@ -100,7 +101,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
))
val root = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
val symlinkFile = root.resolve("top-level/dir/test.symlink")
if (SystemInfo.isWindows) {
@@ -120,7 +121,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
))
val root = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
Assert.assertTrue(Files.getPosixFilePermissions(root.resolve("exec")).contains(PosixFilePermission.OWNER_EXECUTE))
Assert.assertFalse(Files.getPosixFilePermissions(root.resolve("no-exec")).contains(PosixFilePermission.OWNER_EXECUTE))
@@ -134,7 +135,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
))
val root = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
// will be skipped
Assert.assertFalse(root.resolve("dir/test.symlink").exists())
@@ -149,7 +150,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
))
val root = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
val target = root.resolve("dir/test.symlink/test.file")
@@ -167,7 +168,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
val testArchive = createTestFile(archiveType, listOf(TestFile("top-level1/test1.txt"), TestFile("top-level2/test2.txt")))
BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
thrown.expectMessage(object : BaseMatcher<String>() {
val prefix = "should start with previously found prefix"
@@ -183,7 +184,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
})
BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive,
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive,
BuildDependenciesExtractOptions.STRIP_ROOT)
}
@@ -191,13 +192,13 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
fun `extractFileToCacheLocation - up-to-date`() {
val testArchive = createTestFile(archiveType, listOf(TestFile("a")))
val root = BuildDependenciesDownloader.extractFileToCacheLocation(BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(),
testArchive)
val root = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
Assert.assertEquals("a", Files.readString(root.resolve("a")))
assertUpToDate {
val root2 = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
Assert.assertEquals(root.toString(), root2.toString())
}
@@ -205,7 +206,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
assertSomethingWasExtracted {
val root3 = BuildDependenciesDownloader.extractFileToCacheLocation(
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory(), testArchive)
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory, testArchive)
Assert.assertEquals(root.toString(), root3.toString())
}
}
@@ -215,20 +216,20 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
val testArchive = createTestFile(archiveType, listOf(TestFile("a"), TestFile("b")))
val extractRoot = temp.newFolder().toPath()
BuildDependenciesDownloader.extractFile(testArchive, extractRoot, BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesDownloader.extractFile(testArchive, extractRoot, BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
Assert.assertEquals("a", Files.readString(extractRoot.resolve("a")))
Assert.assertEquals("b", Files.readString(extractRoot.resolve("b")))
assertUpToDate {
BuildDependenciesDownloader.extractFile(testArchive, extractRoot,
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
}
Files.delete(extractRoot.resolve("a"))
assertSomethingWasExtracted {
BuildDependenciesDownloader.extractFile(testArchive, extractRoot,
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
}
}
@@ -237,7 +238,7 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
val testArchive = createTestFile(archiveType, listOf(TestFile("a"), TestFile("b")))
val extractRoot = temp.newFolder().toPath()
BuildDependenciesDownloader.extractFile(testArchive, extractRoot, BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesDownloader.extractFile(testArchive, extractRoot, BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
Assert.assertEquals("a", Files.readString(extractRoot.resolve("a")))
Assert.assertEquals("b", Files.readString(extractRoot.resolve("b")))
@@ -245,14 +246,14 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
val otherPresentation = testArchive.parent.resolve(".").resolve(testArchive.fileName)
Assert.assertNotEquals(testArchive, otherPresentation)
BuildDependenciesDownloader.extractFile(otherPresentation, extractRoot,
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
}
assertUpToDate {
val otherPresentation2 = testArchive.resolve("..").resolve("..").resolve(testArchive.parent.fileName).resolve(testArchive.fileName)
Assert.assertNotEquals(testArchive, otherPresentation2)
BuildDependenciesDownloader.extractFile(otherPresentation2, extractRoot,
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
}
}
@@ -261,17 +262,17 @@ class BuildDependenciesExtractTest(private val archiveType: TestArchiveType) {
val testArchive = createTestFile(archiveType, emptyList())
val extractRoot = temp.newFolder().toPath()
BuildDependenciesDownloader.extractFile(testArchive, extractRoot, BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesDownloader.extractFile(testArchive, extractRoot, BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
assertUpToDate {
BuildDependenciesDownloader.extractFile(testArchive, extractRoot,
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
}
Files.writeString(extractRoot.resolve("new"), "xx")
assertSomethingWasExtracted {
BuildDependenciesDownloader.extractFile(testArchive, extractRoot,
BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory())
BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory)
}
}

35
platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/dependencies/JdkDownloaderTest.kt
View File

@@ -1,28 +1,27 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
package org.jetbrains.intellij.build.dependencies
import org.junit.Assert;
import org.junit.Test;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesManualRunOnly.communityRootFromWorkingDirectory
import org.jetbrains.intellij.build.dependencies.JdkDownloader.getJavaExecutable
import org.jetbrains.intellij.build.dependencies.JdkDownloader.getJdkHome
import org.junit.Assert
import org.junit.Test
import java.nio.file.Files
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
public class JdkDownloaderTest {
class JdkDownloaderTest {
@Test
public void allJdkVariantsCouldBeDownloaded() {
BuildDependenciesCommunityRoot communityRoot = BuildDependenciesManualRunOnly.getCommunityRootFromWorkingDirectory();
for (JdkDownloader.OS os : JdkDownloader.OS.values()) {
for (JdkDownloader.Arch arch : Arrays.asList(JdkDownloader.Arch.X86_64, JdkDownloader.Arch.ARM64)) {
if (os == JdkDownloader.OS.WINDOWS && arch == JdkDownloader.Arch.ARM64) {
fun allJdkVariantsCouldBeDownloaded() {
val communityRoot = communityRootFromWorkingDirectory
for (os in JdkDownloader.OS.values()) {
for (arch in listOf(JdkDownloader.Arch.X86_64, JdkDownloader.Arch.ARM64)) {
if (os === JdkDownloader.OS.WINDOWS && arch === JdkDownloader.Arch.ARM64) {
// Not supported yet
continue;
continue
}
Path jdkHome = JdkDownloader.getJdkHome(communityRoot, os, arch, s -> {});
Path javaExecutable = JdkDownloader.getJavaExecutable(jdkHome);
Assert.assertTrue(Files.exists(javaExecutable));
val jdkHome = getJdkHome(communityRoot, os, arch) { }
val javaExecutable = getJavaExecutable(jdkHome)
Assert.assertTrue(Files.exists(javaExecutable))
}
}
}

10
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/JpsBootstrapMain.java
View File

@@ -17,6 +17,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot;
import org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging;
import org.jetbrains.intellij.build.dependencies.JdkDownloader;
import org.jetbrains.intellij.build.dependencies.TeamCityHelper;
import org.jetbrains.jps.incremental.storage.ProjectStamps;
import org.jetbrains.jps.model.JpsModel;
import org.jetbrains.jps.model.module.JpsModule;
@@ -34,7 +35,6 @@ import java.util.logging.Handler;
import java.util.stream.Collectors;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.*;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.underTeamCity;
import static org.jetbrains.jpsBootstrap.JpsBootstrapUtil.getJpsArtifactsResolutionRetryProperties;
import static org.jetbrains.jpsBootstrap.JpsBootstrapUtil.getTeamCitySystemProperties;
import static org.jetbrains.jpsBootstrap.JpsBootstrapUtil.toBooleanChecked;
@@ -56,6 +56,8 @@ public class JpsBootstrapMain {
private static final List<Option> ALL_OPTIONS =
Arrays.asList(OPT_HELP, OPT_VERBOSE, OPT_SYSTEM_PROPERTY, OPT_PROPERTIES_FILE, OPT_JAVA_ARGFILE_TARGET, OPT_BUILD_TARGET_XMX, OPT_ONLY_DOWNLOAD_JDK);
static final boolean underTeamCity = TeamCityHelper.INSTANCE.isUnderTeamCity();
private static Options createCliOptions() {
Options opts = new Options();
@@ -79,7 +81,7 @@ public class JpsBootstrapMain {
fatal(ExceptionUtil.getThrowableText(t));
// Better diagnostics for local users
if (!underTeamCity) {
if (!TeamCityHelper.INSTANCE.isUnderTeamCity()) {
System.err.println("\n###### ERROR EXIT due to FATAL error: " + t.getMessage() + "\n");
String work = jpsBootstrapWorkDir == null ? "PROJECT_HOME/build/jps-bootstrap-work" : jpsBootstrapWorkDir.toString();
System.err.println("###### You may try to delete caches at " + work + " and retry");
@@ -171,7 +173,7 @@ public class JpsBootstrapMain {
);
System.out.println(setParameterServiceMessage.asString());
setParameterServiceMessage = new SetParameterServiceMessage(
"jps.bootstrap.java.executable", JdkDownloader.getJavaExecutable(jdkHome).toString());
"jps.bootstrap.java.executable", JdkDownloader.INSTANCE.getJavaExecutable(jdkHome).toString());
System.out.println(setParameterServiceMessage.asString());
}
else {
@@ -219,7 +221,7 @@ public class JpsBootstrapMain {
}
private List<String> getOpenedPackages() throws Exception {
Path openedPackagesPath = communityHome.getCommunityRoot().resolve("plugins/devkit/devkit-core/src/run/OpenedPackages.txt");
Path openedPackagesPath = communityHome.communityRoot.resolve("plugins/devkit/devkit-core/src/run/OpenedPackages.txt");
List<String> openedPackages = ContainerUtil.filter(Files.readAllLines(openedPackagesPath), it -> !it.isBlank());
List<String> unknownPackages = new ArrayList<>();

2
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/JpsBootstrapUtil.java
View File

@@ -11,7 +11,7 @@ import java.util.Properties;
import java.util.concurrent.*;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.info;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.underTeamCity;
import static org.jetbrains.jpsBootstrap.JpsBootstrapMain.underTeamCity;
public final class JpsBootstrapUtil {
public static final String TEAMCITY_BUILD_PROPERTIES_FILE_ENV = "TEAMCITY_BUILD_PROPERTIES_FILE";

4
platform/jps-bootstrap/src/main/java/org/jetbrains/jpsBootstrap/JpsBuild.java
View File

@@ -30,7 +30,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesLogging.*;
import static org.jetbrains.intellij.build.dependencies.BuildDependenciesUtil.underTeamCity;
import static org.jetbrains.jpsBootstrap.JpsBootstrapMain.underTeamCity;
public final class JpsBuild {
public static final String CLASSES_FROM_JPS_BUILD_ENV_NAME = "JPS_BOOTSTRAP_CLASSES_FROM_JPS_BUILD";
@@ -50,7 +50,7 @@ public final class JpsBuild {
// Set IDEA home path to something or JPS can't instantiate ClasspathBoostrap.java for Groovy JPS
// which calls PathManager.getLibPath() (it should not)
System.setProperty(PathManager.PROPERTY_HOME_PATH, communityRoot.getCommunityRoot().toString());
System.setProperty(PathManager.PROPERTY_HOME_PATH, communityRoot.communityRoot.toString());
System.setProperty("kotlin.incremental.compilation", "true");
System.setProperty(GlobalOptions.COMPILE_PARALLEL_OPTION, "true");

2
plugins/groovy/test/org/jetbrains/plugins/groovy/compiler/GroovyCompilerTestCase.groovy
View File

@@ -115,7 +115,7 @@ abstract class GroovyCompilerTestCase extends JavaCodeInsightFixtureTestCase imp
@Override
protected void tearDown() throws Exception {
Path buildDir = BuildManager.getInstance().getBuildSystemDirectory(myFixture.getProject());
Path buildDir = BuildManager.getInstance().getBuildSystemDirectory(myFixture.getProject())
try {
EdtTestUtil.runInEdtAndWait {
try {