MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-01-08 15:09:39 +07:00
Code Issues Packages Projects Releases Wiki Activity

jps-bootstrap: common downloader code between build scripts and jps-bootstrap

Browse Source 
now with coroutines!
also drop custom telemetry stubs (use opentelemetry directly)

GitOrigin-RevId: 23e483dd04eb4b942b641bc4f8cb954b3b03b5e0
This commit is contained in:
Leonid Shalupov
2023-01-01 18:48:24 +01:00
committed by intellij-monorepo-bot
parent cf8f80f799
commit b53696ce9b
21 changed files with 153 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
build/tasks/intellij.idea.community.build.tasks.iml
View File

@@ -57,6 +57,7 @@
<orderEntry type="library" scope="TEST" name="JUnit5" level="project" />
<orderEntry type="library" scope="TEST" name="JUnit4" level="project" />
<orderEntry type="library" scope="TEST" name="commons-compress" level="project" />
<orderEntry type="module" module-name="intellij.platform.buildScripts.downloader" />
<orderEntry type="library" name="jetbrains-annotations" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.base" />
</component>

2
build/tasks/src/org/jetbrains/intellij/build/TraceManager.kt
View File

@@ -14,6 +14,7 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider
import io.opentelemetry.sdk.trace.data.SpanData
import io.opentelemetry.semconv.resource.attributes.ResourceAttributes
import kotlinx.coroutines.GlobalScope
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
@@ -37,6 +38,7 @@ object TraceManager {
.build()
tracer = openTelemetry.getTracer("build-script")
TracerProviderManager.tracerProvider = tracerProvider
BuildDependenciesDownloader.TRACER = tracer
}
@JvmStatic

7
platform/build-scripts/downloader/intellij.platform.buildScripts.downloader.iml
View File

@@ -14,5 +14,12 @@
<orderEntry type="library" name="zstd-jni" level="project" />
<orderEntry type="library" name="zstd-jni-windows-aarch64" level="project" />
<orderEntry type="library" name="commons-io" level="project" />
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="library" name="ktor-client-auth" level="project" />
<orderEntry type="library" name="ktor-client-cio" level="project" />
<orderEntry type="library" name="ktor-client-encoding" level="project" />
<orderEntry type="library" name="kotlinx-coroutines-jdk8" level="project" />
<orderEntry type="library" name="opentelemetry" level="project" />
<orderEntry type="library" name="opentelemetry-extension-kotlin" level="project" />
</component>
</module>

109
platform/build-scripts/src/org/jetbrains/intellij/build/ktor.kt → platform/build-scripts/downloader/src/ktor.kt
View File

@@ -1,8 +1,7 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("BlockingMethodInNonBlockingContext", "ReplaceGetOrSet")
package org.jetbrains.intellij.build
import com.intellij.diagnostic.telemetry.useWithScope2
import com.intellij.util.concurrency.SynchronizedClearableLazy
import io.ktor.client.HttpClient
import io.ktor.client.call.body
import io.ktor.client.engine.cio.CIO
@@ -26,12 +25,13 @@ import io.ktor.utils.io.jvm.nio.copyTo
import io.opentelemetry.api.common.AttributeKey
import io.opentelemetry.api.common.Attributes
import io.opentelemetry.api.trace.Span
import io.opentelemetry.api.trace.SpanBuilder
import io.opentelemetry.api.trace.StatusCode
import io.opentelemetry.context.Context
import io.opentelemetry.extension.kotlin.asContextElement
import kotlinx.coroutines.*
import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
import org.jetbrains.intellij.build.io.suspendingRetryWithExponentialBackOff
import org.jetbrains.xxh3.Xx3UnencodedString
import java.nio.channels.FileChannel
import java.nio.charset.StandardCharsets
import java.nio.file.Files
@@ -41,6 +41,9 @@ import java.nio.file.StandardOpenOption
import java.nio.file.attribute.FileTime
import java.time.Instant
import java.util.*
import java.util.concurrent.CancellationException
import java.util.concurrent.atomic.AtomicReference
import java.util.function.Supplier
import kotlin.time.Duration.Companion.hours
const val SPACE_REPO_HOST = "packages.jetbrains.team"
@@ -109,6 +112,81 @@ private val httpSpaceClient = SynchronizedClearableLazy {
}
}
// copy from util, do not make public
private class SynchronizedClearableLazy<T>(private val initializer: () -> T) : Lazy<T>, Supplier<T> {
private val computedValue = AtomicReference(notYetInitialized())
@Suppress("UNCHECKED_CAST")
private fun notYetInitialized(): T = NOT_YET_INITIALIZED as T
private fun nullize(t: T): T? = if (isInitialized(t)) t else null
private fun isInitialized(t: T?): Boolean = t !== NOT_YET_INITIALIZED
companion object {
private val NOT_YET_INITIALIZED = object {
override fun toString(): String = "Not yet initialized"
}
}
override fun get(): T = value
override var value: T
get() {
val currentValue = computedValue.get()
if (isInitialized(currentValue)) {
return currentValue
}
// do not call initializer in parallel
synchronized(this) {
// set under lock to ensure that initializer is not called several times
return computedValue.updateAndGet { old ->
if (isInitialized(old)) old else initializer()
}
}
}
set(value) {
computedValue.set(value)
}
override fun isInitialized() = isInitialized(computedValue.get())
override fun toString() = computedValue.toString()
fun drop(): T? = nullize(computedValue.getAndSet(notYetInitialized()))
}
// copy from util, do not make public
internal inline fun <T> Span.use(operation: (Span) -> T): T {
try {
return operation(this)
}
catch (e: CancellationException) {
throw e
}
catch (e: Throwable) {
recordException(e)
setStatus(StatusCode.ERROR)
throw e
}
finally {
end()
}
}
// copy from util, do not make public
internal suspend inline fun <T> SpanBuilder.useWithScope2(crossinline operation: suspend (Span) -> T): T {
val span = startSpan()
return withContext(Context.current().with(span).asContextElement()) {
span.use {
operation(span)
}
}
}
internal fun spanBuilder(spanName: String): SpanBuilder = BuildDependenciesDownloader.TRACER.spanBuilder(spanName)
fun closeKtorClient() {
httpClient.drop()?.close()
httpSpaceClient.drop()?.close()
@@ -132,8 +210,11 @@ suspend fun downloadAsText(url: String): String {
}
}
suspend fun downloadFileToCacheLocation(url: String, context: BuildContext): Path {
return downloadFileToCacheLocation(url, context.paths.communityHomeDirRoot)
fun downloadFileToCacheLocationSync(url: String, communityRoot: BuildDependenciesCommunityRoot): Path {
@Suppress("RAW_RUN_BLOCKING")
return runBlocking(Dispatchers.IO) {
downloadFileToCacheLocation(url, communityRoot)
}
}
suspend fun downloadFileToCacheLocation(url: String, communityRoot: BuildDependenciesCommunityRoot): Path {
@@ -141,7 +222,7 @@ suspend fun downloadFileToCacheLocation(url: String, communityRoot: BuildDepende
val target = BuildDependenciesDownloader.getTargetFile(communityRoot, url)
val targetPath = target.toString()
val lock = fileLocks.getLock(Xx3UnencodedString.hashUnencodedString(targetPath).toInt())
val lock = fileLocks.getLock(targetPath.hashCode())
lock.lock()
try {
val now = Instant.now()
@@ -246,15 +327,3 @@ internal fun CoroutineScope.writeChannel(file: Path): ByteWriteChannel {
}
}.channel
}
@Suppress("HttpUrlsUsage")
internal fun toUrlWithTrailingSlash(serverUrl: String): String {
var result = serverUrl
if (!result.startsWith("http://") && !result.startsWith("https://")) {
result = "http://$result"
}
if (!result.endsWith('/')) {
result += '/'
}
return result
}

187
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/BuildDependenciesDownloader.java
View File

@@ -4,32 +4,31 @@ package org.jetbrains.intellij.build.dependencies;
import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.Striped;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.api.trace.TracerProvider;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import org.jetbrains.intellij.build.dependencies.telemetry.BuildDependenciesNoopTracer;
import org.jetbrains.intellij.build.dependencies.telemetry.BuildDependenciesSpan;
import org.jetbrains.intellij.build.dependencies.telemetry.BuildDependenciesTraceEventAttributes;
import org.jetbrains.intellij.build.dependencies.telemetry.BuildDependenciesTracer;
import org.jetbrains.intellij.build.KtorKt;
import java.io.*;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.math.BigInteger;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
@@ -42,7 +41,6 @@ import java.util.stream.Stream;
public final class BuildDependenciesDownloader {
private static final Logger LOG = Logger.getLogger(BuildDependenciesDownloader.class.getName());
private static final String HTTP_HEADER_CONTENT_LENGTH = "Content-Length";
private static final Striped<Lock> fileLocks = Striped.lock(1024);
private static final AtomicBoolean cleanupFlag = new AtomicBoolean(false);
@@ -51,18 +49,13 @@ public final class BuildDependenciesDownloader {
// increment on semantic changes in download code to invalidate all current caches
// e.g. when some issues in extraction code were fixed
private static final int DOWNLOAD_CODE_VERSION = 1;
private static final int DOWNLOAD_CODE_VERSION = 2;
/**
* Set tracer to get telemetry. e.g. it's set for build scripts to get opentelemetry events
*/
@SuppressWarnings("StaticNonFinalField") public static @NotNull BuildDependenciesTracer TRACER = BuildDependenciesNoopTracer.INSTANCE;
// init is very expensive due to SSL initialization
private static final class HttpClientHolder {
private static final HttpClient httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER)
.version(HttpClient.Version.HTTP_1_1).build();
}
@SuppressWarnings("StaticNonFinalField")
public static volatile @NotNull Tracer TRACER = TracerProvider.noop().get("noop-build-dependencies");
public static DependenciesProperties getDependenciesProperties(BuildDependenciesCommunityRoot communityRoot) {
try {
@@ -125,24 +118,8 @@ public final class BuildDependenciesDownloader {
return path;
}
public static synchronized Path downloadFileToCacheLocation(@NotNull BuildDependenciesCommunityRoot communityRoot, @NotNull URI uri, @Nullable String bearerToken) {
cleanUpIfRequired(communityRoot);
String uriString = uri.toString();
try {
Path targetFile = getTargetFile(communityRoot, uriString);
downloadFile(uri, targetFile, bearerToken);
return targetFile;
}
catch (HttpStatusException e) {
throw e;
}
catch (Exception e) {
throw new RuntimeException("Cannot download " + uriString, e);
}
}
public static Path downloadFileToCacheLocation(@NotNull BuildDependenciesCommunityRoot communityRoot, @NotNull URI uri) {
return downloadFileToCacheLocation(communityRoot, uri, null);
return KtorKt.downloadFileToCacheLocationSync(uri.toString(), communityRoot);
}
public static @NotNull Path getTargetFile(@NotNull BuildDependenciesCommunityRoot communityRoot, @NotNull String uriString) throws IOException {
@@ -314,140 +291,6 @@ public final class BuildDependenciesDownloader {
}
}
private static void downloadFile(URI uri, Path target, String bearerToken) throws Exception {
Lock lock = fileLocks.get(target);
lock.lock();
try {
BuildDependenciesTraceEventAttributes attributes = TRACER.createAttributes();
attributes.setAttribute("uri", uri.toString());
attributes.setAttribute("target", target.toString());
BuildDependenciesSpan span = TRACER.startSpan("download", attributes);
try {
Instant now = Instant.now();
if (Files.exists(target)) {
span.addEvent("skip downloading because target file already exists", TRACER.createAttributes());
// update file modification time to maintain FIFO caches i.e. in persistent cache folder on TeamCity agent
Files.setLastModifiedTime(target, FileTime.from(now));
return;
}
// save to the same disk to ensure that move will be atomic and not as a copy
String tempFileName = target.getFileName() + "-"
+ Long.toString(now.getEpochSecond() - 1634886185, 36) + "-"
+ Integer.toString(now.getNano(), 36);
if (tempFileName.length() > 255) {
tempFileName = tempFileName.substring(tempFileName.length() - 255);
}
Path tempFile = target.getParent().resolve(tempFileName);
try {
LOG.info(" * Downloading " + uri + " -> " + target);
Retry.withExponentialBackOff(() -> {
Files.deleteIfExists(tempFile);
tryToDownloadFile(uri, tempFile, bearerToken);
});
Files.move(tempFile, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
}
finally {
Files.deleteIfExists(tempFile);
}
}
catch (Throwable e) {
span.recordException(e);
span.setStatus(BuildDependenciesSpan.SpanStatus.ERROR);
throw e;
}
finally {
span.close();
}
}
finally {
lock.unlock();
}
}
private static void tryToDownloadFile(URI uri, Path tempFile, String bearerToken) throws Exception {
HttpResponse<Path> response = getResponseFollowingRedirects(uri, tempFile, bearerToken);
int statusCode = response.statusCode();
if (statusCode != 200) {
StringBuilder builder = new StringBuilder("Cannot download\n");
Map<String, List<String>> headers = response.headers().map();
headers.keySet().stream().sorted()
.flatMap(headerName -> headers.get(headerName).stream().map(value -> "Header: " + headerName + ": " + value + "\n"))
.forEach(builder::append);
builder.append('\n');
if (Files.exists(tempFile)) {
try (InputStream inputStream = Files.newInputStream(tempFile)) {
// yes, not trying to guess encoding
// string constructor should be exception free,
// so at worse we'll get some random characters
builder.append(new String(inputStream.readNBytes(1024), StandardCharsets.UTF_8));
}
}
throw new HttpStatusException(builder.toString(), statusCode, uri.toString());
}
long contentLength = response.headers().firstValueAsLong(HTTP_HEADER_CONTENT_LENGTH).orElse(-1);
if (contentLength <= 0) {
throw new IllegalStateException("Header '" + HTTP_HEADER_CONTENT_LENGTH + "' is missing or zero for " + uri);
}
long fileSize = Files.size(tempFile);
if (fileSize != contentLength) {
throw new IllegalStateException("Wrong file length after downloading uri '" + uri +
"' to '" + tempFile +
"': expected length " + contentLength +
"from Content-Length header, but got " + fileSize + " on disk");
}
}
private static HttpResponse<Path> getResponseFollowingRedirects(URI uri, Path tempFile, String bearerToken) throws Exception {
HttpRequest request = createBuildScriptDownloaderRequest(uri, bearerToken);
HttpResponse<Path> response = HttpClientHolder.httpClient.send(request, HttpResponse.BodyHandlers.ofFile(tempFile));
String originHost = uri.getHost();
int REDIRECT_LIMIT = 10;
for (int i = 0; i < REDIRECT_LIMIT; i++) {
int statusCode = response.statusCode();
if (!(statusCode == 301 || statusCode == 302 || statusCode == 307 || statusCode == 308)) {
return response;
}
Optional<String> locationHeader = response.headers().firstValue("Location");
if (locationHeader.isEmpty()) {
locationHeader = response.headers().firstValue("location");
if (locationHeader.isEmpty()) {
return response;
}
}
URI newUri = new URI(locationHeader.get());
request = newUri.getHost().equals(originHost)
? createBuildScriptDownloaderRequest(newUri, bearerToken)
: createBuildScriptDownloaderRequest(newUri, null);
response = HttpClientHolder.httpClient.send(request, HttpResponse.BodyHandlers.ofFile(tempFile));
}
return response;
}
private static HttpRequest createBuildScriptDownloaderRequest(URI uri, String bearerToken) {
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder()
.GET()
.uri(uri)
.setHeader("User-Agent", "Build Script Downloader");
if (bearerToken != null) {
requestBuilder = requestBuilder.setHeader("Authorization", "Bearer " + bearerToken);
}
return requestBuilder.build();
}
public static final class HttpStatusException extends IllegalStateException {
private final int statusCode;
private final String url;

58
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/Retry.java
View File

@@ -1,58 +0,0 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
/**
* Copied from {@link org.jetbrains.idea.maven.aether.RetryProvider}
*/
final class Retry {
private static final Logger LOG = Logger.getLogger(Retry.class.getName());
private static final Random RANDOM = new Random();
private static final double EXP_BACKOFF_FACTOR = 2;
private static final double EXP_BACKOFF_JITTER = 0.3;
private static final long INITIAL_DELAY_MS = 1000;
private static final long BACKOFF_LIMIT_MS = TimeUnit.MINUTES.toMillis(15);
private static final int MAX_ATTEMPTS = 10;
@FunctionalInterface
interface ThrowableRunnable<T extends Exception> {
void run() throws T;
}
static void withExponentialBackOff(ThrowableRunnable<?> runnable) throws Exception {
long effectiveDelay = INITIAL_DELAY_MS;
for (int i = 1; i <= MAX_ATTEMPTS; i++) {
try {
runnable.run();
return;
}
catch (Exception e) {
if (i == MAX_ATTEMPTS) {
LOG.info("Retry attempts limit exceeded, tried " + MAX_ATTEMPTS + " times. Cause: " + e.getMessage());
throw e;
}
LOG.info("Attempt " + i + " of " + MAX_ATTEMPTS + " failed, retrying in " + effectiveDelay + "ms. Cause: " + e.getMessage());
effectiveDelay = exponentialBackOff(effectiveDelay);
}
}
throw new RuntimeException("Should not be reached");
}
private static long exponentialBackOff(long effectiveDelayMs) {
try {
Thread.sleep(effectiveDelayMs);
}
catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new RuntimeException("Unexpected thread interrupt", ex);
}
long nextRawDelay = (long)Math.min(effectiveDelayMs * EXP_BACKOFF_FACTOR, BACKOFF_LIMIT_MS);
long jitter = (long)(RANDOM.nextDouble() * nextRawDelay * EXP_BACKOFF_JITTER);
long jitterSign = RANDOM.nextBoolean() ? 1 : -1;
return nextRawDelay + jitter * jitterSign;
}
}

41
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/telemetry/BuildDependenciesNoopTracer.java
View File

@@ -1,41 +0,0 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies.telemetry;
import org.jetbrains.annotations.NotNull;
public final class BuildDependenciesNoopTracer implements BuildDependenciesTracer {
public static final BuildDependenciesNoopTracer INSTANCE = new BuildDependenciesNoopTracer();
private BuildDependenciesNoopTracer() {
}
@Override
public BuildDependenciesTraceEventAttributes createAttributes() {
return new BuildDependenciesTraceEventAttributes() {
@Override
public void setAttribute(@NotNull String name, @NotNull String value) {
}
};
}
@Override
public BuildDependenciesSpan startSpan(@NotNull String name, @NotNull BuildDependenciesTraceEventAttributes attributes) {
return new BuildDependenciesSpan() {
@Override
public void addEvent(@NotNull String eventName, @NotNull BuildDependenciesTraceEventAttributes eventAttributes) {
}
@Override
public void recordException(@NotNull Throwable throwable) {
}
@Override
public void setStatus(@NotNull BuildDependenciesSpan.SpanStatus status) {
}
@Override
public void close() {
}
};
}
}

16
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/telemetry/BuildDependenciesSpan.java
View File

@@ -1,16 +0,0 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies.telemetry;
import org.jetbrains.annotations.NotNull;
import java.io.Closeable;
public interface BuildDependenciesSpan extends Closeable {
void addEvent(@NotNull String name, @NotNull BuildDependenciesTraceEventAttributes attributes);
void recordException(@NotNull Throwable throwable);
void setStatus(@NotNull SpanStatus status);
enum SpanStatus {
UNSET, OK, ERROR,
}
}

8
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/telemetry/BuildDependenciesTraceEventAttributes.java
View File

@@ -1,8 +0,0 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies.telemetry;
import org.jetbrains.annotations.NotNull;
public interface BuildDependenciesTraceEventAttributes {
void setAttribute(@NotNull String name, @NotNull String value);
}

9
platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/telemetry/BuildDependenciesTracer.java
View File

@@ -1,9 +0,0 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.dependencies.telemetry;
import org.jetbrains.annotations.NotNull;
public interface BuildDependenciesTracer {
BuildDependenciesTraceEventAttributes createAttributes();
BuildDependenciesSpan startSpan(@NotNull String name, @NotNull BuildDependenciesTraceEventAttributes attributes);
}

7
build/tasks/src/org/jetbrains/intellij/build/io/retry.kt → platform/build-scripts/downloader/src/retry.kt
View File

@@ -1,13 +1,12 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.io
package org.jetbrains.intellij.build
import com.intellij.diagnostic.telemetry.use
import io.opentelemetry.api.common.AttributeKey
import io.opentelemetry.api.common.Attributes
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import org.jetbrains.intellij.build.tracer
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
import java.util.*
import java.util.concurrent.TimeUnit
import kotlin.math.min
@@ -63,7 +62,7 @@ suspend fun <T> suspendingRetryWithExponentialBackOff(
}
private fun defaultExceptionConsumer(attempt: Int, e: Exception) {
tracer.spanBuilder("Retrying action with exponential back off").use { span ->
BuildDependenciesDownloader.TRACER.spanBuilder("Retrying action with exponential back off").startSpan().use { span ->
span.addEvent("Attempt failed", Attributes.of(
AttributeKey.longKey("attemptNumber"), attempt.toLong(),
AttributeKey.stringKey("error"), e.toString()

2
platform/build-scripts/src/org/jetbrains/intellij/build/striped.kt → platform/build-scripts/downloader/src/striped.kt
View File

@@ -1,4 +1,4 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build
import kotlinx.coroutines.sync.Mutex

66
platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildDependenciesOpenTelemetryTracer.kt
View File

@@ -1,66 +0,0 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.impl
import io.opentelemetry.api.common.Attributes
import io.opentelemetry.api.trace.Span
import io.opentelemetry.api.trace.StatusCode
import org.jetbrains.intellij.build.TraceManager
import org.jetbrains.intellij.build.dependencies.telemetry.BuildDependenciesSpan
import org.jetbrains.intellij.build.dependencies.telemetry.BuildDependenciesTraceEventAttributes
import org.jetbrains.intellij.build.dependencies.telemetry.BuildDependenciesTracer
class BuildDependenciesOpenTelemetryTracer private constructor() : BuildDependenciesTracer {
companion object {
@JvmField
val INSTANCE: BuildDependenciesTracer = BuildDependenciesOpenTelemetryTracer()
}
override fun createAttributes(): BuildDependenciesTraceEventAttributes = BuildDependenciesOpenTelemetryAttributes()
override fun startSpan(name: String, attributes: BuildDependenciesTraceEventAttributes): BuildDependenciesSpan {
return BuildDependenciesOpenTelemetrySpan(name, attributes)
}
}
private class BuildDependenciesOpenTelemetrySpan(name: String, attributes: BuildDependenciesTraceEventAttributes) : BuildDependenciesSpan {
private val span: Span
init {
val spanBuilder = TraceManager.spanBuilder(name)
spanBuilder.setAllAttributes((attributes as BuildDependenciesOpenTelemetryAttributes).getAttributes())
span = spanBuilder.startSpan()
}
override fun addEvent(name: String, attributes: BuildDependenciesTraceEventAttributes) {
span.addEvent(name, (attributes as BuildDependenciesOpenTelemetryAttributes).getAttributes())
}
override fun recordException(throwable: Throwable) {
span.recordException(throwable)
}
override fun setStatus(status: BuildDependenciesSpan.SpanStatus) {
val statusCode = when (status) {
BuildDependenciesSpan.SpanStatus.UNSET -> StatusCode.UNSET
BuildDependenciesSpan.SpanStatus.OK -> StatusCode.OK
BuildDependenciesSpan.SpanStatus.ERROR -> StatusCode.ERROR
else -> throw IllegalArgumentException("Unsupported span status: $status")
}
span.setStatus(statusCode)
}
override fun close() {
span.end()
}
}
private class BuildDependenciesOpenTelemetryAttributes : BuildDependenciesTraceEventAttributes {
private val builder = Attributes.builder()
fun getAttributes(): Attributes = builder.build()
override fun setAttribute(name: String, value: String) {
builder.put(name, value)
}
}

6
platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildUtils.kt
View File

@@ -35,7 +35,7 @@ object BuildUtils {
fun propertiesToJvmArgs(properties: List<Pair<String, String>>): List<String> {
val result = ArrayList<String>(properties.size)
for ((key, value) in properties) {
addVmProperty(result, key, value.toString())
addVmProperty(result, key, value)
}
return result
}
@@ -61,4 +61,6 @@ fun convertLineSeparators(file: Path, newLineSeparator: String) {
if (data != convertedData) {
Files.writeString(file, convertedData)
}
}
}
fun String.withTrailingSlash() = if (endsWith('/')) this else "${this}/"

7
platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt
View File

@@ -17,7 +17,6 @@ import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.intellij.build.*
import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
import org.jetbrains.intellij.build.dependencies.DependenciesProperties
import org.jetbrains.intellij.build.dependencies.JdkDownloader
import org.jetbrains.intellij.build.impl.JdkUtils.defineJdk
@@ -48,6 +47,7 @@ fun createCompilationContextBlocking(communityHome: BuildDependenciesCommunityRo
projectHome: Path,
defaultOutputRoot: Path,
options: BuildOptions = BuildOptions()): CompilationContextImpl {
@Suppress("RAW_RUN_BLOCKING")
return runBlocking(Dispatchers.Default) {
createCompilationContext(communityHome = communityHome,
projectHome = projectHome,
@@ -155,12 +155,7 @@ class CompilationContextImpl private constructor(
val isCompilationRequired = CompiledClasses.isCompilationRequired(options)
// this is not a proper place to initialize tracker for downloader but this is the only place which is called in most build scripts
val model = coroutineScope {
launch {
BuildDependenciesDownloader.TRACER = BuildDependenciesOpenTelemetryTracer.INSTANCE
}
loadProject(projectHome = projectHome, kotlinBinaries = KotlinBinaries(communityHome, messages), isCompilationRequired)
}

3
platform/build-scripts/src/org/jetbrains/intellij/build/impl/TraceFileUploader.kt
View File

@@ -4,7 +4,6 @@
package org.jetbrains.intellij.build.impl
import com.fasterxml.jackson.jr.ob.JSON
import org.jetbrains.intellij.build.toUrlWithTrailingSlash
import java.io.IOException
import java.net.HttpURLConnection
import java.net.URL
@@ -14,7 +13,7 @@ import java.nio.file.Files
import java.nio.file.Path
open class TraceFileUploader(serverUrl: String, token: String?) {
private val serverUrl = toUrlWithTrailingSlash(serverUrl)
private val serverUrl = serverUrl.withTrailingSlash()
private val serverAuthToken = token
protected open fun log(message: String) {}

1
platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheDownloader.kt
View File

@@ -9,7 +9,6 @@ import com.intellij.util.io.Decompressor
import org.jetbrains.intellij.build.*
import org.jetbrains.intellij.build.impl.compilation.cache.CommitsHistory
import org.jetbrains.intellij.build.impl.compilation.cache.SourcesStateProcessor
import org.jetbrains.intellij.build.io.retryWithExponentialBackOff
import org.jetbrains.jps.cache.model.BuildTargetState
import java.nio.file.Files
import java.nio.file.Path

4
platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheUploader.kt
View File

@@ -16,8 +16,8 @@ import org.jetbrains.intellij.build.*
import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.impl.compilation.cache.CommitsHistory
import org.jetbrains.intellij.build.impl.compilation.cache.SourcesStateProcessor
import org.jetbrains.intellij.build.impl.withTrailingSlash
import org.jetbrains.intellij.build.io.moveFile
import org.jetbrains.intellij.build.io.retryWithExponentialBackOff
import org.jetbrains.intellij.build.io.zipWithCompression
import org.jetbrains.jps.cache.model.BuildTargetState
import org.jetbrains.jps.incremental.storage.ProjectStamps
@@ -168,7 +168,7 @@ internal class PortableCompilationCacheUploader(
}
private class Uploader(serverUrl: String, val authHeader: String) {
private val serverUrl = toUrlWithTrailingSlash(serverUrl)
private val serverUrl = serverUrl.withTrailingSlash()
fun upload(path: String, file: Path): Boolean {
val url = pathToUrl(path)

1
platform/build-scripts/src/org/jetbrains/intellij/build/impl/sign.kt
View File

@@ -50,6 +50,7 @@ import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.executeStep
import org.jetbrains.intellij.build.io.*
import org.jetbrains.intellij.build.retryWithExponentialBackOff
import org.jetbrains.jps.api.GlobalOptions
import java.io.InputStream
import java.io.OutputStream

2
platform/build-scripts/src/org/jetbrains/intellij/build/impl/support/RepairUtilityBuilder.kt
View File

@@ -19,7 +19,7 @@ import org.jetbrains.intellij.build.TraceManager.spanBuilder
import org.jetbrains.intellij.build.dependencies.TeamCityHelper
import org.jetbrains.intellij.build.executeStep
import org.jetbrains.intellij.build.io.runProcess
import org.jetbrains.intellij.build.io.suspendingRetryWithExponentialBackOff
import org.jetbrains.intellij.build.suspendingRetryWithExponentialBackOff
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption

25
platform/jps-bootstrap/pom.xml
View File

@@ -55,6 +55,31 @@
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-client-core-jvm</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-client-cio-jvm</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-client-encoding-jvm</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-client-auth-jvm</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-extension-kotlin</artifactId>
<version>1.22.0</version>
</dependency>
<dependency>
<groupId>com.jetbrains.intellij.platform</groupId>
<artifactId>jps-model</artifactId>