MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

prefer to use random temp file, async telemetry flush

Browse Source 
GitOrigin-RevId: de9d081865f21dfc5fe78e0dd3787f59c72caa67
This commit is contained in:
Vladimir Krivosheev
2023-11-24 18:03:59 +00:00
committed by intellij-monorepo-bot
parent f64a0ad737
commit 8427a58d21
14 changed files with 131 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/src/FullUpdaterBuildTarget.kt
+3 -3
View File
@@ -11,10 +11,10 @@ object FullUpdaterBuildTarget {
@JvmStatic
fun main(args: Array<String>) {
val context = createCommunityBuildContext(IdeaProjectLoaderUtil.guessCommunityHome(javaClass))
val tasks = BuildTasks.create(context)
tasks.compileModules(listOf(UPDATER_MODULE_NAME))
runBlocking(Dispatchers.Default) {
val context = createCommunityBuildContext(IdeaProjectLoaderUtil.guessCommunityHome(javaClass))
val tasks = BuildTasks.create(context)
tasks.compileModules(listOf(UPDATER_MODULE_NAME))
tasks.buildFullUpdaterJar()
}
}
build/src/OpenSourceCommunityInstallersBuildTarget.kt
+4 -4
View File
@@ -1,4 +1,4 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
import com.intellij.platform.diagnostic.telemetry.helpers.useWithScope2
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
@@ -10,7 +10,7 @@ object OpenSourceCommunityInstallersBuildTarget {
@JvmStatic
fun main(args: Array<String>) {
val options = BuildOptions().apply {
// we cannot provide consistent build number for IDEA Community if it's built separately so use *.SNAPSHOT number to avoid confusion
// we cannot provide a consistent build number for IDEA Community if it's built separately so use *.SNAPSHOT number to avoid confusion
buildNumber = null
// do not bother external users about clean/incremental
@@ -20,9 +20,9 @@ object OpenSourceCommunityInstallersBuildTarget {
buildStepsToSkip += BuildOptions.MAC_SIGN_STEP
}
val context = createCommunityBuildContext(IdeaProjectLoaderUtil.guessCommunityHome(javaClass), options)
BuildTasks.create(context).compileProjectAndTests(listOf("intellij.platform.jps.build.tests"))
runBlocking(Dispatchers.Default) {
val context = createCommunityBuildContext(IdeaProjectLoaderUtil.guessCommunityHome(javaClass), options)
BuildTasks.create(context).compileProjectAndTests(listOf("intellij.platform.jps.build.tests"))
buildDistributions(context)
spanBuilder("build standalone JPS").useWithScope2 {
buildCommunityStandaloneJpsBuilder(targetDir = context.paths.artifactDir.resolve("jps"), context = context)
build/src/org/jetbrains/intellij/build/IdeaCommunityProperties.kt
+5 -5
View File
@@ -9,15 +9,15 @@ import org.jetbrains.intellij.build.kotlin.KotlinBinaries
import java.nio.file.Path
internal fun createCommunityBuildContext(
internal suspend fun createCommunityBuildContext(
communityHome: BuildDependenciesCommunityRoot,
options: BuildOptions = BuildOptions(),
projectHome: Path = communityHome.communityRoot,
): BuildContext {
return BuildContextImpl.createContextBlocking(communityHome = communityHome,
projectHome = projectHome,
productProperties = IdeaCommunityProperties(communityHome.communityRoot),
options = options)
return BuildContextImpl.createContext(communityHome = communityHome,
projectHome = projectHome,
productProperties = IdeaCommunityProperties(communityHome.communityRoot),
options = options)
}
open class IdeaCommunityProperties(private val communityHomeDir: Path) : BaseIdeaProperties() {
build/tasks/src/org/jetbrains/intellij/build/TraceManager.kt
+27 -12
View File
@@ -27,34 +27,47 @@ import java.nio.file.Path
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
var traceManagerInitializer: () -> Tracer = {
var traceManagerInitializer: () -> Pair<Tracer, BatchSpanProcessor> = {
@Suppress("OPT_IN_USAGE")
val batchSpanProcessor = BatchSpanProcessor(coroutineScope = GlobalScope, spanExporters = TracerProviderManager.spanExporterProvider())
val tracerProvider = SdkTracerProvider.builder()
.addSpanProcessor(BatchSpanProcessor(coroutineScope = GlobalScope, spanExporters = TracerProviderManager.spanExporterProvider()))
.addSpanProcessor(batchSpanProcessor)
.setResource(Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), "builder")))
.build()
val openTelemetry = OpenTelemetrySdk.builder()
.setTracerProvider(tracerProvider)
.build()
val tracer = openTelemetry.getTracer("build-script")
TracerProviderManager.tracerProvider = tracerProvider
BuildDependenciesDownloader.TRACER = tracer
tracer
tracer to batchSpanProcessor
}
object TraceManager {
private val tracer: Tracer = traceManagerInitializer()
private val tracer: Tracer
private val batchSpanProcessor: BatchSpanProcessor
init {
val config = traceManagerInitializer()
tracer = config.first
batchSpanProcessor = config.second
}
fun spanBuilder(spanName: String): SpanBuilder = tracer.spanBuilder(spanName)
suspend fun flush() {
batchSpanProcessor.flush()
}
suspend fun exportPendingSpans() {
batchSpanProcessor.doFlush(exportOnly = true)
}
}
object TracerProviderManager {
private val shutdownHookAdded = AtomicBoolean()
private val jaegerJsonSpanExporter = AtomicReference<JaegerJsonSpanExporter?>()
var tracerProvider: SdkTracerProvider? = null
var spanExporterProvider: () -> List<AsyncSpanExporter> = {
internal val spanExporterProvider: () -> List<AsyncSpanExporter> = {
val list = mutableListOf(ConsoleSpanExporter(), object : AsyncSpanExporter {
override suspend fun export(spans: Collection<SpanData>) {
jaegerJsonSpanExporter.get()?.export(spans)
@@ -63,6 +76,10 @@ object TracerProviderManager {
override suspend fun shutdown() {
jaegerJsonSpanExporter.getAndSet(null)?.shutdown()
}
override suspend fun flush() {
jaegerJsonSpanExporter.getAndSet(null)?.flush()
}
})
normalizeOtlpEndPoint(System.getenv("OTLP_ENDPOINT"))?.let {
list.add(OtlpSpanExporter(it))
@@ -79,6 +96,7 @@ object TracerProviderManager {
Runtime.getRuntime().addShutdownHook(Thread({
jaegerJsonSpanExporter.getAndSet(null)?.let {
runBlocking {
TraceManager.flush()
it.shutdown()
}
}
@@ -89,6 +107,7 @@ object TracerProviderManager {
suspend fun finish(): Path? {
try {
return jaegerJsonSpanExporter.getAndSet(null)?.let {
TraceManager.flush()
val file = it.file
it.shutdown()
file
@@ -99,8 +118,4 @@ object TracerProviderManager {
return null
}
}
suspend fun flush() {
jaegerJsonSpanExporter.get()?.flush()
}
}
platform/build-scripts/dev-server/src/DevMainImpl.kt
+1 -3
View File
@@ -15,7 +15,6 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.cancel
import kotlinx.coroutines.runBlocking
import org.jetbrains.intellij.build.ConsoleSpanExporter
import org.jetbrains.intellij.build.TracerProviderManager
import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader
import org.jetbrains.intellij.build.traceManagerInitializer
import java.io.File
@@ -43,9 +42,8 @@ fun buildDevMain(): Collection<Path> {
.setTracerProvider(tracerProvider)
.build()
val tracer = openTelemetry.getTracer("build-script")
TracerProviderManager.tracerProvider = tracerProvider
BuildDependenciesDownloader.TRACER = tracer
tracer
tracer to spanProcessor
}
buildProductInProcess(BuildRequest(
platform/build-scripts/src/org/jetbrains/intellij/build/BuildTasks.kt
+4 -1
View File
@@ -1,4 +1,6 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog")
package org.jetbrains.intellij.build
import kotlinx.coroutines.Dispatchers
@@ -44,6 +46,7 @@ interface BuildTasks {
*/
suspend fun buildNonBundledPlugins(mainPluginModules: List<String>)
@Deprecated("Use buildNonBundledPlugins", ReplaceWith("buildNonBundledPlugins(mainPluginModules)"))
fun blockingBuildNonBundledPlugins(mainPluginModules: List<String>) {
runBlocking(Dispatchers.Default) {
buildNonBundledPlugins(mainPluginModules)
@@ -55,7 +58,7 @@ interface BuildTasks {
fun compileModules(moduleNames: Collection<String>?, includingTestsInModules: List<String>)
fun compileModules(moduleNames: Collection<String>?) {
compileModules(moduleNames, emptyList())
compileModules(moduleNames = moduleNames, includingTestsInModules = java.util.List.of())
}
/**
platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildContextImpl.kt
+7 -2
View File
@@ -86,8 +86,10 @@ class BuildContextImpl(
}
companion object {
@Suppress("DeprecatedCallableAddReplaceWith")
@JvmStatic
@JvmOverloads
@Deprecated("Use createContext")
fun createContextBlocking(communityHome: BuildDependenciesCommunityRoot,
projectHome: Path,
productProperties: ProductProperties,
@@ -106,12 +108,15 @@ class BuildContextImpl(
projectHome: Path,
productProperties: ProductProperties,
proprietaryBuildTools: ProprietaryBuildTools = ProprietaryBuildTools.DUMMY,
setupTracer: Boolean = true,
options: BuildOptions = BuildOptions()): BuildContext {
val compilationContext = CompilationContextImpl.createCompilationContext(
communityHome = communityHome,
projectHome = projectHome,
setupTracer = true,
buildOutputRootEvaluator = createBuildOutputRootEvaluator(projectHome, productProperties, options),
setupTracer = setupTracer,
buildOutputRootEvaluator = createBuildOutputRootEvaluator(projectHome = projectHome,
productProperties = productProperties,
buildOptions = options),
options = options,
)
return createContext(compilationContext = compilationContext,
platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt
+2 -2
View File
@@ -58,9 +58,9 @@ suspend fun createCompilationContext(communityHome: BuildDependenciesCommunityRo
projectHome: Path,
defaultOutputRoot: Path,
options: BuildOptions = BuildOptions()): CompilationContextImpl {
val logDir = options.logPath?.let { Path.of(it).toAbsolutePath().normalize() }
val logDir = options.logPath?.let { Path.of(it) }
?: (options.outputRootPath ?: defaultOutputRoot).resolve("log")
TracerProviderManager.setOutput(logDir.resolve("trace.json"))
TracerProviderManager.setOutput(logDir.toAbsolutePath().normalize().resolve("trace.json"))
return CompilationContextImpl.createCompilationContext(communityHome = communityHome,
projectHome = projectHome,
setupTracer = false,
platform/build-scripts/src/org/jetbrains/intellij/build/impl/logging/BuildMessagesImpl.kt
+3 -3
View File
@@ -112,7 +112,7 @@ class BuildMessagesImpl private constructor(private val logger: BuildMessageLogg
override fun block(blockName: String, task: Callable<Unit>) {
runBlocking {
TracerProviderManager.flush()
TraceManager.exportPendingSpans()
}
try {
@@ -124,7 +124,7 @@ class BuildMessagesImpl private constructor(private val logger: BuildMessageLogg
catch (e: Throwable) {
// print all pending spans
runBlocking {
TracerProviderManager.flush()
TraceManager.exportPendingSpans()
}
throw e
}
@@ -132,7 +132,7 @@ class BuildMessagesImpl private constructor(private val logger: BuildMessageLogg
}
finally {
runBlocking {
TracerProviderManager.flush()
TraceManager.exportPendingSpans()
}
processMessage(LogMessage(LogMessage.Kind.BLOCK_FINISHED, blockName))
}
platform/build-scripts/testFramework/src/com/intellij/platform/buildScripts/testFramework/buildScriptTestUtils.kt
+13 -9
View File
@@ -103,13 +103,16 @@ fun runTestBuild(
val reproducibilityTest = BuildArtifactsReproducibilityTest()
repeat(reproducibilityTest.iterations) { iterationNumber ->
launch {
val buildContext = BuildContextImpl.createContext(communityHome = communityHomePath,
projectHome = homePath,
productProperties = productProperties,
proprietaryBuildTools = buildTools,
options = createBuildOptionsForTest(productProperties, buildOptionsCustomizer).also {
reproducibilityTest.configure(it)
})
val buildContext = BuildContextImpl.createContext(
communityHome = communityHomePath,
projectHome = homePath,
productProperties = productProperties,
proprietaryBuildTools = buildTools,
setupTracer = false,
options = createBuildOptionsForTest(productProperties, buildOptionsCustomizer).also {
reproducibilityTest.configure(it)
},
)
doRunTestBuild(
context = buildContext,
traceSpanName = "#$iterationNumber",
@@ -128,6 +131,7 @@ fun runTestBuild(
projectHome = homePath,
productProperties = productProperties,
proprietaryBuildTools = buildTools,
setupTracer = false,
options = createBuildOptionsForTest(productProperties, buildOptionsCustomizer)),
traceSpanName = traceSpanName,
build = { context ->
@@ -235,7 +239,7 @@ private fun copyDebugLog(productProperties: ProductProperties, messages: BuildMe
private suspend inline fun asSingleTraceFile(traceSpanName: String, build: () -> Unit) {
val traceFile = TestLoggerFactory.getTestLogDir().resolve("$traceSpanName-trace.json")
TracerProviderManager.setOutput(traceFile)
TracerProviderManager.setOutput(traceFile.toAbsolutePath().normalize())
try {
build()
}
@@ -245,7 +249,7 @@ private suspend inline fun asSingleTraceFile(traceSpanName: String, build: () ->
}
private suspend fun publishTraceFile() {
val trace = TracerProviderManager.finish()?.takeIf { it.exists() } ?: return
val trace = TracerProviderManager.finish()?.takeIf { Files.exists(it) } ?: return
try {
println("Performance report is written to $trace")
println("##teamcity[publishArtifacts '$trace']")
platform/diagnostic/telemetry.exporters/src/BatchSpanProcessor.kt
+24 -11
View File
@@ -29,7 +29,11 @@ class BatchSpanProcessor(
private val maxExportBatchSize: Int = 512
) : SpanProcessor {
private val queue = Channel<ReadableSpan>(capacity = Channel.UNLIMITED)
private val flushRequested = Channel<CompletableDeferred<Unit>>(capacity = Channel.UNLIMITED)
private val flushRequested = Channel<FlushRequest>(capacity = Channel.UNLIMITED)
private data class FlushRequest(@JvmField val exportOnly: Boolean) {
@JvmField val job: CompletableDeferred<Unit> = CompletableDeferred()
}
init {
coroutineScope.launch {
@@ -37,15 +41,18 @@ class BatchSpanProcessor(
try {
while (true) {
select {
flushRequested.onReceive { result ->
flushRequested.onReceive { request ->
try {
exportCurrentBatch(batch)
for (spanExporter in spanExporters) {
spanExporter.flush()
val isExported = exportCurrentBatch(batch)
if (isExported && !request.exportOnly) {
for (spanExporter in spanExporters) {
spanExporter.flush()
}
}
Unit
}
finally {
result.complete(Unit)
request.job.complete(Unit)
}
}
queue.onReceive { span ->
@@ -103,9 +110,13 @@ class BatchSpanProcessor(
}
suspend fun flush() {
val completableDeferred = CompletableDeferred<Unit>()
if (!flushRequested.trySend(completableDeferred).isClosed) {
completableDeferred.join()
doFlush(exportOnly = false)
}
suspend fun doFlush(exportOnly: Boolean) {
val flushRequest = FlushRequest(exportOnly = exportOnly)
if (!flushRequested.trySend(flushRequest).isClosed) {
flushRequest.job.join()
}
}
@@ -113,9 +124,9 @@ class BatchSpanProcessor(
throw UnsupportedOperationException()
}
private suspend fun exportCurrentBatch(batch: MutableList<SpanData>) {
private suspend fun exportCurrentBatch(batch: MutableList<SpanData>): Boolean {
if (batch.isEmpty()) {
return
return false
}
try {
@@ -124,6 +135,7 @@ class BatchSpanProcessor(
spanExporter.export(batch)
}
}
return true
}
catch (e: CancellationException) {
throw e
@@ -134,6 +146,7 @@ class BatchSpanProcessor(
finally {
batch.clear()
}
return false
}
suspend fun flushOtlp(scopeSpans: Collection<ScopeSpans>) {
platform/diagnostic/telemetry.exporters/src/JaegerJsonSpanExporter.kt
+4 -5
View File
@@ -39,14 +39,13 @@ class JaegerJsonSpanExporter(
init {
// presume that telemetry stuff needs to be saved in log dir
if (!file.isAbsolute || file.parent == null) {
Files.createDirectories(PathManager.getLogDir().toAbsolutePath()).apply {
tempTelemetryPath = this.resolve("telemetry.temp")
telemetryJsonPath = this.resolve(file)
}
val logDir = Files.createDirectories(PathManager.getLogDir().toAbsolutePath())
tempTelemetryPath = Files.createTempFile(logDir, "telemetry-", ".temp")
telemetryJsonPath = logDir.resolve(file)
}
// path is absolute and has a parent
else {
tempTelemetryPath = Files.createDirectories(file.parent.toAbsolutePath()).resolve("telemetry.temp").toAbsolutePath()
tempTelemetryPath = Files.createTempFile(file.parent, "telemetry", ".temp").toAbsolutePath()
telemetryJsonPath = file
}
python/build/src/PythonCommunityPluginBuildTarget.kt
+6 -1
View File
@@ -1,10 +1,15 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("SSBasedInspection")
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import org.jetbrains.intellij.build.IdeaProjectLoaderUtil
import org.jetbrains.intellij.build.pycharm.PythonCommunityPluginBuilder
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
object PythonCommunityPluginBuildTarget {
@JvmStatic
fun main(args: Array<String>) {
fun main(args: Array<String>) = runBlocking(Dispatchers.Default) {
PythonCommunityPluginBuilder(IdeaProjectLoaderUtil.guessCommunityHome(javaClass).communityRoot).build()
}
}
python/build/src/org/jetbrains/intellij/build/pycharm/PythonCommunityPluginBuilder.kt
+28 -22
View File
@@ -1,21 +1,24 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("SSBasedInspection")
package org.jetbrains.intellij.build.pycharm
import io.opentelemetry.api.trace.Span
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.jetbrains.intellij.build.BuildOptions
import org.jetbrains.intellij.build.BuildTasks
import org.jetbrains.intellij.build.IdeaCommunityProperties
import org.jetbrains.intellij.build.ProprietaryBuildTools
import org.jetbrains.intellij.build.dependencies.BuildDependenciesCommunityRoot
import org.jetbrains.intellij.build.impl.BuildContextImpl
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
internal class PythonCommunityPluginBuilder(private val home: Path) {
fun build() {
suspend fun build() {
val pluginBuildNumber = System.getProperty("build.number", "SNAPSHOT")
val homeDir = home
val options = BuildOptions()
@@ -23,33 +26,36 @@ internal class PythonCommunityPluginBuilder(private val home: Path) {
options.outputRootPath = homeDir.resolve("out/pycharmCE")
val communityRoot = BuildDependenciesCommunityRoot(homeDir)
val buildContext = BuildContextImpl.createContextBlocking(communityRoot,
homeDir,
IdeaCommunityProperties(communityRoot.communityRoot),
ProprietaryBuildTools.DUMMY,
options)
BuildTasks.create(buildContext).blockingBuildNonBundledPlugins(listOf(
val buildContext = BuildContextImpl.createContext(communityHome = communityRoot,
projectHome = homeDir,
productProperties = IdeaCommunityProperties(communityRoot.communityRoot),
options = options)
BuildTasks.create(buildContext).buildNonBundledPlugins(listOf(
"intellij.python.community.plugin",
"intellij.reStructuredText",
))
val builtPlugins = mutableListOf<Path>()
Files.walkFileTree(buildContext.paths.artifactDir.resolve("${buildContext.applicationInfo.productCode}-plugins"),
object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult {
if (file.toString().endsWith(".zip")) {
builtPlugins.add(file)
withContext(Dispatchers.IO) {
Files.walkFileTree(buildContext.paths.artifactDir.resolve("${buildContext.applicationInfo.productCode}-plugins"),
object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes?): FileVisitResult {
if (file.toString().endsWith(".zip")) {
builtPlugins.add(file)
}
return FileVisitResult.CONTINUE
}
return FileVisitResult.CONTINUE
}
})
})
}
if (builtPlugins.isEmpty()) {
buildContext.messages.warning("No plugins were built")
Span.current().addEvent("No plugins were built")
return
}
val pluginsPaths = buildContext.paths.buildOutputDir.resolve("plugins-paths.txt")
Files.createDirectories(pluginsPaths.parent)
Files.writeString(pluginsPaths, builtPlugins.joinToString("\n"))
val pluginPaths = buildContext.paths.buildOutputDir.resolve("plugins-paths.txt")
withContext(Dispatchers.IO) {
Files.createDirectories(pluginPaths.parent)
Files.writeString(pluginPaths, builtPlugins.joinToString("\n"))
}
}
}