From 657dce41509b83c03b0ef5abab97fd983055fcbf Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Sat, 11 Jan 2025 08:37:04 +0100 Subject: [PATCH] bazel jps compiler - simplify as we built the only module GitOrigin-RevId: 4e44a0a5352bcc3a6bc64b9adb90fb9ea2950d30 --- build/jvm-rules/src/jps-builder/JpsBuilder.kt | 25 +- .../src/jps-builder/impl/BuildTaskLauncher.kt | 218 ------------ .../src/jps-builder/impl/JpsModuleBuilder.kt | 320 +++++------------- build/jvm-rules/src/jps-builder/impl/load.kt | 4 +- .../src/jps-builder/packager/packager.kt | 2 +- 5 files changed, 80 insertions(+), 489 deletions(-) delete mode 100644 build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt diff --git a/build/jvm-rules/src/jps-builder/JpsBuilder.kt b/build/jvm-rules/src/jps-builder/JpsBuilder.kt index 576d13c3a4c0..a96d50440df1 100644 --- a/build/jvm-rules/src/jps-builder/JpsBuilder.kt +++ b/build/jvm-rules/src/jps-builder/JpsBuilder.kt @@ -5,7 +5,6 @@ package org.jetbrains.bazel.jvm.jps import com.intellij.openapi.diagnostic.IdeaLogRecordFormatter import com.intellij.openapi.diagnostic.Logger -import com.intellij.tracing.Tracer import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.isActive @@ -23,20 +22,14 @@ import org.jetbrains.jps.api.CanceledStatus import org.jetbrains.jps.api.GlobalOptions import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType import org.jetbrains.jps.incremental.* -import org.jetbrains.jps.incremental.relativizer.PathRelativizer -import org.jetbrains.jps.incremental.relativizer.PathRelativizerService import org.jetbrains.jps.incremental.storage.ExperimentalSourceToOutputMapping -import org.jetbrains.jps.incremental.storage.PathTypeAwareRelativizer -import org.jetbrains.jps.incremental.storage.RelativePathType import org.jetbrains.jps.incremental.storage.StorageManager import org.jetbrains.kotlin.config.IncrementalCompilation -import java.io.File import java.io.Writer import java.nio.file.Files import java.nio.file.Path import java.util.Set import kotlin.coroutines.coroutineContext -import kotlin.io.path.invariantSeparatorsPathString // Please note: for performance reasons, we do not set `jps.new.storage.compact.on.close` to true. // As a result, the database file on disk may grow to some extent. @@ -145,22 +138,6 @@ internal suspend fun buildUsingJps( try { val projectDescriptor = storageInitializer.createProjectDescriptor(messageHandler, jpsModel, moduleTarget, relativizer) try { - if (storageInitializer.isCheckRebuildRequired) { - val rebuildRequiredSpan = Tracer.start("JpsProjectBuilder.checkRebuildRequired") - val isRebuildRequired = checkRebuildRequired( - scope = compileScope, - projectDescriptor = projectDescriptor, - moduleTarget = moduleTarget, - isDebugEnabled = isDebugEnabled, - messageHandler = messageHandler, - ) - rebuildRequiredSpan.complete() - - if (isRebuildRequired) { - return false - } - } - val coroutineContext = coroutineContext val context = CompileContextImpl( compileScope, @@ -175,7 +152,7 @@ internal suspend fun buildUsingJps( builderRegistry = BuilderRegistry.getInstance(), messageHandler = messageHandler, isCleanBuild = storageInitializer.isCleanBuild, - ).build(context) + ).build(context, moduleTarget) postBuild( messageHandler = messageHandler, moduleTarget = moduleTarget, diff --git a/build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt b/build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt deleted file mode 100644 index 950b57a570f6..000000000000 --- a/build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -@file:Suppress("UnstableApiUsage") - -package org.jetbrains.bazel.jvm.jps.impl - -import com.intellij.openapi.diagnostic.Logger -import com.intellij.openapi.util.Key -import com.intellij.openapi.util.UserDataHolderBase -import com.intellij.tracing.Tracer -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch -import org.jetbrains.jps.builders.BuildTarget -import org.jetbrains.jps.builders.impl.BuildTargetChunk -import org.jetbrains.jps.incremental.CompileContext -import org.jetbrains.jps.incremental.CompileScope -import org.jetbrains.jps.incremental.GlobalContextKey -import org.jetbrains.jps.incremental.Utils -import org.jetbrains.jps.incremental.messages.BuildMessage -import org.jetbrains.jps.incremental.messages.BuildProgress -import java.util.* -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger - -private val LOG = Logger.getInstance(JpsProjectBuilder::class.java) - -internal class BuildTaskLauncher( - private val context: CompileContext, - private val buildProgress: BuildProgress, - private val builder: JpsProjectBuilder, -) { - private val tasks: MutableList - - init { - val span = Tracer.start("BuildTaskLauncher constructor") - - val targetIndex = context.projectDescriptor.buildTargetIndex - val chunks = targetIndex.getSortedTargetChunks(context) - tasks = ArrayList(chunks.size) - val targetToTask = HashMap, BuildChunkTask>(chunks.size) - for (chunk in chunks) { - val task = BuildChunkTask(chunk) - tasks.add(task) - for (target in chunk.targets) { - targetToTask.put(target, task) - } - } - - val collectTaskDependantsSpan = Tracer.start("IncProjectBuilder.collectTaskDependants") - var taskCounter = 0 - for (task in tasks) { - task.index = taskCounter - taskCounter++ - for (target in task.chunk.targets) { - for (dependency in targetIndex.getDependencies(target, context)) { - val depTask = targetToTask.get(dependency) - if (depTask != null && depTask != task) { - task.addDependency(depTask) - } - } - } - } - collectTaskDependantsSpan.complete() - - val prioritisationSpan = Tracer.start("IncProjectBuilder.prioritisation") - // bitset stores indexes of transitively dependant tasks - val chunkToTransitive = HashMap() - for (task in tasks.asReversed()) { - val dependantTasks = task.tasksDependsOnThis - val directDependants = HashSet(dependantTasks) - val transitiveDependants = BitSet() - for (directDependant in directDependants) { - val dependantChunkTransitiveDependants = chunkToTransitive.get(directDependant) - if (dependantChunkTransitiveDependants != null) { - transitiveDependants.or(dependantChunkTransitiveDependants) - transitiveDependants.set(directDependant.index) - } - } - chunkToTransitive.put(task, transitiveDependants) - } - prioritisationSpan.complete() - - span.complete() - } - - suspend fun buildInParallel() { - val buildSpan = Tracer.start("Parallel build") - coroutineScope { - queueTasks(tasks = tasks.filter { it.isReady }, isDebugLogEnabled = LOG.isDebugEnabled, coroutineScope = this) - } - buildSpan.complete() - } - - private fun queueTasks(tasks: List, isDebugLogEnabled: Boolean, coroutineScope: CoroutineScope) { - for (task in tasks) { - coroutineScope.launch { - executeTask( - chunkLocalContext = ChunkLocalCompileContext(context), - task = task, - isDebugLogEnabled = isDebugLogEnabled, - coroutineScope = coroutineScope, - ) - } - } - } - - private fun executeTask( - chunkLocalContext: CompileContext, - task: BuildChunkTask, - isDebugLogEnabled: Boolean, - coroutineScope: CoroutineScope, - ) { - try { - try { - val isAffectedSpan = Tracer.start("isAffected") - val affected = isBuildChunkAffected(scope = context.scope, chunk = task.chunk) - isAffectedSpan.complete() - if (affected) { - builder.buildTargetChunk(context = chunkLocalContext, chunk = task.chunk, buildProgress = buildProgress) - } - } - finally { - context.projectDescriptor.dataManager.closeSourceToOutputStorages(task.chunk) - } - } - finally { - if (isDebugLogEnabled) { - LOG.debug("Finished compilation of ${task.chunk}") - } - - val nextTasks = task.getNextReadyTasks() - if (!nextTasks.isEmpty()) { - queueTasks(tasks = nextTasks, isDebugLogEnabled = isDebugLogEnabled, coroutineScope = coroutineScope) - } - } - } -} - -internal class BuildChunkTask(@JvmField val chunk: BuildTargetChunk) { - private val notBuildDependenciesCount = AtomicInteger(0) - private val notBuiltDependencies = HashSet() - - @JvmField - val tasksDependsOnThis = ArrayList() - - @JvmField - var index = 0 - - val isReady: Boolean - get() = notBuildDependenciesCount.get() == 0 - - fun addDependency(dependency: BuildChunkTask) { - if (notBuiltDependencies.add(dependency)) { - notBuildDependenciesCount.incrementAndGet() - dependency.tasksDependsOnThis.add(this) - } - } - - fun getNextReadyTasks(): List { - var nextTasks: MutableList? = null - for (task in tasksDependsOnThis) { - val dependenciesCount = task.notBuildDependenciesCount.decrementAndGet() - if (dependenciesCount == 0) { - if (nextTasks == null) { - nextTasks = ArrayList() - } - nextTasks.add(task) - } - } - return nextTasks ?: emptyList() - } -} - -private class ChunkLocalCompileContext(private val sharedContext: CompileContext) : CompileContext by sharedContext { - private val localDataHolder = UserDataHolderBase() - private val deletedKeysSet = ConcurrentHashMap.newKeySet() - - override fun getUserData(key: Key): T? { - return when { - key is GlobalContextKey -> sharedContext.getUserData(key) - deletedKeysSet.contains(key) -> null - else -> localDataHolder.getUserData(key) ?: sharedContext.getUserData(key) - } - } - - override fun putUserData(key: Key, value: T?) { - if (key is GlobalContextKey) { - sharedContext.putUserData(key, value) - return - } - - if (value == null) { - deletedKeysSet.add(key) - } - else { - deletedKeysSet.remove(key) - } - localDataHolder.putUserData(key, value) - } - - override fun processMessage(message: BuildMessage) { - if (message.kind == BuildMessage.Kind.ERROR) { - localDataHolder.putUserData(Utils.ERRORS_DETECTED_KEY, true) - } - sharedContext.processMessage(message) - } - - override fun isCanceled(): Boolean = sharedContext.isCanceled -} - -internal fun isBuildChunkAffected(scope: CompileScope, chunk: BuildTargetChunk): Boolean { - for (target in chunk.targets) { - if (scope.isAffected(target)) { - return true - } - } - return false -} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt b/build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt index 9e4cb7e511c4..0d30d8ad596f 100644 --- a/build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt +++ b/build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt @@ -1,11 +1,9 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "UnstableApiUsage", "ReplaceGetOrSet", "HardCodedStringLiteral") +@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "UnstableApiUsage", "ReplaceGetOrSet", "HardCodedStringLiteral", "RemoveRedundantQualifierName") package org.jetbrains.bazel.jvm.jps.impl -import com.intellij.openapi.util.text.Formats import com.intellij.tracing.Tracer -import com.intellij.util.containers.CollectionFactory import com.intellij.util.containers.FileHashStrategy import it.unimi.dsi.fastutil.objects.Object2ObjectMaps import it.unimi.dsi.fastutil.objects.Object2ObjectOpenCustomHashMap @@ -14,16 +12,13 @@ import org.jetbrains.jps.ModuleChunk import org.jetbrains.jps.builders.BuildRootDescriptor import org.jetbrains.jps.builders.BuildTarget import org.jetbrains.jps.builders.FileProcessor -import org.jetbrains.jps.builders.ModuleBasedTarget import org.jetbrains.jps.builders.impl.BuildOutputConsumerImpl import org.jetbrains.jps.builders.impl.BuildTargetChunk import org.jetbrains.jps.builders.impl.DirtyFilesHolderBase import org.jetbrains.jps.builders.java.JavaBuilderUtil -import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor import org.jetbrains.jps.builders.storage.BuildDataCorruptedException import org.jetbrains.jps.builders.storage.SourceToOutputMapping -import org.jetbrains.jps.cmdline.ProjectDescriptor import org.jetbrains.jps.incremental.* import org.jetbrains.jps.incremental.ModuleLevelBuilder.OutputConsumer import org.jetbrains.jps.incremental.fs.CompilationRound @@ -61,10 +56,10 @@ internal class JpsProjectBuilder( private val elapsedTimeNanosByBuilder = ConcurrentHashMap() private val numberOfSourcesProcessedByBuilder = ConcurrentHashMap() - suspend fun build(context: CompileContextImpl) { + fun build(context: CompileContextImpl, moduleTarget: BazelModuleBuildTarget) { try { val buildSpan = Tracer.start("IncProjectBuilder.runBuild") - runBuild(context) + runBuild(context, moduleTarget = moduleTarget) buildSpan.complete() } catch (e: StopBuildException) { @@ -90,7 +85,7 @@ internal class JpsProjectBuilder( } } - private suspend fun runBuild(context: CompileContextImpl) { + private fun runBuild(context: CompileContextImpl, moduleTarget: BazelModuleBuildTarget) { context.setDone(0.0f) context.addBuildListener(ChainedTargetsBuildListener(context)) @@ -131,21 +126,28 @@ internal class JpsProjectBuilder( } allModuleLevelBuildersBuildStartedSpan.complete() + val projectDescriptor = context.projectDescriptor var buildProgress: BuildProgress? = null try { - val projectDescriptor = context.projectDescriptor val sortedTargetChunks = projectDescriptor.buildTargetIndex.getSortedTargetChunks(context) buildProgress = BuildProgress( projectDescriptor.dataManager, projectDescriptor.buildTargetIndex, sortedTargetChunks, - Predicate { isBuildChunkAffected(scope = context.scope, chunk = it) } + Predicate { context.scope.isAffected(moduleTarget) } ) require(builderRegistry.beforeTasks.isEmpty()) val checkingSourcesSpan = Tracer.start("Building targets") - BuildTaskLauncher(context = context, buildProgress = buildProgress, builder = this).buildInParallel() + // We don't call closeSourceToOutputStorages as we only built a single target and close the database after the build. + // (In any case, for the new storage, it only involves removing the cached map with no actual IO close operation or using MVStore API) + val isAffectedSpan = Tracer.start("isAffected") + val affected = context.scope.isAffected(moduleTarget) + isAffectedSpan.complete() + if (affected) { + buildTargetChunk(context = context, buildProgress = buildProgress, moduleTarget = moduleTarget) + } checkingSourcesSpan.complete() require(builderRegistry.afterTasks.isEmpty()) @@ -155,7 +157,7 @@ internal class JpsProjectBuilder( if (buildProgress != null) { buildProgress.updateExpectedAverageTime() if (isCleanBuild && !Utils.errorsDetected(context) && !context.cancelStatus.isCanceled) { - context.projectDescriptor.dataManager.targetStateManager.setLastSuccessfulRebuildDuration(buildProgress.absoluteBuildTime) + projectDescriptor.dataManager.targetStateManager.setLastSuccessfulRebuildDuration(buildProgress.absoluteBuildTime) } } @@ -177,59 +179,13 @@ internal class JpsProjectBuilder( .forEach { buildMessage: BuilderStatisticsMessage? -> context.processMessage(buildMessage) } } - private fun runBuildersForChunk(context: CompileContext, chunk: BuildTargetChunk, buildProgress: BuildProgress): Boolean { - val targets: Set> = chunk.targets - if (targets.size > 1) { - val moduleTargets = LinkedHashSet() - for (target in targets) { - if (target is ModuleBuildTarget) { - moduleTargets.add(target) - } - else { - val targetsString = targets.joinToString(separator = ", ") { it.presentableName } - val message = "Cannot build \"${target.presentableName}\" because it is included into a circular dependency ($targetsString))" - context.processMessage(CompilerMessage("", BuildMessage.Kind.ERROR, message)) - return false - } - } - - return runModuleLevelBuilders(ModuleInfoAwareCompileContextCopy(context, moduleTargets), ModuleChunk(moduleTargets), buildProgress) - } - - val target = targets.iterator().next() - if (target is ModuleBuildTarget) { - @Suppress("RemoveRedundantQualifierName") - val mbt = java.util.Set.of(target) - return runModuleLevelBuilders(ModuleInfoAwareCompileContextCopy(context, mbt), ModuleChunk(mbt), buildProgress) - } - - @Suppress("UNCHECKED_CAST") - completeRecompiledSourcesSet(context, targets as Collection>) - - /* - In general, the set of files corresponding to changed source file may be different - Need this, for example, to keep up with case changes in file names for case-insensitive OSes: - deleting the output before copying is the only way to ensure the case of the output file's name is exactly the same as source file's case - */ - cleanOldOutputs(context, target) - - val builders = BuilderRegistry.getInstance().targetBuilders - var builderCount = 0 - for (builder in builders) { - buildTarget(target, context, builder) - builderCount++ - buildProgress.updateProgress(target, (builderCount.toDouble()) / builders.size, context) - } - return true - } - - private fun processDeletedPaths(context: CompileContext, targets: Set>): Boolean { + private fun processDeletedPaths(context: CompileContext, target: ModuleBuildTarget): Boolean { var doneSomething = false // cleanup outputs val targetToRemovedSources = HashMap, MutableCollection>() val dirsToDelete = HashSet() - for (target in targets) { + for (target in arrayOf(target)) { val deletedPaths = context.projectDescriptor.fsState.getAndClearDeletedPaths(target) if (deletedPaths.isEmpty()) { continue @@ -242,7 +198,6 @@ internal class JpsProjectBuilder( val dataManager = context.projectDescriptor.dataManager val buildTargetId = dataManager.targetStateManager.getBuildTargetId(target) - val shouldPruneEmptyDirs = target is ModuleBasedTarget<*> val sourceToOutputStorage = dataManager.getSourceToOutputMap(target) val logger = context.loggingManager.projectBuilderLogger // actually delete outputs associated with removed paths @@ -253,7 +208,7 @@ internal class JpsProjectBuilder( val deletedOutputPaths = ArrayList() val outputToSourceRegistry = dataManager.outputToTargetMapping for (output in outputToSourceRegistry.removeTargetAndGetSafeToDeleteOutputs(outputs, buildTargetId, sourceToOutputStorage)) { - val deleted = BuildOperations.deleteRecursivelyAndCollectDeleted(Path.of(output), deletedOutputPaths, if (shouldPruneEmptyDirs) dirsToDelete else null) + val deleted = BuildOperations.deleteRecursivelyAndCollectDeleted(Path.of(output), deletedOutputPaths, dirsToDelete) if (deleted) { doneSomething = true } @@ -266,19 +221,17 @@ internal class JpsProjectBuilder( } } - if (target is ModuleBuildTarget) { - // check if the deleted source was associated with a form - val sourceToFormMap = dataManager.getSourceToFormMap(target) - val boundForms = sourceToFormMap.getOutputs(deletedSource) - if (boundForms != null) { - for (formPath in boundForms) { - val formFile = Path.of(formPath) - if (Files.exists(formFile)) { - FSOperations.markDirty(context, CompilationRound.CURRENT, formFile.toFile()) - } + // check if the deleted source was associated with a form + val sourceToFormMap = dataManager.getSourceToFormMap(target) + val boundForms = sourceToFormMap.getOutputs(deletedSource) + if (boundForms != null) { + for (formPath in boundForms) { + val formFile = Path.of(formPath) + if (Files.exists(formFile)) { + FSOperations.markDirty(context, CompilationRound.CURRENT, formFile.toFile()) } - sourceToFormMap.remove(deletedSource) } + sourceToFormMap.remove(deletedSource) } } } @@ -303,32 +256,16 @@ internal class JpsProjectBuilder( return doneSomething } - private fun > buildTarget(target: T, context: CompileContext, builder: TargetBuilder<*, *>) { - if (builder.getTargetTypes().contains(target.getTargetType())) { - val holder = object : DirtyFilesHolderBase(context) { - override fun processDirtyFiles(processor: FileProcessor) { - context.projectDescriptor.fsState.processFilesToRecompile(context, target, processor) - } - } - val outputConsumer = BuildOutputConsumerImpl(target, context) - val start = System.nanoTime() - @Suppress("UNCHECKED_CAST") - (builder as TargetBuilder).build(target, holder, outputConsumer, context) - storeBuilderStatistics(builder, System.nanoTime() - start, outputConsumer.numberOfProcessedSources) - outputConsumer.fireFileGeneratedEvent() - context.checkCanceled() - } - } - // return true if changed something, false otherwise - private fun runModuleLevelBuilders(context: CompileContext, chunk: ModuleChunk, buildProgress: BuildProgress): Boolean { + private fun runModuleLevelBuilders(context: CompileContext, moduleTarget: BazelModuleBuildTarget, buildProgress: BuildProgress): Boolean { + val chunk = ModuleChunk(setOf(moduleTarget)) for (category in BuilderCategory.entries) { for (builder in builderRegistry.getBuilders(category)) { builder.chunkBuildStarted(context, chunk) } } - completeRecompiledSourcesSet(context, chunk.targets) + completeRecompiledSourcesSet(context, moduleTarget) var doneSomething = false var rebuildFromScratchRequested = false @@ -382,7 +319,7 @@ internal class JpsProjectBuilder( try { for (builder in builders) { outputConsumer.setCurrentBuilderName(builder.presentableName) - processDeletedPaths(context, chunk.targets) + processDeletedPaths(context, moduleTarget) val start = System.nanoTime() val processedSourcesBefore = outputConsumer.getNumberOfProcessedSources() val buildResult = builder.build(context, chunk, dirtyFilesHolder, outputConsumer) @@ -486,26 +423,26 @@ internal class JpsProjectBuilder( } } - internal fun buildTargetChunk(context: CompileContext, chunk: BuildTargetChunk, buildProgress: BuildProgress) { - val buildSpan = Tracer.start { "Building ${chunk.presentableName}" } + private fun buildTargetChunk(context: CompileContext, buildProgress: BuildProgress, moduleTarget: BazelModuleBuildTarget) { + val buildSpan = Tracer.start { "Building ${moduleTarget.presentableName}" } val fsState = context.projectDescriptor.fsState var doneSomething: Boolean + val targets = java.util.Set.of>(moduleTarget) try { - context.setCompilationStartStamp(chunk.targets, System.currentTimeMillis()) + context.setCompilationStartStamp(targets, System.currentTimeMillis()) - sendBuildingTargetMessages(chunk.targets, BuildingTargetProgressMessage.Event.STARTED) + sendBuildingTargetMessages(targets, BuildingTargetProgressMessage.Event.STARTED) Utils.ERRORS_DETECTED_KEY.set(context, false) - for (target in chunk.targets) { - ensureFsStateInitialized(context = context, target = target) - } + ensureFsStateInitialized(context = context, target = moduleTarget) - doneSomething = processDeletedPaths(context, chunk.targets) + doneSomething = processDeletedPaths(context, moduleTarget) - fsState.beforeChunkBuildStart(context, chunk) + val chunk = BuildTargetChunk(targets) + fsState.beforeChunkBuildStart(context, targets) - val runBuildersSpan = Tracer.start { "runBuilders " + chunk.presentableName } - doneSomething = doneSomething or runBuildersForChunk(context, chunk, buildProgress) + val runBuildersSpan = Tracer.start { "runBuilders " + moduleTarget.presentableName } + doneSomething = doneSomething or runModuleLevelBuilders(context, moduleTarget, buildProgress) runBuildersSpan.complete() fsState.clearContextRoundData(context) @@ -523,31 +460,28 @@ internal class JpsProjectBuilder( } catch (e: Throwable) { val message = StringBuilder() - message.append(chunk.presentableName).append(": ").append(e.javaClass.getName()) + message.append(moduleTarget.presentableName).append(": ").append(e.javaClass.getName()) e.message?.let { message.append(": ").append(it) } throw ProjectBuildException(message.toString(), e) } finally { - buildProgress.onTargetChunkFinished(chunk, context) + buildProgress.onTargetChunkFinished(targets, context) try { // restore deleted paths that were not processed by 'integrate' val map = Utils.REMOVED_SOURCES_KEY.get(context) if (map != null) { for (entry in map.entries) { - val paths = entry.value - if (paths != null) { - for (path in paths) { - fsState.registerDeleted(context, entry.key, Path.of(path), null) - } + for (path in entry.value) { + fsState.registerDeleted(context, entry.key, Path.of(path), null) } } } } finally { Utils.REMOVED_SOURCES_KEY.set(context, null) - sendBuildingTargetMessages(chunk.targets, BuildingTargetProgressMessage.Event.FINISHED) + sendBuildingTargetMessages(targets, BuildingTargetProgressMessage.Event.FINISHED) buildSpan.complete() } } @@ -670,27 +604,6 @@ private class ChainedTargetsBuildListener(private val context: CompileContextImp } } -private fun calculateEstimatedBuildTime( - projectDescriptor: ProjectDescriptor, - target: ModuleBuildTarget, - messageHandler: ConsoleMessageHandler, -): Long { - // compute estimated times for dirty targets - var affectedTargets = 0 - val avgTimeToBuild = projectDescriptor.dataManager.targetStateManager.getAverageBuildTime(target.targetType) - val estimatedBuildTime = if (avgTimeToBuild > 0) { - affectedTargets = 1 - // 1. in general case, this time should include dependency analysis and cache update times - // 2. need to check isAffected() since some targets (like artifacts) may be unaffected even for rebuild - avgTimeToBuild - } - else { - 0L - } - messageHandler.info("Affected build targets count: $affectedTargets") - return estimatedBuildTime -} - internal fun reportRebuiltModules(context: CompileContextImpl) { val modules = BuildTargetConfiguration.MODULES_WITH_TARGET_CONFIG_CHANGED_KEY.get(context) if (modules.isNullOrEmpty()) { @@ -714,71 +627,48 @@ private fun isTargetOutputCleared(context: CompileContext, target: BuildTarget<* } } -private class ModuleInfoAwareCompileContextCopy( - private val context: CompileContext, - private val moduleTargets: Collection, -) : CompileContext by context { - override fun processMessage(message: BuildMessage) { - if (message is CompilerMessage) { - for (target in moduleTargets) { - message.addModuleName(target.module.name) - } - } - context.processMessage(message) - } - - @Suppress("OVERRIDE_DEPRECATION") - override fun isProjectRebuild() = false - - override fun isCanceled(): Boolean = context.isCanceled -} - /** * if an output file is generated from multiple sources, make sure all of them are added for recompilation */ @Suppress("SpellCheckingInspection") -private fun , R : BuildRootDescriptor> completeRecompiledSourcesSet(context: CompileContext, targets: Collection) { +private fun completeRecompiledSourcesSet(context: CompileContext, moduleBuildTarget: BazelModuleBuildTarget) { val scope = context.scope - for (target in targets) { - if (scope.isBuildForced(target)) { - // assuming build is either forced for all targets in a chunk or for none of them - return - } + if (scope.isBuildForced(moduleBuildTarget)) { + // assuming build is either forced for all targets in a chunk or for none of them + return } val projectDescriptor = context.projectDescriptor - val affectedOutputs = CollectionFactory.createFilePathSet() - val affectedSources = CollectionFactory.createFilePathSet() + val affectedOutputs = HashSet() + val affectedSources = HashSet() val mappings = ArrayList() - for (target in targets) { - projectDescriptor.fsState.processFilesToRecompile(context, target, object : FileProcessor { - private var srcToOut: SourceToOutputMapping? = null + projectDescriptor.fsState.processFilesToRecompile(context, moduleBuildTarget, object : FileProcessor { + private var srcToOut: SourceToOutputMapping? = null - override fun apply(target: T, file: File, root: R): Boolean { - val src = file.invariantSeparatorsPath - if (!affectedSources.add(src)) { - return true - } - - // lazy init - var srcToOut = this.srcToOut - if (srcToOut == null) { - srcToOut = projectDescriptor.dataManager.getSourceToOutputMap(target) - mappings.add(srcToOut) - this.srcToOut = srcToOut - } - - val outs = srcToOut.getOutputs(src) ?: return true - // Temporary hack for KTIJ-197 - // Change of only one input of *.kotlin_module files didn't trigger recompilation of all inputs in old behavior. - // Now it does. It isn't yet obvious whether it is right or wrong behavior. Let's leave old behavior for a - // while for safety and keeping kotlin incremental JPS tests green - outs.filterTo(affectedOutputs) { "kotlin_module" != it.substringAfterLast('.') } + override fun apply(target: BazelModuleBuildTarget, file: File, root: JavaSourceRootDescriptor): Boolean { + val src = file.invariantSeparatorsPath + if (!affectedSources.add(src)) { return true } - }) - } + + // lazy init + var srcToOut = this.srcToOut + if (srcToOut == null) { + srcToOut = projectDescriptor.dataManager.getSourceToOutputMap(target) + mappings.add(srcToOut) + this.srcToOut = srcToOut + } + + val outs = srcToOut.getOutputs(src) ?: return true + // Temporary hack for KTIJ-197 + // Change of only one input of *.kotlin_module files didn't trigger recompilation of all inputs in old behavior. + // Now it does. It isn't yet obvious whether it is right or wrong behavior. Let's leave old behavior for a + // while for safety and keeping kotlin incremental JPS tests green + outs.filterTo(affectedOutputs) { "kotlin_module" != it.substringAfterLast('.') } + return true + } + }) if (affectedOutputs.isEmpty()) { return @@ -800,16 +690,6 @@ private fun , R : BuildRootDescriptor> completeRecompiledSour } } -private fun cleanOldOutputs(context: CompileContext, target: BuildTarget) { - if (!context.scope.isBuildForced(target)) { - BuildOperations.cleanOutputsCorrespondingToChangedFiles(context, object : DirtyFilesHolderBase>(context) { - override fun processDirtyFiles(processor: FileProcessor>) { - context.projectDescriptor.fsState.processFilesToRecompile(context, target, processor) - } - }) - } -} - private fun notifyChunkRebuildRequested(context: CompileContext, chunk: ModuleChunk, builder: ModuleLevelBuilder) { var infoMessage = "Builder \"${builder.presentableName}\" requested rebuild of module chunk \"${chunk.name}\"" var kind = BuildMessage.Kind.JPS_INFO @@ -832,51 +712,3 @@ private fun saveInstrumentedClasses(outputConsumer: ChunkBuildOutputConsumerImpl } } } - -internal fun checkRebuildRequired( - scope: CompileScope, - projectDescriptor: ProjectDescriptor, - moduleTarget: ModuleBuildTarget, - isDebugEnabled: Boolean, - messageHandler: ConsoleMessageHandler, -): Boolean { - val targetStateManager = projectDescriptor.dataManager.targetStateManager - val timeThreshold = targetStateManager.getLastSuccessfulRebuildDuration() * 95 / 100 // 95% of last registered clean rebuild time - if (timeThreshold <= 0) { - if (isDebugEnabled) { - messageHandler.debug("Rebuild heuristic: no stats available") - } - return false - } - - // check that this is a whole-project incremental build - // checking only JavaModuleBuildTargetType because these target types directly correspond to project modules - val type = JavaModuleBuildTargetType.PRODUCTION - if (!scope.isAllTargetsOfTypeAffected(type)) { - if (isDebugEnabled) { - messageHandler.debug("Rebuild heuristic: skipping the check because some targets are excluded from compilation scope," + - " e.g. targets of type ${type.typeId}") - } - return false - } - - // compute estimated times for dirty targets - val estimatedWorkTime = calculateEstimatedBuildTime(projectDescriptor, moduleTarget, messageHandler) - if (isDebugEnabled) { - messageHandler.debug("Rebuild heuristic: estimated build time / timeThreshold : $estimatedWorkTime / $timeThreshold") - } - - if (estimatedWorkTime < timeThreshold) { - return false - } - - val message = """ - Too many files require recompilation, forcing full rebuild. - * Estimated build duration (linear): ${Formats.formatDuration(estimatedWorkTime)} - * Last successful rebuild duration (linear): ${Formats.formatDuration(targetStateManager.getLastSuccessfulRebuildDuration())} - * Rebuild heuristic time threshold: ${Formats.formatDuration(timeThreshold)} - """.trimIndent() - messageHandler.info(message) - return true -} - diff --git a/build/jvm-rules/src/jps-builder/impl/load.kt b/build/jvm-rules/src/jps-builder/impl/load.kt index 75deb7417a6c..aaa5ad4c2ff6 100644 --- a/build/jvm-rules/src/jps-builder/impl/load.kt +++ b/build/jvm-rules/src/jps-builder/impl/load.kt @@ -71,9 +71,9 @@ private class BazelBuildTargetIndex(moduleTarget: ModuleBuildTarget) : BuildTarg override fun isDummy(target: BuildTarget<*>) = false @Suppress("OVERRIDE_DEPRECATION", "removal") - override fun getDependenciesRecursively(target: BuildTarget<*>, context: CompileContext): Set> = emptySet() + override fun getDependenciesRecursively(target: BuildTarget<*>, context: CompileContext): Set> = java.util.Set.of() - override fun getDependencies(target: BuildTarget<*>, context: CompileContext): Collection> = emptyList() + override fun getDependencies(target: BuildTarget<*>, context: CompileContext): Collection> = java.util.List.of() override fun getModuleBasedTargets(module: JpsModule, selector: ModuleTargetSelector): List = targets diff --git a/build/jvm-rules/src/jps-builder/packager/packager.kt b/build/jvm-rules/src/jps-builder/packager/packager.kt index f34263fe8d3e..9e77daaa7feb 100644 --- a/build/jvm-rules/src/jps-builder/packager/packager.kt +++ b/build/jvm-rules/src/jps-builder/packager/packager.kt @@ -67,7 +67,7 @@ suspend fun packageToJar( } if (classesToBeDeleted.isNotEmpty()) { - messageHandler.out.appendLine("Classes to be deleted: ${classesToBeDeleted.size}") + messageHandler.debug("Non-abi classes to be deleted: ${classesToBeDeleted.size}") } } }