From db3b4f716275a18418ae348fce8fc3f53d37f0b2 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Tue, 17 Sep 2024 08:16:38 +0200 Subject: [PATCH] experimental compact storage for JPS Cache (part 5 - store OutputToTargetMapping) GitOrigin-RevId: 36feec030cee2cbd5554a4fc0a3b80dd74ea764c --- jps/jps-builders/api-dump.txt | 14 +- .../jetbrains/jps/cmdline/BuildRunner.java | 9 +- .../jps/cmdline/ProjectDescriptor.java | 29 +- .../jps/incremental/BuildOperations.java | 44 +-- .../jps/incremental/IncProjectBuilder.java | 208 ++++++------ .../jps/incremental/fs/BuildFSState.java | 17 +- .../jps/incremental/fs/FilesDelta.java | 10 +- .../storage/AbstractStateStorage.java | 27 +- .../incremental/storage/BuildDataManager.java | 305 +++++++++--------- .../storage/ExperimentalBuildDataManager.kt | 74 +++++ .../ExperimentalOneToManyPathMapping.kt | 34 +- .../ExperimentalOutputToTargetMapping.kt | 163 ++++++++++ .../ExperimentalSourceToOutputMapping.kt | 62 ++-- .../storage/OneToManyPathMapping.kt | 14 + .../storage/OutputToTargetRegistry.java | 46 ++- .../storage/PerTargetMapManager.kt | 12 +- .../jps/incremental/storage/StorageManager.kt | 31 +- .../storage/dataTypes/LongListKeyDataType.kt | 45 +++ .../storage/dataTypes/LongPairKeyDataType.kt | 8 +- .../jetbrains/jps/builders/BuildResult.java | 18 +- .../jps/builders/JpsBuildTestCase.java | 4 +- .../storage/HashStampStorageFuzzTest.kt | 2 +- .../storage/SourceToOutputMappingFuzzTest.kt | 10 +- .../incremental/storage/StorageManagerTest.kt | 7 +- platform/util/base/api-dump.txt | 1 + .../util/containers/CollectionFactory.java | 11 +- 26 files changed, 778 insertions(+), 427 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalBuildDataManager.kt create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOutputToTargetMapping.kt create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongListKeyDataType.kt diff --git a/jps/jps-builders/api-dump.txt b/jps/jps-builders/api-dump.txt index 3601d1e015f6..02e83e9cf40a 100644 --- a/jps/jps-builders/api-dump.txt +++ b/jps/jps-builders/api-dump.txt @@ -1883,7 +1883,7 @@ f:org.jetbrains.jps.cmdline.PreloadedData f:org.jetbrains.jps.cmdline.ProjectDescriptor - f:dataManager:org.jetbrains.jps.incremental.storage.BuildDataManager - f:fsState:org.jetbrains.jps.incremental.fs.BuildFSState -- (org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V +- (org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.ProjectStamps,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V - getBuildRootIndex():org.jetbrains.jps.builders.BuildRootIndex - getBuildTargetIndex():org.jetbrains.jps.builders.BuildTargetIndex - getEncodingConfiguration():org.jetbrains.jps.incremental.CompilerEncodingConfiguration @@ -2608,7 +2608,7 @@ a:org.jetbrains.jps.incremental.storage.AbstractStateStorage - appendData(java.lang.Object,java.lang.Object):V - f:clean():V - f:close():V -- flush(Z):V +- f:flush(Z):V - f:force():V - pf:getKeyIterator(java.util.function.Function):java.util.Iterator - getKeysIterator():java.util.Iterator @@ -2623,14 +2623,13 @@ f:org.jetbrains.jps.incremental.storage.BuildDataManager - cleanTargetStorages(org.jetbrains.jps.builders.BuildTarget):V - close():V - closeSourceToOutputStorages(org.jetbrains.jps.builders.impl.BuildTargetChunk):V -- createDependencyGraph(java.io.File,Z):V +- createDependencyGraph(java.nio.file.Path,Z):V - flush(Z):V - getDataPaths():org.jetbrains.jps.builders.storage.BuildDataPaths - getDependencyGraph():org.jetbrains.jps.dependency.GraphConfiguration - getFileStampService():org.jetbrains.jps.incremental.storage.ProjectStamps - getFileStampStorage(org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.incremental.storage.StampsStorage -- s:getMappingsRoot(java.io.File):java.io.File -- getOutputToTargetRegistry():org.jetbrains.jps.incremental.storage.OutputToTargetRegistry +- s:getMappingsRoot(java.nio.file.Path):java.nio.file.Path - getRelativizer():org.jetbrains.jps.incremental.relativizer.PathRelativizerService - getSourceToOutputMap(org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.builders.storage.SourceToOutputMapping - getStorage(org.jetbrains.jps.builders.BuildTarget,org.jetbrains.jps.builders.storage.StorageProvider):org.jetbrains.jps.incremental.storage.StorageOwner @@ -2695,11 +2694,6 @@ f:org.jetbrains.jps.incremental.storage.OneToManyPathsMapping - remove(java.lang.String):V - removeData(java.lang.String,java.lang.String):V - setOutputs(java.lang.String,java.util.List):V -f:org.jetbrains.jps.incremental.storage.OutputToTargetRegistry -- org.jetbrains.jps.incremental.storage.AbstractStateStorage -- getSafeToDeleteOutputs(java.util.Collection,I):java.util.Collection -- removeMapping(java.lang.String,I):V -- removeMapping(java.util.Collection,I):V f:org.jetbrains.jps.incremental.storage.ProjectStamps - sf:PORTABLE_CACHES:Z - sf:PORTABLE_CACHES_PROPERTY:java.lang.String diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java index 99babe6f0505..fc9eb86a892d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java @@ -101,7 +101,8 @@ public final class BuildRunner { storageManager = createStorageManager(dataStorageRoot); fileStampService = initProjectStampStorage(dataStorageRoot, targetsState); - dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager); + dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager); + dataManager.fileStampService = fileStampService; if (dataManager.versionDiffers()) { myForceCleanCaches = true; msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO, @@ -129,20 +130,20 @@ public final class BuildRunner { storageManager = createStorageManager(dataStorageRoot); targetsState = new BuildTargetsState(dataPaths, jpsModel, buildRootIndex); fileStampService = initProjectStampStorage(dataStorageRoot, targetsState); - dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager); + dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager); // the second attempt succeeded msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO, JpsBuildBundle.message("build.message.project.rebuild.forced.0", e.getMessage()))); } return new ProjectDescriptor( - jpsModel, fsState, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex + jpsModel, fsState, fileStampService, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex ); } private static @Nullable StorageManager createStorageManager(@NotNull Path dataStorageRoot) { if (USE_EXPERIMENTAL_STORAGE || ProjectStamps.PORTABLE_CACHES) { - StorageManager manager = new StorageManager(dataStorageRoot.resolve("jps-portable-cache.db"), 10_000); + StorageManager manager = new StorageManager(dataStorageRoot.resolve("jps-portable-cache.db")); manager.open(); return manager; } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java index a1c00bfa44ac..aa0a2dd7e764 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java @@ -1,8 +1,6 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.cmdline; -import org.jetbrains.annotations.ApiStatus; -import org.jetbrains.annotations.TestOnly; import org.jetbrains.jps.builders.BuildRootIndex; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.BuildTargetIndex; @@ -30,10 +28,10 @@ import java.util.Set; public final class ProjectDescriptor { private final JpsProject myProject; private final JpsModel myModel; - @TestOnly - private ProjectStamps deprecatedStamps; public final BuildFSState fsState; public final BuildDataManager dataManager; + private final ProjectStamps myProjectStamps; + private final BuildLoggingManager myLoggingManager; private final ModuleExcludeIndex myModuleExcludeIndex; private int myUseCounter = 1; @@ -43,11 +41,6 @@ public final class ProjectDescriptor { private final BuildTargetIndex myBuildTargetIndex; private final IgnoredFileIndex myIgnoredFileIndex; - /** - * @deprecated Use {@link ProjectDescriptor#ProjectDescriptor(JpsModel, BuildFSState, BuildDataManager, BuildLoggingManager, ModuleExcludeIndex, BuildTargetIndex, BuildRootIndex, IgnoredFileIndex)} - */ - @Deprecated(forRemoval = true) - @ApiStatus.Internal public ProjectDescriptor(JpsModel model, BuildFSState fsState, ProjectStamps projectStamps, @@ -57,22 +50,12 @@ public final class ProjectDescriptor { BuildTargetIndex buildTargetIndex, BuildRootIndex buildRootIndex, IgnoredFileIndex ignoredFileIndex) { - this(model, fsState, dataManager, loggingManager, moduleExcludeIndex, buildTargetIndex, buildRootIndex, ignoredFileIndex); - deprecatedStamps = projectStamps; - } - - public ProjectDescriptor(JpsModel model, - BuildFSState fsState, - BuildDataManager dataManager, - BuildLoggingManager loggingManager, - ModuleExcludeIndex moduleExcludeIndex, - BuildTargetIndex buildTargetIndex, - BuildRootIndex buildRootIndex, - IgnoredFileIndex ignoredFileIndex) { myModel = model; myIgnoredFileIndex = ignoredFileIndex; myProject = model.getProject(); this.fsState = fsState; + myProjectStamps = projectStamps; + dataManager.fileStampService = projectStamps; this.dataManager = dataManager; myBuildTargetIndex = buildTargetIndex; myBuildRootIndex = buildRootIndex; @@ -154,10 +137,6 @@ public final class ProjectDescriptor { */ @Deprecated(forRemoval = true) public ProjectStamps getProjectStamps() { - //noinspection TestOnlyProblems - if (deprecatedStamps != null) { - return deprecatedStamps; - } //noinspection removal return dataManager.getFileStampService(); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java index e3abe9fd6a5e..94fa30099dee 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java @@ -116,17 +116,17 @@ public final class BuildOperations { } private static boolean dropRemovedPaths(CompileContext context, BuildTargetChunk chunk) throws IOException { - final Map, Collection> map = Utils.REMOVED_SOURCES_KEY.get(context); + Map, Collection> map = Utils.REMOVED_SOURCES_KEY.get(context); boolean dropped = false; if (map != null) { for (BuildTarget target : chunk.getTargets()) { - final Collection paths = map.remove(target); + Collection paths = map.remove(target); if (paths != null) { - final SourceToOutputMapping storage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target); + SourceToOutputMapping storage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target); for (String path : paths) { storage.remove(path); - dropped = true; } + dropped = true; } } } @@ -161,26 +161,28 @@ public final class BuildOperations { else { targetId = idsCache.getInt(target); } - final String srcPath = file.getPath(); - final Collection outputs = srcToOut.getOutputs(srcPath); - if (outputs != null) { - final boolean shouldPruneOutputDirs = target instanceof ModuleBasedTarget; - final List deletedForThisSource = new ArrayList<>(outputs.size()); - for (String output : outputs) { - deleteRecursively(output, deletedForThisSource, shouldPruneOutputDirs ? dirsToDelete : null); - } - deletedPaths.addAll(deletedForThisSource); - dataManager.getOutputToTargetRegistry().removeMapping(deletedForThisSource, targetId); - Set cleaned = cleanedSources.get(target); - if (cleaned == null) { - cleaned = FileCollectionFactory.createCanonicalFileSet(); - cleanedSources.put(target, cleaned); - } - cleaned.add(file); + + Collection outputs = srcToOut.getOutputs(file.getPath()); + if (outputs == null) { + return true; } + + boolean shouldPruneOutputDirs = target instanceof ModuleBasedTarget; + List deletedForThisSource = new ArrayList<>(outputs.size()); + for (String output : outputs) { + deleteRecursively(output, deletedForThisSource, shouldPruneOutputDirs ? dirsToDelete : null); + } + + deletedPaths.addAll(deletedForThisSource); + dataManager.getOutputToTargetMapping().removeMappings(deletedForThisSource, targetId, srcToOut); + Set cleaned = cleanedSources.get(target); + if (cleaned == null) { + cleaned = FileCollectionFactory.createCanonicalFileSet(); + cleanedSources.put(target, cleaned); + } + cleaned.add(file); return true; } - }); if (!deletedPaths.isEmpty()) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 60a347c4858d..e31112825433 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -712,29 +712,96 @@ public final class IncProjectBuilder { registerTargetsWithClearedOutput(context, Collections.singletonList(target)); } - private static void clearOutputFiles(CompileContext context, - SourceToOutputMapping mapping, - BuildTargetType targetType, - int targetId) throws IOException { - Set dirsToDelete = targetType instanceof ModuleBasedBuildTargetType ? FileCollectionFactory.createCanonicalFileSet() : null; - OutputToTargetRegistry outputToTargetRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetRegistry(); - for (SourceToOutputMappingCursor cursor = mapping.cursor(); cursor.hasNext(); ) { - cursor.next(); - String [] outs = cursor.getOutputPaths(); - if (outs.length > 0) { - List deletedPaths = new ArrayList<>(); - for (String out : outs) { - BuildOperations.deleteRecursively(out, deletedPaths, dirsToDelete); + private boolean processDeletedPaths(CompileContext context, final Set> targets) throws ProjectBuildException { + boolean doneSomething = false; + try { + // cleanup outputs + final Map, Collection> targetToRemovedSources = new HashMap<>(); + + Set dirsToDelete = FileCollectionFactory.createCanonicalFileSet(); + for (BuildTarget target : targets) { + Collection deletedPaths = myProjectDescriptor.fsState.getAndClearDeletedPaths(target); + if (deletedPaths.isEmpty()) { + continue; } - outputToTargetRegistry.removeMapping(Arrays.asList(outs), targetId); - if (!deletedPaths.isEmpty()) { - context.processMessage(new FileDeletedEvent(deletedPaths)); + + targetToRemovedSources.put(target, deletedPaths); + if (isTargetOutputCleared(context, target)) { + continue; + } + final int buildTargetId = context.getProjectDescriptor().getTargetsState().getBuildTargetId(target); + final boolean shouldPruneEmptyDirs = target instanceof ModuleBasedTarget; + BuildDataManager dataManager = context.getProjectDescriptor().dataManager; + final SourceToOutputMapping sourceToOutputStorage = dataManager.getSourceToOutputMap(target); + final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); + // actually delete outputs associated with removed paths + final Collection pathsForIteration; + if (myIsTestMode) { + // ensure predictable order in test logs + pathsForIteration = new ArrayList<>(deletedPaths); + Collections.sort((List)pathsForIteration); + } + else { + pathsForIteration = deletedPaths; + } + for (String deletedSource : pathsForIteration) { + // deleting outputs corresponding to a non-existing source + Collection outputs = sourceToOutputStorage.getOutputs(deletedSource); + if (outputs != null && !outputs.isEmpty()) { + List deletedOutputPaths = new ArrayList<>(); + OutputToTargetMapping outputToSourceRegistry = dataManager.getOutputToTargetMapping(); + for (String output : outputToSourceRegistry.removeTargetAndGetSafeToDeleteOutputs(outputs, buildTargetId, sourceToOutputStorage)) { + final boolean deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, shouldPruneEmptyDirs ? dirsToDelete : null); + if (deleted) { + doneSomething = true; + } + } + if (!deletedOutputPaths.isEmpty()) { + if (logger.isEnabled()) { + logger.logDeletedFiles(deletedOutputPaths); + } + context.processMessage(new FileDeletedEvent(deletedOutputPaths)); + } + } + + if (target instanceof ModuleBuildTarget) { + // check if the deleted source was associated with a form + OneToManyPathMapping sourceToFormMap = dataManager.getSourceToFormMap(target); + Collection boundForms = sourceToFormMap.getOutputs(deletedSource); + if (boundForms != null) { + for (String formPath : boundForms) { + final File formFile = new File(formPath); + if (formFile.exists()) { + FSOperations.markDirty(context, CompilationRound.CURRENT, formFile); + } + } + sourceToFormMap.remove(deletedSource); + } + } } } - } - if (dirsToDelete != null) { + if (!targetToRemovedSources.isEmpty()) { + final Map, Collection> existing = Utils.REMOVED_SOURCES_KEY.get(context); + if (existing != null) { + for (Map.Entry, Collection> entry : existing.entrySet()) { + final Collection paths = targetToRemovedSources.get(entry.getKey()); + if (paths != null) { + paths.addAll(entry.getValue()); + } + else { + targetToRemovedSources.put(entry.getKey(), entry.getValue()); + } + } + } + Utils.REMOVED_SOURCES_KEY.set(context, targetToRemovedSources); + } + FSOperations.pruneEmptyDirs(context, dirsToDelete); } + catch (IOException e) { + throw new ProjectBuildException(e); + } + return doneSomething; } private static void registerTargetsWithClearedOutput(CompileContext context, Collection> targets) { @@ -1458,98 +1525,29 @@ public final class IncProjectBuilder { myMessageDispatcher.processMessage(new BuildingTargetProgressMessage(targets, event)); } - private boolean processDeletedPaths(CompileContext context, final Set> targets) throws ProjectBuildException { - boolean doneSomething = false; - try { - // cleanup outputs - final Map, Collection> targetToRemovedSources = new HashMap<>(); - - Set dirsToDelete = FileCollectionFactory.createCanonicalFileSet(); - for (BuildTarget target : targets) { - final Collection deletedPaths = myProjectDescriptor.fsState.getAndClearDeletedPaths(target); - if (deletedPaths.isEmpty()) { - continue; + private static void clearOutputFiles(CompileContext context, + SourceToOutputMapping mapping, + BuildTargetType targetType, + int targetId) throws IOException { + Set dirsToDelete = targetType instanceof ModuleBasedBuildTargetType ? FileCollectionFactory.createCanonicalFileSet() : null; + OutputToTargetMapping outputToTargetRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetMapping(); + for (SourceToOutputMappingCursor cursor = mapping.cursor(); cursor.hasNext(); ) { + cursor.next(); + String [] outs = cursor.getOutputPaths(); + if (outs.length > 0) { + List deletedPaths = new ArrayList<>(); + for (String out : outs) { + BuildOperations.deleteRecursively(out, deletedPaths, dirsToDelete); } - targetToRemovedSources.put(target, deletedPaths); - if (isTargetOutputCleared(context, target)) { - continue; - } - final int buildTargetId = context.getProjectDescriptor().getTargetsState().getBuildTargetId(target); - final boolean shouldPruneEmptyDirs = target instanceof ModuleBasedTarget; - BuildDataManager dataManager = context.getProjectDescriptor().dataManager; - final SourceToOutputMapping sourceToOutputStorage = dataManager.getSourceToOutputMap(target); - final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); - // actually delete outputs associated with removed paths - final Collection pathsForIteration; - if (myIsTestMode) { - // ensure predictable order in test logs - pathsForIteration = new ArrayList<>(deletedPaths); - Collections.sort((List)pathsForIteration); - } - else { - pathsForIteration = deletedPaths; - } - for (String deletedSource : pathsForIteration) { - // deleting outputs corresponding to non-existing source - final Collection outputs = sourceToOutputStorage.getOutputs(deletedSource); - if (outputs != null && !outputs.isEmpty()) { - List deletedOutputPaths = new ArrayList<>(); - final OutputToTargetRegistry outputToSourceRegistry = dataManager.getOutputToTargetRegistry(); - for (String output : outputToSourceRegistry.getSafeToDeleteOutputs(outputs, buildTargetId)) { - final boolean deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, shouldPruneEmptyDirs ? dirsToDelete : null); - if (deleted) { - doneSomething = true; - } - } - for (String outputPath : outputs) { - outputToSourceRegistry.removeMapping(outputPath, buildTargetId); - } - if (!deletedOutputPaths.isEmpty()) { - if (logger.isEnabled()) { - logger.logDeletedFiles(deletedOutputPaths); - } - context.processMessage(new FileDeletedEvent(deletedOutputPaths)); - } - } - - if (target instanceof ModuleBuildTarget) { - // check if the deleted source was associated with a form - OneToManyPathMapping sourceToFormMap = dataManager.getSourceToFormMap(target); - Collection boundForms = sourceToFormMap.getOutputs(deletedSource); - if (boundForms != null) { - for (String formPath : boundForms) { - final File formFile = new File(formPath); - if (formFile.exists()) { - FSOperations.markDirty(context, CompilationRound.CURRENT, formFile); - } - } - sourceToFormMap.remove(deletedSource); - } - } + outputToTargetRegistry.removeMappings(Arrays.asList(outs), targetId, mapping); + if (!deletedPaths.isEmpty()) { + context.processMessage(new FileDeletedEvent(deletedPaths)); } } - if (!targetToRemovedSources.isEmpty()) { - final Map, Collection> existing = Utils.REMOVED_SOURCES_KEY.get(context); - if (existing != null) { - for (Map.Entry, Collection> entry : existing.entrySet()) { - final Collection paths = targetToRemovedSources.get(entry.getKey()); - if (paths != null) { - paths.addAll(entry.getValue()); - } - else { - targetToRemovedSources.put(entry.getKey(), entry.getValue()); - } - } - } - Utils.REMOVED_SOURCES_KEY.set(context, targetToRemovedSources); - } - + } + if (dirsToDelete != null) { FSOperations.pruneEmptyDirs(context, dirsToDelete); } - catch (IOException e) { - throw new ProjectBuildException(e); - } - return doneSomething; } // return true if changed something, false otherwise diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java index f3161dbde8d0..52bd8b713ef3 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java @@ -194,11 +194,8 @@ public final class BuildFSState { } public Collection getAndClearDeletedPaths(BuildTarget target) { - final FilesDelta delta = myDeltas.get(target); - if (delta != null) { - return delta.getAndClearDeletedPaths(); - } - return Collections.emptyList(); + FilesDelta delta = myDeltas.get(target); + return delta == null ? List.of() : delta.getAndClearDeletedPaths(); } private @NotNull FilesDelta getDelta(BuildTarget buildTarget) { @@ -238,10 +235,10 @@ public final class BuildFSState { */ public boolean markDirty(@Nullable CompileContext context, File file, - final BuildRootDescriptor rd, + BuildRootDescriptor buildRootDescriptor, @Nullable StampsStorage stampStorage, boolean saveEventStamp) throws IOException { - return markDirty(context, CompilationRound.NEXT, file, rd, stampStorage, saveEventStamp); + return markDirty(context, CompilationRound.NEXT, file, buildRootDescriptor, stampStorage, saveEventStamp); } public boolean markDirty(@Nullable CompileContext context, @@ -384,7 +381,7 @@ public final class BuildFSState { */ public boolean markAllUpToDate(@NotNull CompileContext context, @NotNull BuildRootDescriptor buildRootDescriptor, - @NotNull StampsStorage stampStorage, + @Nullable StampsStorage stampStorage, long targetBuildStartStamp) throws IOException { boolean marked = false; final BuildTarget target = buildRootDescriptor.getTarget(); @@ -415,7 +412,9 @@ public final class BuildFSState { } else { marked = true; - stampStorage.updateStamp(nioFile, target, currentFileTimestamp); + if (stampStorage != null) { + stampStorage.updateStamp(nioFile, target, currentFileTimestamp); + } } } else { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java index c825b1492542..950949e05ecb 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java @@ -244,13 +244,15 @@ public final class FilesDelta { } } - public Set getAndClearDeletedPaths() { + public @NotNull Set getAndClearDeletedPaths() { lockData(); try { + if (myDeletedPaths.isEmpty()) { + return Set.of(); + } + try { - Set result = CollectionFactory.createFilePathLinkedSet(); - result.addAll(myDeletedPaths); - return result; + return CollectionFactory.createFilePathLinkedSet(myDeletedPaths); } finally { myDeletedPaths.clear(); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java index fb17f53e8782..8c1fbd9b6230 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java @@ -6,15 +6,13 @@ import com.intellij.util.io.*; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.function.Function; public abstract class AbstractStateStorage implements StorageOwner { @@ -107,11 +105,28 @@ public abstract class AbstractStateStorage implements StorageOwner { } } + /** + * @deprecated Use {@link #getKeysIterator()} + */ + @TestOnly + @ApiStatus.Internal + @Deprecated(forRemoval = true) + public final Collection getKeys() throws IOException { + return getAllKeys(); + } + public @NotNull Iterator getKeysIterator() throws IOException { + //noinspection TestOnlyProblems + return getAllKeys().iterator(); + } + + @TestOnly + @ApiStatus.Internal + public final @NotNull List getAllKeys() throws IOException { synchronized (dataLock) { List result = new ArrayList<>(); map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result)); - return result.iterator(); + return result.isEmpty() ? List.of() : result; } } @@ -132,7 +147,7 @@ public abstract class AbstractStateStorage implements StorageOwner { } @Override - public void flush(boolean memoryCachesOnly) { + public final void flush(boolean memoryCachesOnly) { if (!memoryCachesOnly) { force(); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index 712b6af2b091..6753b767724d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -42,7 +42,6 @@ import java.util.concurrent.Future; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.stream.Stream; /** @@ -61,10 +60,11 @@ public final class BuildDataManager { private final @NotNull ConcurrentMap, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>(); private final @NotNull ConcurrentMap, SourceToOutputMappingWrapper> buildTargetToSourceToOutputMapping = new ConcurrentHashMap<>(); - // only for new experimental storage - private final @NotNull ConcurrentMap, PerTargetMapManager> targetToMapManager = new ConcurrentHashMap<>(); + private final @Nullable ExperimentalBuildDataManager newDataManager; - private final @Nullable ProjectStamps fileStampService; + // make private after updating bootstrap for Kotlin tests + @ApiStatus.Internal + public @Nullable ProjectStamps fileStampService; private final @Nullable OneToManyPathsMapping sourceToFormMap; private final Mappings myMappings; @@ -73,8 +73,7 @@ public final class BuildDataManager { private final NodeSourcePathMapper myDepGraphPathMapper; private final BuildDataPaths myDataPaths; private final BuildTargetsState myTargetsState; - @Nullable private final StorageManager storageManager; - private final OutputToTargetRegistry myOutputToTargetRegistry; + private final @Nullable OutputToTargetRegistry outputToTargetMapping; private final File myVersionFile; private final PathRelativizerService myRelativizer; private boolean myProcessConstantsIncrementally = !Boolean.parseBoolean(System.getProperty(PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY, "false")); @@ -84,33 +83,33 @@ public final class BuildDataManager { public BuildDataManager(BuildDataPaths dataPaths, BuildTargetsState targetsState, @NotNull PathRelativizerService relativizer) throws IOException { - this(dataPaths, targetsState, relativizer, null, null); + this(dataPaths, targetsState, relativizer, null); } @ApiStatus.Internal public BuildDataManager(BuildDataPaths dataPaths, BuildTargetsState targetsState, @NotNull PathRelativizerService relativizer, - @Nullable ProjectStamps fileStampService, @Nullable StorageManager storageManager) throws IOException { - this.fileStampService = fileStampService; + newDataManager = storageManager == null ? null :new ExperimentalBuildDataManager(storageManager, relativizer); myDataPaths = dataPaths; myTargetsState = targetsState; - this.storageManager = storageManager; + Path dataStorageRoot = myDataPaths.getDataStorageRoot().toPath(); try { sourceToFormMap = storageManager == null ? new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer) : null; - myOutputToTargetRegistry = new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer); - File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot()); + outputToTargetMapping = storageManager == null ? new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer) : null; + Path mappingsRoot = getMappingsRoot(dataStorageRoot); if (JavaBuilderUtil.isDepGraphEnabled()) { myMappings = null; createDependencyGraph(mappingsRoot, false); - FileUtilRt.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), false)); // delete older mappings data if available + // delete older mappings data if available + FileUtilRt.deleteRecursively(getMappingsRoot(dataStorageRoot, false)); LOG.info("Using DependencyGraph-based build incremental analysis"); } else { - myMappings = new Mappings(mappingsRoot, relativizer); - FileUtil.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), true)); // delete dep-graph data if available + myMappings = new Mappings(mappingsRoot.toFile(), relativizer); + FileUtilRt.deleteRecursively(getMappingsRoot(dataStorageRoot, true)); // delete dep-graph data if available myMappings.setProcessConstantsIncrementally(isProcessConstantsIncrementally()); } } @@ -122,15 +121,15 @@ public final class BuildDataManager { } throw e; } - myVersionFile = new File(myDataPaths.getDataStorageRoot(), "version.dat"); + myVersionFile = dataStorageRoot.resolve("version.dat").toFile(); myDepGraphPathMapper = new PathSourceMapper(relativizer::toFull, relativizer::toRelative); myRelativizer = relativizer; } @ApiStatus.Internal public void clearCache() { - if (storageManager != null) { - storageManager.clearCache(); + if (newDataManager != null) { + newDataManager.clearCache(); } } @@ -153,8 +152,8 @@ public final class BuildDataManager { public void cleanStaleTarget(@NotNull BuildTargetType targetType, @NotNull String targetId) throws IOException { try { FileUtilRt.deleteRecursively(getDataPaths().getTargetDataRoot(targetType, targetId)); - if (storageManager != null) { - storageManager.removeMaps(targetId, targetType.getTypeId()); + if (newDataManager != null) { + newDataManager.removeStaleTarget(targetId, targetType.getTypeId()); } } finally { @@ -162,12 +161,23 @@ public final class BuildDataManager { } } - public OutputToTargetRegistry getOutputToTargetRegistry() { - return myOutputToTargetRegistry; + @ApiStatus.Internal + public @NotNull OutputToTargetMapping getOutputToTargetMapping() { + return newDataManager == null ? Objects.requireNonNull(outputToTargetMapping) : newDataManager.getOutputToTargetMapping(); + } + + /** + * @deprecated Use {@link #getOutputToTargetMapping()} + * @return + */ + @ApiStatus.Internal + @Deprecated(forRemoval = true) + public @NotNull OutputToTargetRegistry getOutputToTargetRegistry() { + return Objects.requireNonNull(outputToTargetMapping); } public @NotNull SourceToOutputMapping getSourceToOutputMap(@NotNull BuildTarget target) throws IOException { - if (storageManager == null) { + if (newDataManager == null) { try { return buildTargetToSourceToOutputMapping.computeIfAbsent(target, this::createSourceToOutputMap); } @@ -177,12 +187,12 @@ public final class BuildDataManager { } } else { - return getPerTargetMapManager(target).getSourceToOutputMapping(); + return newDataManager.getSourceToOutputMapping(target); } } private @NotNull SourceToOutputMappingWrapper createSourceToOutputMap(@NotNull BuildTarget target) { - SourceToOutputMapping map; + SourceToOutputMappingImpl map; try { Path file = myDataPaths.getTargetDataRootDir(target).resolve(SRC_TO_OUTPUT_STORAGE).resolve(SRC_TO_OUTPUT_FILE_NAME); map = new SourceToOutputMappingImpl(file, myRelativizer); @@ -191,21 +201,16 @@ public final class BuildDataManager { LOG.info(e); throw new BuildDataCorruptedException(e); } - return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target)); + return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target), outputToTargetMapping); } public @Nullable StampsStorage getFileStampStorage(@NotNull BuildTarget target) { - if (storageManager == null) { + if (newDataManager == null) { return fileStampService == null ? null : fileStampService.getStampStorage(); } - return getPerTargetMapManager(target).stamp; - } - - private @NotNull PerTargetMapManager getPerTargetMapManager(@NotNull BuildTarget target) { - return targetToMapManager.computeIfAbsent(target, it -> { - assert storageManager != null; - return new PerTargetMapManager(storageManager, myRelativizer, it, mapping -> new SourceToOutputMappingWrapper(mapping, myTargetsState.getBuildTargetId(it))); - }); + else { + return newDataManager.getFileStampStorage(target); + } } /** @@ -228,7 +233,7 @@ public final class BuildDataManager { @ApiStatus.Internal public @NotNull OneToManyPathMapping getSourceToFormMap(@NotNull BuildTarget target) { - return sourceToFormMap == null ? getPerTargetMapManager(target).getSourceToForm() : sourceToFormMap; + return newDataManager == null ? Objects.requireNonNull(sourceToFormMap) : newDataManager.getSourceToForm(target); } @ApiStatus.Internal @@ -263,11 +268,13 @@ public final class BuildDataManager { } finally { SourceToOutputMappingWrapper sourceToOutput = buildTargetToSourceToOutputMapping.remove(target); - if (sourceToOutput != null && sourceToOutput.myDelegate instanceof StorageOwner) { + if (sourceToOutput != null && sourceToOutput.myDelegate != null) { ((StorageOwner)sourceToOutput.myDelegate).close(); } - targetToMapManager.remove(target); + if (newDataManager != null) { + newDataManager.closeTargetMaps(target); + } } } finally { @@ -285,7 +292,7 @@ public final class BuildDataManager { } public void clean(@NotNull Consumer> asyncTaskCollector) throws IOException { - if (fileStampService != null) { + if (newDataManager == null && fileStampService != null) { try { ((StorageOwner)fileStampService.getStampStorage()).clean(); } @@ -297,10 +304,9 @@ public final class BuildDataManager { try { allTargetStorages(asyncTaskCollector).clean(); buildTargetToSourceToOutputMapping.clear(); - targetToMapManager.clear(); myTargetStorages.clear(); - if (storageManager != null) { - storageManager.clean(); + if (newDataManager != null) { + newDataManager.removeAllMaps(); } } finally { @@ -311,18 +317,20 @@ public final class BuildDataManager { } finally { try { - wipeStorage(getOutputToSourceRegistryRoot(), myOutputToTargetRegistry); + if (outputToTargetMapping != null) { + wipeStorage(getOutputToSourceRegistryRoot(), outputToTargetMapping); + } } finally { - File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot()); - final Mappings mappings = myMappings; + Path mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot().toPath()); + Mappings mappings = myMappings; if (mappings != null) { synchronized (mappings) { mappings.clean(); } } else { - FileUtil.delete(mappingsRoot); + FileUtilRt.deleteRecursively(mappingsRoot); } if (JavaBuilderUtil.isDepGraphEnabled()) { @@ -335,7 +343,7 @@ public final class BuildDataManager { saveVersion(); } - public void createDependencyGraph(File mappingsRoot, boolean deleteExisting) throws IOException { + public void createDependencyGraph(@NotNull Path mappingsRoot, boolean deleteExisting) throws IOException { try { synchronized (myGraphManagementLock) { DependencyGraph depGraph = myDepGraph; @@ -343,7 +351,7 @@ public final class BuildDataManager { if (deleteExisting) { FileUtil.delete(mappingsRoot); } - myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.getAbsolutePath()))); + myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.toString()))); } else { try { @@ -353,7 +361,7 @@ public final class BuildDataManager { if (deleteExisting) { FileUtil.delete(mappingsRoot); } - myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.getAbsolutePath()))); + myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.toString()))); } } } @@ -368,20 +376,21 @@ public final class BuildDataManager { } public void flush(boolean memoryCachesOnly) { - if (storageManager == null) { + if (newDataManager == null) { if (fileStampService != null) { ((StorageOwner)fileStampService.getStampStorage()).flush(false); } + + assert outputToTargetMapping != null; + outputToTargetMapping.flush(memoryCachesOnly); + assert sourceToFormMap != null; + sourceToFormMap.flush(memoryCachesOnly); } else if (!memoryCachesOnly) { - storageManager.commit(); + newDataManager.commit(); } allTargetStorages().flush(memoryCachesOnly); - myOutputToTargetRegistry.flush(memoryCachesOnly); - if (sourceToFormMap != null) { - sourceToFormMap.flush(memoryCachesOnly); - } Mappings mappings = myMappings; if (mappings != null) { @@ -402,18 +411,12 @@ public final class BuildDataManager { buildTargetToSourceToOutputMapping.clear(); } - if (storageManager != null) { - targetToMapManager.clear(); - try { - storageManager.close(); - } - catch (Throwable e) { - LOG.error(e); - } + if (newDataManager == null) { + flushOldStorages(); } - else if (fileStampService != null) { + else { try { - fileStampService.close(); + newDataManager.close(); } catch (Throwable e) { LOG.error(e); @@ -421,47 +424,62 @@ public final class BuildDataManager { } } finally { - try { - myOutputToTargetRegistry.close(); - } - finally { + Mappings mappings = myMappings; + if (mappings != null) { try { - if (sourceToFormMap != null) { - synchronized (sourceToFormMap) { - sourceToFormMap.close(); - } - } + mappings.close(); } - finally { - final Mappings mappings = myMappings; - if (mappings != null) { - try { - mappings.close(); - } - catch (BuildDataCorruptedException e) { - throw e.getCause(); - } - } + catch (BuildDataCorruptedException e) { + throw e.getCause(); + } + } - synchronized (myGraphManagementLock) { - DependencyGraph depGraph = myDepGraph; - if (depGraph != null) { - myDepGraph = null; - try { - depGraph.close(); - } - catch (BuildDataCorruptedException e) { - throw e.getCause(); - } - } + synchronized (myGraphManagementLock) { + DependencyGraph depGraph = myDepGraph; + if (depGraph != null) { + myDepGraph = null; + try { + depGraph.close(); + } + catch (BuildDataCorruptedException e) { + throw e.getCause(); } } } } } + private void flushOldStorages() { + if (fileStampService != null) { + try { + fileStampService.close(); + } + catch (Throwable e) { + LOG.error(e); + } + } + + try { + assert outputToTargetMapping != null; + outputToTargetMapping.close(); + } + catch (Throwable e) { + LOG.error(e); + } + + try { + assert sourceToFormMap != null; + synchronized (sourceToFormMap) { + sourceToFormMap.close(); + } + } + catch (Throwable e) { + LOG.error(e); + } + } + public void closeSourceToOutputStorages(@NotNull BuildTargetChunk chunk) throws IOException { - if (storageManager != null) { + if (newDataManager != null) { for (BuildTarget target : chunk.getTargets()) { buildTargetToSourceToOutputMapping.remove(target); } @@ -476,7 +494,7 @@ public final class BuildDataManager { continue; } - StorageOwner delegate = (StorageOwner)sourceToOutputMapping.myDelegate; + StorageOwner delegate = sourceToOutputMapping.myDelegate; try { delegate.close(); } @@ -518,12 +536,12 @@ public final class BuildDataManager { return myRelativizer; } - public static File getMappingsRoot(final File dataStorageRoot) { + public static @NotNull Path getMappingsRoot(@NotNull Path dataStorageRoot) { return getMappingsRoot(dataStorageRoot, JavaBuilderUtil.isDepGraphEnabled()); } - private static File getMappingsRoot(final File dataStorageRoot, boolean forDepGraph) { - return new File(dataStorageRoot, forDepGraph? MAPPINGS_STORAGE + "-graph" : MAPPINGS_STORAGE); + private static Path getMappingsRoot(@NotNull Path dataStorageRoot, boolean forDepGraph) { + return dataStorageRoot.resolve(forDepGraph? MAPPINGS_STORAGE + "-graph" : MAPPINGS_STORAGE); } private static void wipeStorage(@NotNull Path root, @Nullable StorageOwner storage) { @@ -587,18 +605,48 @@ public final class BuildDataManager { myRelativizer.reportUnhandledPaths(); } - final class SourceToOutputMappingWrapper implements SourceToOutputMapping, Supplier<@Nullable StorageOwner> { - private final SourceToOutputMapping myDelegate; - private final int myBuildTargetId; + private @NotNull StorageOwner allTargetStorages(@NotNull Consumer> asyncTaskCollector) { + return new CompositeStorageOwner() { + @Override + public void clean() throws IOException { + try { + close(); + } + finally { + asyncTaskCollector.accept(FileUtil.asyncDelete(myDataPaths.getTargetsDataRoot())); + } + } - SourceToOutputMappingWrapper(SourceToOutputMapping delegate, int buildTargetId) { + @Override + protected Iterable getChildStorages() { + return () -> { + return Stream.concat( + myTargetStorages.values().stream(), + buildTargetToSourceToOutputMapping.values().stream() + .map(wrapper -> { + SourceToOutputMapping delegate = wrapper.myDelegate; + return delegate == null ? null : (StorageOwner)delegate; + }) + .filter(o -> o != null) + ).iterator(); + }; + } + }; + } + + private @NotNull StorageOwner allTargetStorages() { + return allTargetStorages(f -> {}); + } + + private static final class SourceToOutputMappingWrapper implements SourceToOutputMapping { + private final SourceToOutputMappingImpl myDelegate; + private final int myBuildTargetId; + private final OutputToTargetRegistry outputToTargetMapping; + + SourceToOutputMappingWrapper(SourceToOutputMappingImpl delegate, int buildTargetId, OutputToTargetRegistry outputToTargetMapping) { myDelegate = delegate; myBuildTargetId = buildTargetId; - } - - @Override - public @Nullable StorageOwner get() { - return myDelegate instanceof StorageOwner? (StorageOwner)myDelegate : null; + this.outputToTargetMapping = outputToTargetMapping; } @Override @@ -607,7 +655,7 @@ public final class BuildDataManager { myDelegate.setOutputs(srcPath, outputs); } finally { - myOutputToTargetRegistry.addMapping(outputs, myBuildTargetId); + outputToTargetMapping.addMappings(outputs, myBuildTargetId); } } @@ -617,7 +665,7 @@ public final class BuildDataManager { myDelegate.setOutput(srcPath, outputPath); } finally { - myOutputToTargetRegistry.addMapping(outputPath, myBuildTargetId); + outputToTargetMapping.addMapping(outputPath, myBuildTargetId); } } @@ -627,7 +675,7 @@ public final class BuildDataManager { myDelegate.appendOutput(srcPath, outputPath); } finally { - myOutputToTargetRegistry.addMapping(outputPath, myBuildTargetId); + outputToTargetMapping.addMapping(outputPath, myBuildTargetId); } } @@ -657,39 +705,6 @@ public final class BuildDataManager { } } - private @NotNull StorageOwner allTargetStorages() { - return allTargetStorages(f -> {}); - } - - private @NotNull StorageOwner allTargetStorages(@NotNull Consumer> asyncTaskCollector) { - return new CompositeStorageOwner() { - @Override - public void clean() throws IOException { - try { - close(); - } - finally { - asyncTaskCollector.accept(FileUtil.asyncDelete(myDataPaths.getTargetsDataRoot())); - } - } - - @Override - protected Iterable getChildStorages() { - return () -> { - return Stream.concat( - myTargetStorages.values().stream(), - buildTargetToSourceToOutputMapping.values().stream() - .map(wrapper -> { - SourceToOutputMapping delegate = wrapper.myDelegate; - return delegate instanceof StorageOwner ? (StorageOwner)delegate : null; - }) - .filter(o -> o != null) - ).iterator(); - }; - } - }; - } - private static DependencyGraph asSynchronizedGraph(DependencyGraph graph) { //noinspection IOResourceOpenedButNotSafelyClosed DependencyGraph delegate = new LoggingDependencyGraph(graph, msg -> LOG.info(msg)); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalBuildDataManager.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalBuildDataManager.kt new file mode 100644 index 000000000000..06c964cf62d0 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalBuildDataManager.kt @@ -0,0 +1,74 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage + +import com.intellij.util.concurrency.SynchronizedClearableLazy +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.storage.SourceToOutputMapping +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import java.util.concurrent.ConcurrentHashMap + +internal class ExperimentalBuildDataManager( + private val storageManager: StorageManager, + private val relativizer: PathRelativizerService, +) { + // only for new experimental storage + private val targetToMapManager = ConcurrentHashMap, PerTargetMapManager>() + + /** + * A map not scoped to a target is problematic because we cannot transfer built target bytecode with JPS build data. + * Normally, a source file in a module isn't expected to compile into multiple output directories. + * We can address this later, but for now, let's support it as the old storage did. + */ + private val outputToTargetMapping = SynchronizedClearableLazy { + ExperimentalOutputToTargetMapping(storageManager) + } + + fun clearCache() { + storageManager.clearCache() + } + + fun removeStaleTarget(targetId: String, targetTypeId: String) { + storageManager.removeMaps(targetId, targetTypeId) + outputToTargetMapping.value.removeTarget(targetId, targetTypeId) + } + + private fun getPerTargetMapManager(target: BuildTarget<*>): PerTargetMapManager { + return targetToMapManager.computeIfAbsent(target) { + PerTargetMapManager(storageManager, relativizer, it, outputToTargetMapping) + } + } + + fun getFileStampStorage(target: BuildTarget<*>): StampsStorage<*> { + return getPerTargetMapManager(target).stamp + } + + fun getSourceToOutputMapping(target: BuildTarget<*>): SourceToOutputMapping { + return getPerTargetMapManager(target).sourceToOutputMapping + } + + fun getOutputToTargetMapping(): OutputToTargetMapping = outputToTargetMapping.value + + fun getSourceToForm(target: BuildTarget<*>): ExperimentalOneToManyPathMapping { + return getPerTargetMapManager(target).sourceToForm + } + + fun closeTargetMaps(target: BuildTarget<*>) { + targetToMapManager.remove(target) + } + + fun removeAllMaps() { + outputToTargetMapping.drop() + targetToMapManager.clear() + storageManager.clean() + } + + fun commit() { + storageManager.commit() + } + + fun close() { + outputToTargetMapping.drop() + targetToMapManager.clear() + storageManager.close() + } +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt index a56638d60792..f77f29e5a05a 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt @@ -10,7 +10,7 @@ import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash @ApiStatus.Internal open class ExperimentalOneToManyPathMapping( @JvmField protected val mapHandle: MapHandle>, - @JvmField protected val relativizer: PathRelativizerService, + @JvmField internal val relativizer: PathRelativizerService, private val valueOffset: Int = 0, ) : OneToManyPathMapping { protected fun getKey(path: String): LongArray = stringTo128BitHash(relativizer.toRelative(path)) @@ -22,23 +22,31 @@ open class ExperimentalOneToManyPathMapping( return Array(list.size - valueOffset) { relativizer.toFull(list.get(it + valueOffset)) }.asList() } - final override fun setOutputs(path: String, outPaths: List) { + protected fun normalizeOutputPaths(outPaths: List, relativeSourcePath: String?): Array? { + return when { + outPaths.isEmpty() -> null + relativeSourcePath != null -> { + Array(outPaths.size + 1) { + if (it == 0) relativeSourcePath else relativizer.toRelative(outPaths.get(it - 1)) + } + } + else -> { + Array(outPaths.size) { + relativizer.toRelative(outPaths.get(it)) + } + } + } + } + + override fun setOutputs(path: String, outPaths: List) { val relativeSourcePath = relativizer.toRelative(path) val key = stringTo128BitHash(relativeSourcePath) - if (outPaths.isEmpty()) { + val normalizeOutputPaths = normalizeOutputPaths(outPaths, null) + if (normalizeOutputPaths == null) { mapHandle.map.remove(key) } - else if (valueOffset == 1) { - val listWithRelativePaths = Array(outPaths.size + 1) { - if (it == 0) relativeSourcePath else relativizer.toRelative(outPaths.get(it - 1)) - } - mapHandle.map.put(key, listWithRelativePaths) - } else { - val listWithRelativePaths = Array(outPaths.size) { - relativizer.toRelative(outPaths.get(it)) - } - mapHandle.map.put(key, listWithRelativePaths) + mapHandle.map.put(key, normalizeOutputPaths) } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOutputToTargetMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOutputToTargetMapping.kt new file mode 100644 index 000000000000..3671add12a68 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOutputToTargetMapping.kt @@ -0,0 +1,163 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage + +import com.dynatrace.hash4j.hashing.Hashing +import org.h2.mvstore.MVMap.Decision +import org.h2.mvstore.MVMap.DecisionMaker +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.jps.builders.storage.SourceToOutputMapping +import org.jetbrains.jps.incremental.storage.dataTypes.LongListKeyDataType +import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType +import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash + +@ApiStatus.Internal +class ExperimentalOutputToTargetMapping( + storageManager: StorageManager, +) : OutputToTargetMapping { + private val mapHandle = storageManager.openMap("out-to-target-v1", LongPairKeyDataType, LongListKeyDataType) + + override fun removeTargetAndGetSafeToDeleteOutputs( + outputPaths: Collection, + currentTargetId: Int, + srcToOut: SourceToOutputMapping, + ): Collection { + val size = outputPaths.size + if (size == 0) { + return emptyList() + } + + srcToOut as ExperimentalSourceToOutputMapping + val decisionMaker = LongListRemoveItemDecisionMaker(srcToOut.targetHashId) + val relativizer = srcToOut.relativizer + + val result = ArrayList(size) + for (outPath in outputPaths) { + val key = stringTo128BitHash(relativizer.toRelative(outPath)) + mapHandle.map.operate(key, null, decisionMaker) + if (!decisionMaker.outStillUsed) { + result.add(outPath) + } + } + return result + } + + override fun removeMappings(outputPaths: Collection, buildTargetId: Int, srcToOut: SourceToOutputMapping) { + srcToOut as ExperimentalSourceToOutputMapping + val relativizer = srcToOut.relativizer + val decisionMaker = LongListRemoveItemDecisionMaker(srcToOut.targetHashId) + for (outPath in outputPaths) { + val key = stringTo128BitHash(relativizer.toRelative(outPath)) + mapHandle.map.operate(key, null, decisionMaker) + } + } + + fun addMappings(normalizeOutputPaths: Array, targetHashId: Long) { + val decisionMaker = LongListAddItemDecisionMaker(targetHashId) + for (outPath in normalizeOutputPaths) { + mapHandle.map.operate(stringTo128BitHash(outPath), null, decisionMaker) + } + } + + fun addMapping(normalizeOutputPath: String, targetHashId: Long) { + val decisionMaker = LongListAddItemDecisionMaker(targetHashId) + mapHandle.map.operate(stringTo128BitHash(normalizeOutputPath), null, decisionMaker) + } + + fun removeTarget(targetId: String, targetTypeId: String) { + val map = mapHandle.map + val iterator = map.cursor(null) + val decisionMaker = LongListRemoveItemDecisionMaker(targetToHash(targetId, targetTypeId)) + while (iterator.hasNext()) { + mapHandle.map.operate(iterator.next(), null, decisionMaker) + } + } +} + +private class LongListAddItemDecisionMaker(private val toAdd: Long) : DecisionMaker() { + override fun decide(existingValue: LongArray?, providedValue: LongArray?): Decision { + return when { + existingValue == null || existingValue.isEmpty() -> Decision.PUT + existingValue.contains(toAdd) -> Decision.ABORT + else -> Decision.PUT + } + } + + // it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel + override fun selectValue(existingValue: T?, ignore: T?): T? { + if (existingValue == null || existingValue.isEmpty()) { + @Suppress("UNCHECKED_CAST") + return longArrayOf(toAdd) as T + } + else { + // we checked `contains` in `decide` + @Suppress("UNCHECKED_CAST") + return addElementToEnd(existingValue, toAdd) as T? + } + } +} + +private class LongListRemoveItemDecisionMaker(private val toRemove: Long) : DecisionMaker() { + private var indexToRemove: Int = -1 + @JvmField + var outStillUsed = false + + override fun reset() { + indexToRemove = -1 + outStillUsed = false + } + + override fun decide(existingValue: LongArray?, ignore: LongArray?): Decision { + return when { + existingValue == null -> Decision.ABORT + // empty value list is not normal, recover - just delete record + existingValue.isEmpty() -> Decision.REMOVE + else -> { + indexToRemove = existingValue.indexOf(toRemove) + when { + indexToRemove == -1 -> { + outStillUsed = true + Decision.ABORT + } + existingValue.size == 1 -> Decision.REMOVE + else -> { + outStillUsed = true + Decision.PUT + } + } + } + } + } + + // it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel + override fun selectValue(existingValue: T, providedValue: T?): T { + assert(indexToRemove != -1) + @Suppress("UNCHECKED_CAST") + return removeElementAtIndex(existingValue!!, indexToRemove) as T + } +} + +private fun removeElementAtIndex(old: LongArray, index: Int): LongArray { + val newSize = old.size - 1 + val result = LongArray(newSize) + System.arraycopy(old, 0, result, 0, index) + if (index < newSize) { + System.arraycopy(old, index + 1, result, index, newSize - index) + } + return result +} + +private fun addElementToEnd(old: LongArray, element: Long): LongArray { + val result = LongArray(old.size + 1) + System.arraycopy(old, 0, result, 0, old.size) + result[old.size] = element + return result +} + +internal fun targetToHash(targetId: String, targetTypeId: String): Long { + val b1 = targetId.toByteArray() + val b2 = targetTypeId.toByteArray() + return Hashing.xxh3_64().hashStream() + .putByteArray(b1) + .putByteArray(b2) + .asLong +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt index c1a7e17b4312..0f419cf30095 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt @@ -7,7 +7,6 @@ import org.h2.mvstore.MVMap.DecisionMaker import org.jetbrains.annotations.ApiStatus.Internal import org.jetbrains.annotations.TestOnly import org.jetbrains.annotations.VisibleForTesting -import org.jetbrains.jps.builders.BuildTarget import org.jetbrains.jps.builders.storage.SourceToOutputMapping import org.jetbrains.jps.incremental.relativizer.PathRelativizerService import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType @@ -18,6 +17,8 @@ import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash class ExperimentalSourceToOutputMapping private constructor( mapHandle: MapHandle>, relativizer: PathRelativizerService, + private val outputToTargetMapping: ExperimentalOutputToTargetMapping?, + @JvmField internal val targetHashId: Long, ) : SourceToOutputMapping, ExperimentalOneToManyPathMapping(mapHandle = mapHandle, relativizer = relativizer, valueOffset = 1) { companion object { // we have a lot of targets - reduce GC and reuse map builder @@ -26,19 +27,6 @@ class ExperimentalSourceToOutputMapping private constructor( it.setValueType(StringListDataType) } - fun createSourceToOutputMap( - storageManager: StorageManager, - relativizer: PathRelativizerService, - target: BuildTarget<*>, - ): ExperimentalSourceToOutputMapping { - return createSourceToOutputMap( - storageManager = storageManager, - relativizer = relativizer, - targetId = target.id, - targetTypeId = target.targetType.typeId, - ) - } - @VisibleForTesting @Internal fun createSourceToOutputMap( @@ -46,6 +34,7 @@ class ExperimentalSourceToOutputMapping private constructor( relativizer: PathRelativizerService, targetId: String, targetTypeId: String, + outputToTargetMapping: ExperimentalOutputToTargetMapping?, ): ExperimentalSourceToOutputMapping { // we can use composite key and sort by target id, but as we compile targets in parallel: // * avoid blocking - in-memory lock per map root, @@ -54,32 +43,39 @@ class ExperimentalSourceToOutputMapping private constructor( return ExperimentalSourceToOutputMapping( mapHandle = storageManager.openMap(mapName, mapBuilder), relativizer = relativizer, + outputToTargetMapping = outputToTargetMapping, + targetHashId = targetToHash(targetId, targetTypeId), ) } } - //override fun getKey(sourcePath: String): LongArray { - // val stringKey = relativizer.toRelative(sourcePath).toByteArray() - // - // // We should sort by target id for data locality - - // // the high integer (high) is shifted to the higher 32 bits of the long value, - // // while the low integer (low) occupies the lower 32 bits. - // // When we do compare two long values, the higher 32 bits (which contain the high integer) dictate the order. - // val low = Hashing.komihash5_0().hashBytesToInt(stringKey) - // val high = targetId - // return longArrayOf((high.toLong() shl 32) or (low.toLong() and 0xFFFFFFFFL), Hashing.xxh3_64().hashBytesToLong(stringKey)) - //} + override fun setOutputs(path: String, outPaths: List) { + val relativeSourcePath = super.relativizer.toRelative(path) + val key = stringTo128BitHash(relativeSourcePath) + val normalizeOutputPaths = super.normalizeOutputPaths(outPaths, relativeSourcePath) + if (normalizeOutputPaths == null) { + mapHandle.map.remove(key) + } + else { + mapHandle.map.put(key, normalizeOutputPaths) + outputToTargetMapping?.addMappings(normalizeOutputPaths, targetHashId) + } + } override fun setOutput(sourcePath: String, outputPath: String) { val relativeSourcePath = relativizer.toRelative(sourcePath) - mapHandle.map.put(stringTo128BitHash(relativeSourcePath), arrayOf(relativeSourcePath, relativizer.toRelative(outputPath))) + val relativeOutputPath = relativizer.toRelative(outputPath) + mapHandle.map.put(stringTo128BitHash(relativeSourcePath), arrayOf(relativeSourcePath, relativeOutputPath)) + outputToTargetMapping?.addMapping(relativeOutputPath, targetHashId) } override fun appendOutput(sourcePath: String, outputPath: String) { val relativeSourcePath = relativizer.toRelative(sourcePath) + val relativeOutputPath = relativizer.toRelative(outputPath) mapHandle.map.operate(stringTo128BitHash(relativeSourcePath), null, - AddItemDecisionMaker(sourcePath = relativeSourcePath, toAdd = relativizer.toRelative(outputPath))) + AddItemDecisionMaker(sourcePath = relativeSourcePath, toAdd = relativeOutputPath)) + outputToTargetMapping?.addMapping(relativeOutputPath, targetHashId) } override fun removeOutput(sourcePath: String, outputPath: String) { @@ -157,6 +153,10 @@ private class AddItemDecisionMaker(private val sourcePath: String, private val t private class RemoveItemDecisionMaker(private val toRemove: String) : DecisionMaker>() { private var indexToRemove: Int = -1 + override fun reset() { + indexToRemove = -1 + } + override fun decide(existingValue: Array?, ignore: Array?): Decision { return when { existingValue == null -> Decision.ABORT @@ -197,8 +197,8 @@ private fun removeElementAtIndex(old: Array, index: Int): Array } private fun addElementToEnd(old: Array, element: String): Array { - val result = arrayOfNulls(old.size + 1) - System.arraycopy(old, 0, result, 0, old.size) - result[old.size] = element - return result + val result = arrayOfNulls(old.size + 1) + System.arraycopy(old, 0, result, 0, old.size) + result[old.size] = element + return result } \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt index c7f3e1f3c18c..8ef20215ce5b 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt @@ -2,6 +2,7 @@ package org.jetbrains.jps.incremental.storage import org.jetbrains.annotations.ApiStatus +import org.jetbrains.jps.builders.storage.SourceToOutputMapping import java.io.IOException internal interface OneToManyPathMapping { @@ -19,4 +20,17 @@ internal interface OneToManyPathMapping { interface SourceToOutputMappingCursor : Iterator { /** [next] must be called beforehand */ val outputPaths: Array +} + +@ApiStatus.Internal +interface OutputToTargetMapping { + @Throws(IOException::class) + fun removeTargetAndGetSafeToDeleteOutputs( + outputPaths: Collection, + currentTargetId: Int, + srcToOut: SourceToOutputMapping, + ): Collection + + @Throws(IOException::class) + fun removeMappings(outputPaths: Collection, buildTargetId: Int, srcToOut: SourceToOutputMapping) } \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java index cd9c4f153c54..adb05ae76bd3 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java @@ -10,7 +10,9 @@ import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; import it.unimi.dsi.fastutil.ints.IntSets; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; +import org.jetbrains.jps.builders.storage.SourceToOutputMapping; import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; import java.io.DataInput; @@ -21,10 +23,10 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; -public final class OutputToTargetRegistry extends AbstractStateStorage { +@ApiStatus.Internal +public final class OutputToTargetRegistry extends AbstractStateStorage implements OutputToTargetMapping { private static final DataExternalizer DATA_EXTERNALIZER = new DataExternalizer<>() { @Override public void save(@NotNull DataOutput out, IntSet value) throws IOException { @@ -56,27 +58,25 @@ public final class OutputToTargetRegistry extends AbstractStateStorage outputPaths, int buildTargetId) throws IOException { + void addMappings(@NotNull Collection outputPaths, int buildTargetId) throws IOException { IntSet set = IntSets.singleton(buildTargetId); for (String outputPath : outputPaths) { appendData(pathHashCode(outputPath), set); } } - public void removeMapping(String outputPath, int buildTargetId) throws IOException { - removeMapping(Collections.singleton(outputPath), buildTargetId); - } - - public void removeMapping(Collection outputPaths, int buildTargetId) throws IOException { + @Override + public void removeMappings(@NotNull Collection outputPaths, int buildTargetId, @NotNull SourceToOutputMapping srcToOut) throws IOException { if (outputPaths.isEmpty()) { return; } + for (String outputPath : outputPaths) { - final int key = pathHashCode(outputPath); + int key = pathHashCode(outputPath); synchronized (dataLock) { - final IntSet state = getState(key); + IntSet state = getState(key); if (state != null) { - final boolean removed = state.remove(buildTargetId); + boolean removed = state.remove(buildTargetId); if (state.isEmpty()) { remove(key); } @@ -88,7 +88,10 @@ public final class OutputToTargetRegistry extends AbstractStateStorage getSafeToDeleteOutputs(Collection outputPaths, int currentTargetId) throws IOException { + @Override + public @NotNull Collection removeTargetAndGetSafeToDeleteOutputs(Collection outputPaths, + int currentTargetId, + @NotNull SourceToOutputMapping srcToOut) throws IOException { int size = outputPaths.size(); if (size == 0) { return outputPaths; @@ -98,11 +101,22 @@ public final class OutputToTargetRegistry extends AbstractStateStorage, - sourceToOutputMappingWrapper: UnaryOperator, + outputToTargetMapping: Supplier, ) { @JvmField val stamp: StampsStorage<*> = if (ProjectStamps.PORTABLE_CACHES) { @@ -33,11 +33,13 @@ internal class PerTargetMapManager( } val sourceToOutputMapping: SourceToOutputMapping by lazy { - sourceToOutputMappingWrapper.apply(ExperimentalSourceToOutputMapping.createSourceToOutputMap( + ExperimentalSourceToOutputMapping.createSourceToOutputMap( storageManager = storageManager, relativizer = relativizer, - target = target, - )) + targetId = target.id, + targetTypeId = target.targetType.typeId, + outputToTargetMapping = outputToTargetMapping.get(), + ) } val sourceToForm: ExperimentalOneToManyPathMapping by lazy { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt index d5937da1ebcf..107a5de13833 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt @@ -19,7 +19,7 @@ private val MV_STORE_CACHE_SIZE_IN_MB = System.getProperty("jps.new.storage.cach private val LOG = logger() @ApiStatus.Internal -class StorageManager(@JvmField val file: Path, private val allowedCompactionTimeOnClose: Int) { +class StorageManager constructor(@JvmField val file: Path) { private val storeValue = SynchronizedClearableLazy { LOG.debug { "Opening storage $file" } createOrResetMvStore(file = file, readOnly = false, logSupplier = { LOG }) @@ -61,8 +61,8 @@ class StorageManager(@JvmField val file: Path, private val allowedCompactionTime storeValue.valueIfInitialized?.let { storeValue.drop() - val isCompactOnClose = System.getProperty("jps.new.storage.compact.on.close", "true").toBoolean() - it.close(if (isCompactOnClose) 0 else allowedCompactionTimeOnClose) + val isCompactOnClose = System.getProperty("jps.new.storage.compact.on.close", "false").toBoolean() + it.close() if (isCompactOnClose && Files.exists(file)) { val time = measureTime { MVStoreTool.compact(file.toString(), false) @@ -94,10 +94,10 @@ class StorageManager(@JvmField val file: Path, private val allowedCompactionTime } } - fun removeMaps(targetId: String, typeId: String) { + fun removeMaps(targetId: String, targetTypeId: String) { val store = storeValue.value for (mapName in store.mapNames) { - if (mapName.startsWith(getMapName(targetId = targetId, targetTypeId = typeId, suffix = ""))) { + if (mapName.startsWith(getMapName(targetId = targetId, targetTypeId = targetTypeId, suffix = ""))) { store.removeMap(mapName) } } @@ -166,20 +166,29 @@ private fun tryOpenMvStore(file: Path?, readOnly: Boolean, logSupplier: () -> Lo val store = MVStore.Builder() .fileName(file?.toAbsolutePath()?.toString()) .backgroundExceptionHandler(storeErrorHandler) - // We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread. - // Additionally, we ensure that the write operation will not slow down any tasks, - // as the actual save will be done in a background thread. - // Use an 8MB threshold for auto-commit instead of the default 1MB - - // if writes are performed too often, do not save intermediate B-Tree pages to disk. - .autoCommitBufferSize(8192) .cacheSize(MV_STORE_CACHE_SIZE_IN_MB) .let { if (readOnly) it.readOnly() else it } + // We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread. + // Additionally, we ensure that the write operation will not slow down any tasks, + // as the actual save will be done in a background thread. + // Use a 16MB BUFFER for auto-commit instead of the default 1MB - + // if writes are performed too often, do not save intermediate B-Tree pages to disk. + // Or... just disable auto-commit based on the size of unsaved data and save once in 1 minute + .autoCommitBufferSize(0) .open() storeErrorHandler.isStoreOpened = true // versioning isn't required, otherwise the file size will be larger than needed store.setVersionsToKeep(0) + + // We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread. + // Additionally, we ensure that the write operation will not slow down any tasks, + // as the actual save will be done in a background thread. + // Use a 16MB BUFFER for auto-commit instead of the default 1MB - + // if writes are performed too often, do not save intermediate B-Tree pages to disk. + // Or... just disable auto-commit based on the size of unsaved data and save once in 1 minute + store.autoCommitDelay = 60_000 return store } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongListKeyDataType.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongListKeyDataType.kt new file mode 100644 index 000000000000..c2dc1883351e --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongListKeyDataType.kt @@ -0,0 +1,45 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage.dataTypes + +import org.h2.mvstore.DataUtils.readVarInt +import org.h2.mvstore.WriteBuffer +import org.h2.mvstore.type.DataType +import java.nio.ByteBuffer + +internal object LongListKeyDataType : DataType { + override fun isMemoryEstimationAllowed() = true + + // don't care about non-ASCII strings for memory estimation + override fun getMemory(obj: LongArray): Int = obj.size * Long.SIZE_BYTES + + override fun createStorage(size: Int): Array = arrayOfNulls(size) + + override fun write(buff: WriteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + for (value in (storage as Array)) { + buff.putVarInt(value.size) + for (item in value) { + buff.putLong(item) + } + } + } + + override fun write(buff: WriteBuffer, obj: LongArray) = throw IllegalStateException("Must not be called") + + override fun read(buff: ByteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + storage as Array + for (i in 0 until len) { + storage[i] = LongArray(readVarInt(buff)) { + buff.getLong() + } + } + } + + override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called") + + override fun binarySearch(key: LongArray, storage: Any, size: Int, initialGuess: Int): Int = throw IllegalStateException("Must not be called") + + @Suppress("DuplicatedCode") + override fun compare(a: LongArray, b: LongArray): Int = throw IllegalStateException("Must not be called") +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt index 7e93e7bee70d..2c054c8b40f1 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt @@ -17,15 +17,15 @@ internal object LongPairKeyDataType : DataType { override fun isMemoryEstimationAllowed() = true // don't care about non-ASCII strings for memory estimation - override fun getMemory(obj: LongArray): Int = 16 + override fun getMemory(obj: LongArray): Int = 2 * Long.SIZE_BYTES override fun createStorage(size: Int): Array = arrayOfNulls(size) override fun write(buff: WriteBuffer, storage: Any, len: Int) { @Suppress("UNCHECKED_CAST") - for (key in (storage as Array)) { - buff.putLong(key[0]) - buff.putLong(key[1]) + for (value in (storage as Array)) { + buff.putLong(value[0]) + buff.putLong(value[1]) } } diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java index 295c1b90916b..5ceb1e6a1b57 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java @@ -6,7 +6,6 @@ import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntSet; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import org.jetbrains.annotations.NotNull; @@ -81,19 +80,24 @@ public final class BuildResult implements MessageHandler { } - OutputToTargetRegistry registry = pd.dataManager.getOutputToTargetRegistry(); - List keys = new IntArrayList(registry.getKeysIterator()); - Collections.sort(keys); + OutputToTargetRegistry registry = (OutputToTargetRegistry)pd.dataManager.getOutputToTargetMapping(); + List keys = registry.getAllKeys(); + if (keys.size() > 1) { + keys.sort(null); + } stream.println("Begin Of OutputToTarget"); for (Integer key : keys) { IntSet targetsIds = registry.getState(key); - if (targetsIds == null) continue; - final List targetsNames = new ArrayList<>(); + if (targetsIds == null) { + continue; + } + + List targetsNames = new ArrayList<>(); targetsIds.forEach(value -> { BuildTarget target = id2Target.get(value); targetsNames.add(target != null ? getTargetIdWithTypeId(target) : ""); }); - Collections.sort(targetsNames); + targetsNames.sort(null); stream.println(hashCodeToOutputPath.get(key.intValue()) + " -> " + targetsNames); } stream.println("End Of OutputToTarget"); diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java index bce188f0be61..337c971aa29d 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java @@ -201,8 +201,8 @@ public abstract class JpsBuildTestCase extends UsefulTestCase { BuildTargetsState targetsState = new BuildTargetsState(dataPaths, myModel, buildRootIndex); PathRelativizerService relativizer = new PathRelativizerService(myModel.getProject()); ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState); - BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, projectStamps, null); - return new ProjectDescriptor(myModel, new BuildFSState(true), dataManager, buildLoggingManager, index, + BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, null); + return new ProjectDescriptor(myModel, new BuildFSState(true), projectStamps, dataManager, buildLoggingManager, index, targetIndex, buildRootIndex, ignoredFileIndex); } catch (IOException e) { diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt index b3a1a9d300fd..9c0fac3fdae4 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt @@ -25,7 +25,7 @@ class HashStampStorageFuzzTest { @BeforeProperty fun setUp() { file = Files.createTempFile("mvstore", ".db") - storageManager = StorageManager(file!!, 0) + storageManager = StorageManager(file!!) hashStampStorage = HashStampStorage.createSourceToStampMap( storageManager = storageManager, relativizer = PathRelativizerService(), diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt index bbed981d8d54..9181cca072f1 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt @@ -18,18 +18,21 @@ class SourceToOutputMappingFuzzTest { } private lateinit var mapping: ExperimentalSourceToOutputMapping + private lateinit var targetMapping: ExperimentalOutputToTargetMapping private lateinit var storageManager: StorageManager private var file: Path? = null @BeforeProperty fun setUp() { file = Files.createTempFile("mvstore", ".db") - storageManager = StorageManager(file!!, 0) + storageManager = StorageManager(file!!) + targetMapping = ExperimentalOutputToTargetMapping(storageManager) mapping = ExperimentalSourceToOutputMapping.createSourceToOutputMap( storageManager = storageManager, relativizer = PathRelativizerService(), targetId = "test-module", - targetTypeId = "java" + targetTypeId = "java", + outputToTargetMapping = targetMapping, ) } @@ -118,6 +121,9 @@ class SourceToOutputMappingFuzzTest { } assertThat(actualMap).isEqualTo(expectedMap) + for (outputPaths in actualMap.values) { + assertThat(targetMapping.removeTargetAndGetSafeToDeleteOutputs(outputPaths, -1, mapping)).isEqualTo(outputPaths) + } } private fun checkCursorAndSourceIterator(source: String, outputs: List) { diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt index 4d4053b50d6c..9aa4209c4e3c 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt @@ -33,19 +33,20 @@ class StorageManagerTest { @Test fun staleMap(@TempDir tempDir: Path) { val file = tempDir.resolve("jps-cache.db") - val storageManager = StorageManager(file, 0) + val storageManager = StorageManager(file) try { val mapping = ExperimentalSourceToOutputMapping.createSourceToOutputMap( storageManager = storageManager, relativizer = PathRelativizerService(), targetId = "test-module", - targetTypeId = "java" + targetTypeId = "java", + outputToTargetMapping = null, ) mapping.appendOutput("foo/bar/Baz.java", "out/bar/Baz.class") assertThat(mapping.getOutputs("foo/bar/Baz.java")).containsExactly("out/bar/Baz.class") - storageManager.removeMaps(targetId = "test-module", typeId = "java") + storageManager.removeMaps(targetId = "test-module", targetTypeId = "java") assertThat(mapping.getOutputs("foo/bar/Baz.java")).isNull() } finally { diff --git a/platform/util/base/api-dump.txt b/platform/util/base/api-dump.txt index c61ac850ff04..684741d7ee3a 100644 --- a/platform/util/base/api-dump.txt +++ b/platform/util/base/api-dump.txt @@ -420,6 +420,7 @@ f:com.intellij.util.containers.CollectionFactory - s:createCustomHashingStrategySet(com.intellij.util.containers.HashingStrategy):java.util.Set - s:createFilePathLinkedMap():java.util.Map - s:createFilePathLinkedSet():java.util.Set +- s:createFilePathLinkedSet(java.util.Set):java.util.Set - s:createFilePathMap():java.util.Map - s:createFilePathMap(I):java.util.Map - s:createFilePathMap(I,Z):java.util.Map diff --git a/platform/util/base/src/com/intellij/util/containers/CollectionFactory.java b/platform/util/base/src/com/intellij/util/containers/CollectionFactory.java index 50cc7156d8f5..49d2809a61b0 100644 --- a/platform/util/base/src/com/intellij/util/containers/CollectionFactory.java +++ b/platform/util/base/src/com/intellij/util/containers/CollectionFactory.java @@ -279,17 +279,22 @@ public final class CollectionFactory { public static @NotNull Set createFilePathLinkedSet() { return SystemInfoRt.isFileSystemCaseSensitive - ? new ObjectLinkedOpenHashSet<>() + ? new LinkedHashSet<>() : new ObjectLinkedOpenCustomHashSet<>(FastUtilHashingStrategies.getCaseInsensitiveStringStrategy()); } + public static @NotNull Set createFilePathLinkedSet(@NotNull Set source) { + return SystemInfoRt.isFileSystemCaseSensitive + ? new LinkedHashSet<>(source) + : new ObjectLinkedOpenCustomHashSet<>(source, FastUtilHashingStrategies.getCaseInsensitiveStringStrategy()); + } + /** * Create a linked map with key hash strategy according to file system path case sensitivity. */ public static @NotNull Map createFilePathLinkedMap() { - //noinspection SSBasedInspection return SystemInfoRt.isFileSystemCaseSensitive - ? new Object2ObjectLinkedOpenHashMap<>() + ? new LinkedHashMap<>() : new Object2ObjectLinkedOpenCustomHashMap<>(FastUtilHashingStrategies.getCaseInsensitiveStringStrategy()); }