MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

experimental compact storage for JPS Cache (part 5 - store OutputToTargetMapping)

Browse Source 
GitOrigin-RevId: 36feec030cee2cbd5554a4fc0a3b80dd74ea764c
This commit is contained in:
Vladimir Krivosheev
2024-09-19 05:28:30 +00:00
committed by intellij-monorepo-bot
parent 3b63b96418
commit db3b4f7162
26 changed files with 778 additions and 427 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps/jps-builders/api-dump.txt
+4 -10
View File
@@ -1883,7 +1883,7 @@ f:org.jetbrains.jps.cmdline.PreloadedData
f:org.jetbrains.jps.cmdline.ProjectDescriptor
- f:dataManager:org.jetbrains.jps.incremental.storage.BuildDataManager
- f:fsState:org.jetbrains.jps.incremental.fs.BuildFSState
- <init>(org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V
- <init>(org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.ProjectStamps,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V
- getBuildRootIndex():org.jetbrains.jps.builders.BuildRootIndex
- getBuildTargetIndex():org.jetbrains.jps.builders.BuildTargetIndex
- getEncodingConfiguration():org.jetbrains.jps.incremental.CompilerEncodingConfiguration
@@ -2608,7 +2608,7 @@ a:org.jetbrains.jps.incremental.storage.AbstractStateStorage
- appendData(java.lang.Object,java.lang.Object):V
- f:clean():V
- f:close():V
- flush(Z):V
- f:flush(Z):V
- f:force():V
- pf:getKeyIterator(java.util.function.Function):java.util.Iterator
- getKeysIterator():java.util.Iterator
@@ -2623,14 +2623,13 @@ f:org.jetbrains.jps.incremental.storage.BuildDataManager
- cleanTargetStorages(org.jetbrains.jps.builders.BuildTarget):V
- close():V
- closeSourceToOutputStorages(org.jetbrains.jps.builders.impl.BuildTargetChunk):V
- createDependencyGraph(java.io.File,Z):V
- createDependencyGraph(java.nio.file.Path,Z):V
- flush(Z):V
- getDataPaths():org.jetbrains.jps.builders.storage.BuildDataPaths
- getDependencyGraph():org.jetbrains.jps.dependency.GraphConfiguration
- getFileStampService():org.jetbrains.jps.incremental.storage.ProjectStamps
- getFileStampStorage(org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.incremental.storage.StampsStorage
- s:getMappingsRoot(java.io.File):java.io.File
- getOutputToTargetRegistry():org.jetbrains.jps.incremental.storage.OutputToTargetRegistry
- s:getMappingsRoot(java.nio.file.Path):java.nio.file.Path
- getRelativizer():org.jetbrains.jps.incremental.relativizer.PathRelativizerService
- getSourceToOutputMap(org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.builders.storage.SourceToOutputMapping
- getStorage(org.jetbrains.jps.builders.BuildTarget,org.jetbrains.jps.builders.storage.StorageProvider):org.jetbrains.jps.incremental.storage.StorageOwner
@@ -2695,11 +2694,6 @@ f:org.jetbrains.jps.incremental.storage.OneToManyPathsMapping
- remove(java.lang.String):V
- removeData(java.lang.String,java.lang.String):V
- setOutputs(java.lang.String,java.util.List):V
f:org.jetbrains.jps.incremental.storage.OutputToTargetRegistry
- org.jetbrains.jps.incremental.storage.AbstractStateStorage
- getSafeToDeleteOutputs(java.util.Collection,I):java.util.Collection
- removeMapping(java.lang.String,I):V
- removeMapping(java.util.Collection,I):V
f:org.jetbrains.jps.incremental.storage.ProjectStamps
- sf:PORTABLE_CACHES:Z
- sf:PORTABLE_CACHES_PROPERTY:java.lang.String
jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java
+5 -4
View File
@@ -101,7 +101,8 @@ public final class BuildRunner {
storageManager = createStorageManager(dataStorageRoot);
fileStampService = initProjectStampStorage(dataStorageRoot, targetsState);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager);
dataManager.fileStampService = fileStampService;
if (dataManager.versionDiffers()) {
myForceCleanCaches = true;
msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO,
@@ -129,20 +130,20 @@ public final class BuildRunner {
storageManager = createStorageManager(dataStorageRoot);
targetsState = new BuildTargetsState(dataPaths, jpsModel, buildRootIndex);
fileStampService = initProjectStampStorage(dataStorageRoot, targetsState);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager);
// the second attempt succeeded
msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO,
JpsBuildBundle.message("build.message.project.rebuild.forced.0", e.getMessage())));
}
return new ProjectDescriptor(
jpsModel, fsState, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex
jpsModel, fsState, fileStampService, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex
);
}
private static @Nullable StorageManager createStorageManager(@NotNull Path dataStorageRoot) {
if (USE_EXPERIMENTAL_STORAGE || ProjectStamps.PORTABLE_CACHES) {
StorageManager manager = new StorageManager(dataStorageRoot.resolve("jps-portable-cache.db"), 10_000);
StorageManager manager = new StorageManager(dataStorageRoot.resolve("jps-portable-cache.db"));
manager.open();
return manager;
}
jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java
+4 -25
View File
@@ -1,8 +1,6 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.cmdline;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.TestOnly;
import org.jetbrains.jps.builders.BuildRootIndex;
import org.jetbrains.jps.builders.BuildTarget;
import org.jetbrains.jps.builders.BuildTargetIndex;
@@ -30,10 +28,10 @@ import java.util.Set;
public final class ProjectDescriptor {
private final JpsProject myProject;
private final JpsModel myModel;
@TestOnly
private ProjectStamps deprecatedStamps;
public final BuildFSState fsState;
public final BuildDataManager dataManager;
private final ProjectStamps myProjectStamps;
private final BuildLoggingManager myLoggingManager;
private final ModuleExcludeIndex myModuleExcludeIndex;
private int myUseCounter = 1;
@@ -43,11 +41,6 @@ public final class ProjectDescriptor {
private final BuildTargetIndex myBuildTargetIndex;
private final IgnoredFileIndex myIgnoredFileIndex;
/**
* @deprecated Use {@link ProjectDescriptor#ProjectDescriptor(JpsModel, BuildFSState, BuildDataManager, BuildLoggingManager, ModuleExcludeIndex, BuildTargetIndex, BuildRootIndex, IgnoredFileIndex)}
*/
@Deprecated(forRemoval = true)
@ApiStatus.Internal
public ProjectDescriptor(JpsModel model,
BuildFSState fsState,
ProjectStamps projectStamps,
@@ -57,22 +50,12 @@ public final class ProjectDescriptor {
BuildTargetIndex buildTargetIndex,
BuildRootIndex buildRootIndex,
IgnoredFileIndex ignoredFileIndex) {
this(model, fsState, dataManager, loggingManager, moduleExcludeIndex, buildTargetIndex, buildRootIndex, ignoredFileIndex);
deprecatedStamps = projectStamps;
}
public ProjectDescriptor(JpsModel model,
BuildFSState fsState,
BuildDataManager dataManager,
BuildLoggingManager loggingManager,
ModuleExcludeIndex moduleExcludeIndex,
BuildTargetIndex buildTargetIndex,
BuildRootIndex buildRootIndex,
IgnoredFileIndex ignoredFileIndex) {
myModel = model;
myIgnoredFileIndex = ignoredFileIndex;
myProject = model.getProject();
this.fsState = fsState;
myProjectStamps = projectStamps;
dataManager.fileStampService = projectStamps;
this.dataManager = dataManager;
myBuildTargetIndex = buildTargetIndex;
myBuildRootIndex = buildRootIndex;
@@ -154,10 +137,6 @@ public final class ProjectDescriptor {
*/
@Deprecated(forRemoval = true)
public ProjectStamps getProjectStamps() {
//noinspection TestOnlyProblems
if (deprecatedStamps != null) {
return deprecatedStamps;
}
//noinspection removal
return dataManager.getFileStampService();
}
jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java
+23 -21
View File
@@ -116,17 +116,17 @@ public final class BuildOperations {
}
private static boolean dropRemovedPaths(CompileContext context, BuildTargetChunk chunk) throws IOException {
final Map<BuildTarget<?>, Collection<String>> map = Utils.REMOVED_SOURCES_KEY.get(context);
Map<BuildTarget<?>, Collection<String>> map = Utils.REMOVED_SOURCES_KEY.get(context);
boolean dropped = false;
if (map != null) {
for (BuildTarget<?> target : chunk.getTargets()) {
final Collection<String> paths = map.remove(target);
Collection<String> paths = map.remove(target);
if (paths != null) {
final SourceToOutputMapping storage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
SourceToOutputMapping storage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
for (String path : paths) {
storage.remove(path);
dropped = true;
}
dropped = true;
}
}
}
@@ -161,26 +161,28 @@ public final class BuildOperations {
else {
targetId = idsCache.getInt(target);
}
final String srcPath = file.getPath();
final Collection<String> outputs = srcToOut.getOutputs(srcPath);
if (outputs != null) {
final boolean shouldPruneOutputDirs = target instanceof ModuleBasedTarget;
final List<String> deletedForThisSource = new ArrayList<>(outputs.size());
for (String output : outputs) {
deleteRecursively(output, deletedForThisSource, shouldPruneOutputDirs ? dirsToDelete : null);
}
deletedPaths.addAll(deletedForThisSource);
dataManager.getOutputToTargetRegistry().removeMapping(deletedForThisSource, targetId);
Set<File> cleaned = cleanedSources.get(target);
if (cleaned == null) {
cleaned = FileCollectionFactory.createCanonicalFileSet();
cleanedSources.put(target, cleaned);
}
cleaned.add(file);
Collection<String> outputs = srcToOut.getOutputs(file.getPath());
if (outputs == null) {
return true;
}
boolean shouldPruneOutputDirs = target instanceof ModuleBasedTarget;
List<String> deletedForThisSource = new ArrayList<>(outputs.size());
for (String output : outputs) {
deleteRecursively(output, deletedForThisSource, shouldPruneOutputDirs ? dirsToDelete : null);
}
deletedPaths.addAll(deletedForThisSource);
dataManager.getOutputToTargetMapping().removeMappings(deletedForThisSource, targetId, srcToOut);
Set<File> cleaned = cleanedSources.get(target);
if (cleaned == null) {
cleaned = FileCollectionFactory.createCanonicalFileSet();
cleanedSources.put(target, cleaned);
}
cleaned.add(file);
return true;
}
});
if (!deletedPaths.isEmpty()) {
jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java
+103 -105
View File
@@ -712,29 +712,96 @@ public final class IncProjectBuilder {
registerTargetsWithClearedOutput(context, Collections.singletonList(target));
}
private static void clearOutputFiles(CompileContext context,
SourceToOutputMapping mapping,
BuildTargetType<?> targetType,
int targetId) throws IOException {
Set<File> dirsToDelete = targetType instanceof ModuleBasedBuildTargetType<?> ? FileCollectionFactory.createCanonicalFileSet() : null;
OutputToTargetRegistry outputToTargetRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetRegistry();
for (SourceToOutputMappingCursor cursor = mapping.cursor(); cursor.hasNext(); ) {
cursor.next();
String [] outs = cursor.getOutputPaths();
if (outs.length > 0) {
List<String> deletedPaths = new ArrayList<>();
for (String out : outs) {
BuildOperations.deleteRecursively(out, deletedPaths, dirsToDelete);
private boolean processDeletedPaths(CompileContext context, final Set<? extends BuildTarget<?>> targets) throws ProjectBuildException {
boolean doneSomething = false;
try {
// cleanup outputs
final Map<BuildTarget<?>, Collection<String>> targetToRemovedSources = new HashMap<>();
Set<File> dirsToDelete = FileCollectionFactory.createCanonicalFileSet();
for (BuildTarget<?> target : targets) {
Collection<String> deletedPaths = myProjectDescriptor.fsState.getAndClearDeletedPaths(target);
if (deletedPaths.isEmpty()) {
continue;
}
outputToTargetRegistry.removeMapping(Arrays.asList(outs), targetId);
if (!deletedPaths.isEmpty()) {
context.processMessage(new FileDeletedEvent(deletedPaths));
targetToRemovedSources.put(target, deletedPaths);
if (isTargetOutputCleared(context, target)) {
continue;
}
final int buildTargetId = context.getProjectDescriptor().getTargetsState().getBuildTargetId(target);
final boolean shouldPruneEmptyDirs = target instanceof ModuleBasedTarget;
BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
final SourceToOutputMapping sourceToOutputStorage = dataManager.getSourceToOutputMap(target);
final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger();
// actually delete outputs associated with removed paths
final Collection<String> pathsForIteration;
if (myIsTestMode) {
// ensure predictable order in test logs
pathsForIteration = new ArrayList<>(deletedPaths);
Collections.sort((List<String>)pathsForIteration);
}
else {
pathsForIteration = deletedPaths;
}
for (String deletedSource : pathsForIteration) {
// deleting outputs corresponding to a non-existing source
Collection<String> outputs = sourceToOutputStorage.getOutputs(deletedSource);
if (outputs != null && !outputs.isEmpty()) {
List<String> deletedOutputPaths = new ArrayList<>();
OutputToTargetMapping outputToSourceRegistry = dataManager.getOutputToTargetMapping();
for (String output : outputToSourceRegistry.removeTargetAndGetSafeToDeleteOutputs(outputs, buildTargetId, sourceToOutputStorage)) {
final boolean deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, shouldPruneEmptyDirs ? dirsToDelete : null);
if (deleted) {
doneSomething = true;
}
}
if (!deletedOutputPaths.isEmpty()) {
if (logger.isEnabled()) {
logger.logDeletedFiles(deletedOutputPaths);
}
context.processMessage(new FileDeletedEvent(deletedOutputPaths));
}
}
if (target instanceof ModuleBuildTarget) {
// check if the deleted source was associated with a form
OneToManyPathMapping sourceToFormMap = dataManager.getSourceToFormMap(target);
Collection<String> boundForms = sourceToFormMap.getOutputs(deletedSource);
if (boundForms != null) {
for (String formPath : boundForms) {
final File formFile = new File(formPath);
if (formFile.exists()) {
FSOperations.markDirty(context, CompilationRound.CURRENT, formFile);
}
}
sourceToFormMap.remove(deletedSource);
}
}
}
}
}
if (dirsToDelete != null) {
if (!targetToRemovedSources.isEmpty()) {
final Map<BuildTarget<?>, Collection<String>> existing = Utils.REMOVED_SOURCES_KEY.get(context);
if (existing != null) {
for (Map.Entry<BuildTarget<?>, Collection<String>> entry : existing.entrySet()) {
final Collection<String> paths = targetToRemovedSources.get(entry.getKey());
if (paths != null) {
paths.addAll(entry.getValue());
}
else {
targetToRemovedSources.put(entry.getKey(), entry.getValue());
}
}
}
Utils.REMOVED_SOURCES_KEY.set(context, targetToRemovedSources);
}
FSOperations.pruneEmptyDirs(context, dirsToDelete);
}
catch (IOException e) {
throw new ProjectBuildException(e);
}
return doneSomething;
}
private static void registerTargetsWithClearedOutput(CompileContext context, Collection<? extends BuildTarget<?>> targets) {
@@ -1458,98 +1525,29 @@ public final class IncProjectBuilder {
myMessageDispatcher.processMessage(new BuildingTargetProgressMessage(targets, event));
}
private boolean processDeletedPaths(CompileContext context, final Set<? extends BuildTarget<?>> targets) throws ProjectBuildException {
boolean doneSomething = false;
try {
// cleanup outputs
final Map<BuildTarget<?>, Collection<String>> targetToRemovedSources = new HashMap<>();
Set<File> dirsToDelete = FileCollectionFactory.createCanonicalFileSet();
for (BuildTarget<?> target : targets) {
final Collection<String> deletedPaths = myProjectDescriptor.fsState.getAndClearDeletedPaths(target);
if (deletedPaths.isEmpty()) {
continue;
private static void clearOutputFiles(CompileContext context,
SourceToOutputMapping mapping,
BuildTargetType<?> targetType,
int targetId) throws IOException {
Set<File> dirsToDelete = targetType instanceof ModuleBasedBuildTargetType<?> ? FileCollectionFactory.createCanonicalFileSet() : null;
OutputToTargetMapping outputToTargetRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetMapping();
for (SourceToOutputMappingCursor cursor = mapping.cursor(); cursor.hasNext(); ) {
cursor.next();
String [] outs = cursor.getOutputPaths();
if (outs.length > 0) {
List<String> deletedPaths = new ArrayList<>();
for (String out : outs) {
BuildOperations.deleteRecursively(out, deletedPaths, dirsToDelete);
}
targetToRemovedSources.put(target, deletedPaths);
if (isTargetOutputCleared(context, target)) {
continue;
}
final int buildTargetId = context.getProjectDescriptor().getTargetsState().getBuildTargetId(target);
final boolean shouldPruneEmptyDirs = target instanceof ModuleBasedTarget;
BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
final SourceToOutputMapping sourceToOutputStorage = dataManager.getSourceToOutputMap(target);
final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger();
// actually delete outputs associated with removed paths
final Collection<String> pathsForIteration;
if (myIsTestMode) {
// ensure predictable order in test logs
pathsForIteration = new ArrayList<>(deletedPaths);
Collections.sort((List<String>)pathsForIteration);
}
else {
pathsForIteration = deletedPaths;
}
for (String deletedSource : pathsForIteration) {
// deleting outputs corresponding to non-existing source
final Collection<String> outputs = sourceToOutputStorage.getOutputs(deletedSource);
if (outputs != null && !outputs.isEmpty()) {
List<String> deletedOutputPaths = new ArrayList<>();
final OutputToTargetRegistry outputToSourceRegistry = dataManager.getOutputToTargetRegistry();
for (String output : outputToSourceRegistry.getSafeToDeleteOutputs(outputs, buildTargetId)) {
final boolean deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, shouldPruneEmptyDirs ? dirsToDelete : null);
if (deleted) {
doneSomething = true;
}
}
for (String outputPath : outputs) {
outputToSourceRegistry.removeMapping(outputPath, buildTargetId);
}
if (!deletedOutputPaths.isEmpty()) {
if (logger.isEnabled()) {
logger.logDeletedFiles(deletedOutputPaths);
}
context.processMessage(new FileDeletedEvent(deletedOutputPaths));
}
}
if (target instanceof ModuleBuildTarget) {
// check if the deleted source was associated with a form
OneToManyPathMapping sourceToFormMap = dataManager.getSourceToFormMap(target);
Collection<String> boundForms = sourceToFormMap.getOutputs(deletedSource);
if (boundForms != null) {
for (String formPath : boundForms) {
final File formFile = new File(formPath);
if (formFile.exists()) {
FSOperations.markDirty(context, CompilationRound.CURRENT, formFile);
}
}
sourceToFormMap.remove(deletedSource);
}
}
outputToTargetRegistry.removeMappings(Arrays.asList(outs), targetId, mapping);
if (!deletedPaths.isEmpty()) {
context.processMessage(new FileDeletedEvent(deletedPaths));
}
}
if (!targetToRemovedSources.isEmpty()) {
final Map<BuildTarget<?>, Collection<String>> existing = Utils.REMOVED_SOURCES_KEY.get(context);
if (existing != null) {
for (Map.Entry<BuildTarget<?>, Collection<String>> entry : existing.entrySet()) {
final Collection<String> paths = targetToRemovedSources.get(entry.getKey());
if (paths != null) {
paths.addAll(entry.getValue());
}
else {
targetToRemovedSources.put(entry.getKey(), entry.getValue());
}
}
}
Utils.REMOVED_SOURCES_KEY.set(context, targetToRemovedSources);
}
}
if (dirsToDelete != null) {
FSOperations.pruneEmptyDirs(context, dirsToDelete);
}
catch (IOException e) {
throw new ProjectBuildException(e);
}
return doneSomething;
}
// return true if changed something, false otherwise
jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java
+8 -9
View File
@@ -194,11 +194,8 @@ public final class BuildFSState {
}
public Collection<String> getAndClearDeletedPaths(BuildTarget<?> target) {
final FilesDelta delta = myDeltas.get(target);
if (delta != null) {
return delta.getAndClearDeletedPaths();
}
return Collections.emptyList();
FilesDelta delta = myDeltas.get(target);
return delta == null ? List.of() : delta.getAndClearDeletedPaths();
}
private @NotNull FilesDelta getDelta(BuildTarget<?> buildTarget) {
@@ -238,10 +235,10 @@ public final class BuildFSState {
*/
public boolean markDirty(@Nullable CompileContext context,
File file,
final BuildRootDescriptor rd,
BuildRootDescriptor buildRootDescriptor,
@Nullable StampsStorage<?> stampStorage,
boolean saveEventStamp) throws IOException {
return markDirty(context, CompilationRound.NEXT, file, rd, stampStorage, saveEventStamp);
return markDirty(context, CompilationRound.NEXT, file, buildRootDescriptor, stampStorage, saveEventStamp);
}
public boolean markDirty(@Nullable CompileContext context,
@@ -384,7 +381,7 @@ public final class BuildFSState {
*/
public boolean markAllUpToDate(@NotNull CompileContext context,
@NotNull BuildRootDescriptor buildRootDescriptor,
@NotNull StampsStorage<?> stampStorage,
@Nullable StampsStorage<?> stampStorage,
long targetBuildStartStamp) throws IOException {
boolean marked = false;
final BuildTarget<?> target = buildRootDescriptor.getTarget();
@@ -415,7 +412,9 @@ public final class BuildFSState {
}
else {
marked = true;
stampStorage.updateStamp(nioFile, target, currentFileTimestamp);
if (stampStorage != null) {
stampStorage.updateStamp(nioFile, target, currentFileTimestamp);
}
}
}
else {
jps/jps-builders/src/org/jetbrains/jps/incremental/fs/FilesDelta.java
+6 -4
View File
@@ -244,13 +244,15 @@ public final class FilesDelta {
}
}
public Set<String> getAndClearDeletedPaths() {
public @NotNull Set<String> getAndClearDeletedPaths() {
lockData();
try {
if (myDeletedPaths.isEmpty()) {
return Set.of();
}
try {
Set<String> result = CollectionFactory.createFilePathLinkedSet();
result.addAll(myDeletedPaths);
return result;
return CollectionFactory.createFilePathLinkedSet(myDeletedPaths);
}
finally {
myDeletedPaths.clear();
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java
+21 -6
View File
@@ -6,15 +6,13 @@ import com.intellij.util.io.*;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.*;
import java.util.function.Function;
public abstract class AbstractStateStorage<Key, T> implements StorageOwner {
@@ -107,11 +105,28 @@ public abstract class AbstractStateStorage<Key, T> implements StorageOwner {
}
}
/**
* @deprecated Use {@link #getKeysIterator()}
*/
@TestOnly
@ApiStatus.Internal
@Deprecated(forRemoval = true)
public final Collection<Key> getKeys() throws IOException {
return getAllKeys();
}
public @NotNull Iterator<Key> getKeysIterator() throws IOException {
//noinspection TestOnlyProblems
return getAllKeys().iterator();
}
@TestOnly
@ApiStatus.Internal
public final @NotNull List<Key> getAllKeys() throws IOException {
synchronized (dataLock) {
List<Key> result = new ArrayList<>();
map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result));
return result.iterator();
return result.isEmpty() ? List.of() : result;
}
}
@@ -132,7 +147,7 @@ public abstract class AbstractStateStorage<Key, T> implements StorageOwner {
}
@Override
public void flush(boolean memoryCachesOnly) {
public final void flush(boolean memoryCachesOnly) {
if (!memoryCachesOnly) {
force();
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java
+160 -145
View File
@@ -42,7 +42,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Stream;
/**
@@ -61,10 +60,11 @@ public final class BuildDataManager {
private final @NotNull ConcurrentMap<BuildTarget<?>, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>();
private final @NotNull ConcurrentMap<BuildTarget<?>, SourceToOutputMappingWrapper> buildTargetToSourceToOutputMapping = new ConcurrentHashMap<>();
// only for new experimental storage
private final @NotNull ConcurrentMap<BuildTarget<?>, PerTargetMapManager> targetToMapManager = new ConcurrentHashMap<>();
private final @Nullable ExperimentalBuildDataManager newDataManager;
private final @Nullable ProjectStamps fileStampService;
// make private after updating bootstrap for Kotlin tests
@ApiStatus.Internal
public @Nullable ProjectStamps fileStampService;
private final @Nullable OneToManyPathsMapping sourceToFormMap;
private final Mappings myMappings;
@@ -73,8 +73,7 @@ public final class BuildDataManager {
private final NodeSourcePathMapper myDepGraphPathMapper;
private final BuildDataPaths myDataPaths;
private final BuildTargetsState myTargetsState;
@Nullable private final StorageManager storageManager;
private final OutputToTargetRegistry myOutputToTargetRegistry;
private final @Nullable OutputToTargetRegistry outputToTargetMapping;
private final File myVersionFile;
private final PathRelativizerService myRelativizer;
private boolean myProcessConstantsIncrementally = !Boolean.parseBoolean(System.getProperty(PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY, "false"));
@@ -84,33 +83,33 @@ public final class BuildDataManager {
public BuildDataManager(BuildDataPaths dataPaths,
BuildTargetsState targetsState,
@NotNull PathRelativizerService relativizer) throws IOException {
this(dataPaths, targetsState, relativizer, null, null);
this(dataPaths, targetsState, relativizer, null);
}
@ApiStatus.Internal
public BuildDataManager(BuildDataPaths dataPaths,
BuildTargetsState targetsState,
@NotNull PathRelativizerService relativizer,
@Nullable ProjectStamps fileStampService,
@Nullable StorageManager storageManager) throws IOException {
this.fileStampService = fileStampService;
newDataManager = storageManager == null ? null :new ExperimentalBuildDataManager(storageManager, relativizer);
myDataPaths = dataPaths;
myTargetsState = targetsState;
this.storageManager = storageManager;
Path dataStorageRoot = myDataPaths.getDataStorageRoot().toPath();
try {
sourceToFormMap = storageManager == null ? new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer) : null;
myOutputToTargetRegistry = new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer);
File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot());
outputToTargetMapping = storageManager == null ? new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer) : null;
Path mappingsRoot = getMappingsRoot(dataStorageRoot);
if (JavaBuilderUtil.isDepGraphEnabled()) {
myMappings = null;
createDependencyGraph(mappingsRoot, false);
FileUtilRt.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), false)); // delete older mappings data if available
// delete older mappings data if available
FileUtilRt.deleteRecursively(getMappingsRoot(dataStorageRoot, false));
LOG.info("Using DependencyGraph-based build incremental analysis");
}
else {
myMappings = new Mappings(mappingsRoot, relativizer);
FileUtil.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), true)); // delete dep-graph data if available
myMappings = new Mappings(mappingsRoot.toFile(), relativizer);
FileUtilRt.deleteRecursively(getMappingsRoot(dataStorageRoot, true)); // delete dep-graph data if available
myMappings.setProcessConstantsIncrementally(isProcessConstantsIncrementally());
}
}
@@ -122,15 +121,15 @@ public final class BuildDataManager {
}
throw e;
}
myVersionFile = new File(myDataPaths.getDataStorageRoot(), "version.dat");
myVersionFile = dataStorageRoot.resolve("version.dat").toFile();
myDepGraphPathMapper = new PathSourceMapper(relativizer::toFull, relativizer::toRelative);
myRelativizer = relativizer;
}
@ApiStatus.Internal
public void clearCache() {
if (storageManager != null) {
storageManager.clearCache();
if (newDataManager != null) {
newDataManager.clearCache();
}
}
@@ -153,8 +152,8 @@ public final class BuildDataManager {
public void cleanStaleTarget(@NotNull BuildTargetType<?> targetType, @NotNull String targetId) throws IOException {
try {
FileUtilRt.deleteRecursively(getDataPaths().getTargetDataRoot(targetType, targetId));
if (storageManager != null) {
storageManager.removeMaps(targetId, targetType.getTypeId());
if (newDataManager != null) {
newDataManager.removeStaleTarget(targetId, targetType.getTypeId());
}
}
finally {
@@ -162,12 +161,23 @@ public final class BuildDataManager {
}
}
public OutputToTargetRegistry getOutputToTargetRegistry() {
return myOutputToTargetRegistry;
@ApiStatus.Internal
public @NotNull OutputToTargetMapping getOutputToTargetMapping() {
return newDataManager == null ? Objects.requireNonNull(outputToTargetMapping) : newDataManager.getOutputToTargetMapping();
}
/**
* @deprecated Use {@link #getOutputToTargetMapping()}
* @return
*/
@ApiStatus.Internal
@Deprecated(forRemoval = true)
public @NotNull OutputToTargetRegistry getOutputToTargetRegistry() {
return Objects.requireNonNull(outputToTargetMapping);
}
public @NotNull SourceToOutputMapping getSourceToOutputMap(@NotNull BuildTarget<?> target) throws IOException {
if (storageManager == null) {
if (newDataManager == null) {
try {
return buildTargetToSourceToOutputMapping.computeIfAbsent(target, this::createSourceToOutputMap);
}
@@ -177,12 +187,12 @@ public final class BuildDataManager {
}
}
else {
return getPerTargetMapManager(target).getSourceToOutputMapping();
return newDataManager.getSourceToOutputMapping(target);
}
}
private @NotNull SourceToOutputMappingWrapper createSourceToOutputMap(@NotNull BuildTarget<?> target) {
SourceToOutputMapping map;
SourceToOutputMappingImpl map;
try {
Path file = myDataPaths.getTargetDataRootDir(target).resolve(SRC_TO_OUTPUT_STORAGE).resolve(SRC_TO_OUTPUT_FILE_NAME);
map = new SourceToOutputMappingImpl(file, myRelativizer);
@@ -191,21 +201,16 @@ public final class BuildDataManager {
LOG.info(e);
throw new BuildDataCorruptedException(e);
}
return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target));
return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target), outputToTargetMapping);
}
public @Nullable StampsStorage<?> getFileStampStorage(@NotNull BuildTarget<?> target) {
if (storageManager == null) {
if (newDataManager == null) {
return fileStampService == null ? null : fileStampService.getStampStorage();
}
return getPerTargetMapManager(target).stamp;
}
private @NotNull PerTargetMapManager getPerTargetMapManager(@NotNull BuildTarget<?> target) {
return targetToMapManager.computeIfAbsent(target, it -> {
assert storageManager != null;
return new PerTargetMapManager(storageManager, myRelativizer, it, mapping -> new SourceToOutputMappingWrapper(mapping, myTargetsState.getBuildTargetId(it)));
});
else {
return newDataManager.getFileStampStorage(target);
}
}
/**
@@ -228,7 +233,7 @@ public final class BuildDataManager {
@ApiStatus.Internal
public @NotNull OneToManyPathMapping getSourceToFormMap(@NotNull BuildTarget<?> target) {
return sourceToFormMap == null ? getPerTargetMapManager(target).getSourceToForm() : sourceToFormMap;
return newDataManager == null ? Objects.requireNonNull(sourceToFormMap) : newDataManager.getSourceToForm(target);
}
@ApiStatus.Internal
@@ -263,11 +268,13 @@ public final class BuildDataManager {
}
finally {
SourceToOutputMappingWrapper sourceToOutput = buildTargetToSourceToOutputMapping.remove(target);
if (sourceToOutput != null && sourceToOutput.myDelegate instanceof StorageOwner) {
if (sourceToOutput != null && sourceToOutput.myDelegate != null) {
((StorageOwner)sourceToOutput.myDelegate).close();
}
targetToMapManager.remove(target);
if (newDataManager != null) {
newDataManager.closeTargetMaps(target);
}
}
}
finally {
@@ -285,7 +292,7 @@ public final class BuildDataManager {
}
public void clean(@NotNull Consumer<Future<?>> asyncTaskCollector) throws IOException {
if (fileStampService != null) {
if (newDataManager == null && fileStampService != null) {
try {
((StorageOwner)fileStampService.getStampStorage()).clean();
}
@@ -297,10 +304,9 @@ public final class BuildDataManager {
try {
allTargetStorages(asyncTaskCollector).clean();
buildTargetToSourceToOutputMapping.clear();
targetToMapManager.clear();
myTargetStorages.clear();
if (storageManager != null) {
storageManager.clean();
if (newDataManager != null) {
newDataManager.removeAllMaps();
}
}
finally {
@@ -311,18 +317,20 @@ public final class BuildDataManager {
}
finally {
try {
wipeStorage(getOutputToSourceRegistryRoot(), myOutputToTargetRegistry);
if (outputToTargetMapping != null) {
wipeStorage(getOutputToSourceRegistryRoot(), outputToTargetMapping);
}
}
finally {
File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot());
final Mappings mappings = myMappings;
Path mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot().toPath());
Mappings mappings = myMappings;
if (mappings != null) {
synchronized (mappings) {
mappings.clean();
}
}
else {
FileUtil.delete(mappingsRoot);
FileUtilRt.deleteRecursively(mappingsRoot);
}
if (JavaBuilderUtil.isDepGraphEnabled()) {
@@ -335,7 +343,7 @@ public final class BuildDataManager {
saveVersion();
}
public void createDependencyGraph(File mappingsRoot, boolean deleteExisting) throws IOException {
public void createDependencyGraph(@NotNull Path mappingsRoot, boolean deleteExisting) throws IOException {
try {
synchronized (myGraphManagementLock) {
DependencyGraph depGraph = myDepGraph;
@@ -343,7 +351,7 @@ public final class BuildDataManager {
if (deleteExisting) {
FileUtil.delete(mappingsRoot);
}
myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.getAbsolutePath())));
myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.toString())));
}
else {
try {
@@ -353,7 +361,7 @@ public final class BuildDataManager {
if (deleteExisting) {
FileUtil.delete(mappingsRoot);
}
myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.getAbsolutePath())));
myDepGraph = asSynchronizedGraph(new DependencyGraphImpl(Containers.createPersistentContainerFactory(mappingsRoot.toString())));
}
}
}
@@ -368,20 +376,21 @@ public final class BuildDataManager {
}
public void flush(boolean memoryCachesOnly) {
if (storageManager == null) {
if (newDataManager == null) {
if (fileStampService != null) {
((StorageOwner)fileStampService.getStampStorage()).flush(false);
}
assert outputToTargetMapping != null;
outputToTargetMapping.flush(memoryCachesOnly);
assert sourceToFormMap != null;
sourceToFormMap.flush(memoryCachesOnly);
}
else if (!memoryCachesOnly) {
storageManager.commit();
newDataManager.commit();
}
allTargetStorages().flush(memoryCachesOnly);
myOutputToTargetRegistry.flush(memoryCachesOnly);
if (sourceToFormMap != null) {
sourceToFormMap.flush(memoryCachesOnly);
}
Mappings mappings = myMappings;
if (mappings != null) {
@@ -402,18 +411,12 @@ public final class BuildDataManager {
buildTargetToSourceToOutputMapping.clear();
}
if (storageManager != null) {
targetToMapManager.clear();
try {
storageManager.close();
}
catch (Throwable e) {
LOG.error(e);
}
if (newDataManager == null) {
flushOldStorages();
}
else if (fileStampService != null) {
else {
try {
fileStampService.close();
newDataManager.close();
}
catch (Throwable e) {
LOG.error(e);
@@ -421,47 +424,62 @@ public final class BuildDataManager {
}
}
finally {
try {
myOutputToTargetRegistry.close();
}
finally {
Mappings mappings = myMappings;
if (mappings != null) {
try {
if (sourceToFormMap != null) {
synchronized (sourceToFormMap) {
sourceToFormMap.close();
}
}
mappings.close();
}
finally {
final Mappings mappings = myMappings;
if (mappings != null) {
try {
mappings.close();
}
catch (BuildDataCorruptedException e) {
throw e.getCause();
}
}
catch (BuildDataCorruptedException e) {
throw e.getCause();
}
}
synchronized (myGraphManagementLock) {
DependencyGraph depGraph = myDepGraph;
if (depGraph != null) {
myDepGraph = null;
try {
depGraph.close();
}
catch (BuildDataCorruptedException e) {
throw e.getCause();
}
}
synchronized (myGraphManagementLock) {
DependencyGraph depGraph = myDepGraph;
if (depGraph != null) {
myDepGraph = null;
try {
depGraph.close();
}
catch (BuildDataCorruptedException e) {
throw e.getCause();
}
}
}
}
}
private void flushOldStorages() {
if (fileStampService != null) {
try {
fileStampService.close();
}
catch (Throwable e) {
LOG.error(e);
}
}
try {
assert outputToTargetMapping != null;
outputToTargetMapping.close();
}
catch (Throwable e) {
LOG.error(e);
}
try {
assert sourceToFormMap != null;
synchronized (sourceToFormMap) {
sourceToFormMap.close();
}
}
catch (Throwable e) {
LOG.error(e);
}
}
public void closeSourceToOutputStorages(@NotNull BuildTargetChunk chunk) throws IOException {
if (storageManager != null) {
if (newDataManager != null) {
for (BuildTarget<?> target : chunk.getTargets()) {
buildTargetToSourceToOutputMapping.remove(target);
}
@@ -476,7 +494,7 @@ public final class BuildDataManager {
continue;
}
StorageOwner delegate = (StorageOwner)sourceToOutputMapping.myDelegate;
StorageOwner delegate = sourceToOutputMapping.myDelegate;
try {
delegate.close();
}
@@ -518,12 +536,12 @@ public final class BuildDataManager {
return myRelativizer;
}
public static File getMappingsRoot(final File dataStorageRoot) {
public static @NotNull Path getMappingsRoot(@NotNull Path dataStorageRoot) {
return getMappingsRoot(dataStorageRoot, JavaBuilderUtil.isDepGraphEnabled());
}
private static File getMappingsRoot(final File dataStorageRoot, boolean forDepGraph) {
return new File(dataStorageRoot, forDepGraph? MAPPINGS_STORAGE + "-graph" : MAPPINGS_STORAGE);
private static Path getMappingsRoot(@NotNull Path dataStorageRoot, boolean forDepGraph) {
return dataStorageRoot.resolve(forDepGraph? MAPPINGS_STORAGE + "-graph" : MAPPINGS_STORAGE);
}
private static void wipeStorage(@NotNull Path root, @Nullable StorageOwner storage) {
@@ -587,18 +605,48 @@ public final class BuildDataManager {
myRelativizer.reportUnhandledPaths();
}
final class SourceToOutputMappingWrapper implements SourceToOutputMapping, Supplier<@Nullable StorageOwner> {
private final SourceToOutputMapping myDelegate;
private final int myBuildTargetId;
private @NotNull StorageOwner allTargetStorages(@NotNull Consumer<Future<?>> asyncTaskCollector) {
return new CompositeStorageOwner() {
@Override
public void clean() throws IOException {
try {
close();
}
finally {
asyncTaskCollector.accept(FileUtil.asyncDelete(myDataPaths.getTargetsDataRoot()));
}
}
SourceToOutputMappingWrapper(SourceToOutputMapping delegate, int buildTargetId) {
@Override
protected Iterable<? extends StorageOwner> getChildStorages() {
return () -> {
return Stream.concat(
myTargetStorages.values().stream(),
buildTargetToSourceToOutputMapping.values().stream()
.map(wrapper -> {
SourceToOutputMapping delegate = wrapper.myDelegate;
return delegate == null ? null : (StorageOwner)delegate;
})
.filter(o -> o != null)
).iterator();
};
}
};
}
private @NotNull StorageOwner allTargetStorages() {
return allTargetStorages(f -> {});
}
private static final class SourceToOutputMappingWrapper implements SourceToOutputMapping {
private final SourceToOutputMappingImpl myDelegate;
private final int myBuildTargetId;
private final OutputToTargetRegistry outputToTargetMapping;
SourceToOutputMappingWrapper(SourceToOutputMappingImpl delegate, int buildTargetId, OutputToTargetRegistry outputToTargetMapping) {
myDelegate = delegate;
myBuildTargetId = buildTargetId;
}
@Override
public @Nullable StorageOwner get() {
return myDelegate instanceof StorageOwner? (StorageOwner)myDelegate : null;
this.outputToTargetMapping = outputToTargetMapping;
}
@Override
@@ -607,7 +655,7 @@ public final class BuildDataManager {
myDelegate.setOutputs(srcPath, outputs);
}
finally {
myOutputToTargetRegistry.addMapping(outputs, myBuildTargetId);
outputToTargetMapping.addMappings(outputs, myBuildTargetId);
}
}
@@ -617,7 +665,7 @@ public final class BuildDataManager {
myDelegate.setOutput(srcPath, outputPath);
}
finally {
myOutputToTargetRegistry.addMapping(outputPath, myBuildTargetId);
outputToTargetMapping.addMapping(outputPath, myBuildTargetId);
}
}
@@ -627,7 +675,7 @@ public final class BuildDataManager {
myDelegate.appendOutput(srcPath, outputPath);
}
finally {
myOutputToTargetRegistry.addMapping(outputPath, myBuildTargetId);
outputToTargetMapping.addMapping(outputPath, myBuildTargetId);
}
}
@@ -657,39 +705,6 @@ public final class BuildDataManager {
}
}
private @NotNull StorageOwner allTargetStorages() {
return allTargetStorages(f -> {});
}
private @NotNull StorageOwner allTargetStorages(@NotNull Consumer<Future<?>> asyncTaskCollector) {
return new CompositeStorageOwner() {
@Override
public void clean() throws IOException {
try {
close();
}
finally {
asyncTaskCollector.accept(FileUtil.asyncDelete(myDataPaths.getTargetsDataRoot()));
}
}
@Override
protected Iterable<? extends StorageOwner> getChildStorages() {
return () -> {
return Stream.concat(
myTargetStorages.values().stream(),
buildTargetToSourceToOutputMapping.values().stream()
.map(wrapper -> {
SourceToOutputMapping delegate = wrapper.myDelegate;
return delegate instanceof StorageOwner ? (StorageOwner)delegate : null;
})
.filter(o -> o != null)
).iterator();
};
}
};
}
private static DependencyGraph asSynchronizedGraph(DependencyGraph graph) {
//noinspection IOResourceOpenedButNotSafelyClosed
DependencyGraph delegate = new LoggingDependencyGraph(graph, msg -> LOG.info(msg));
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalBuildDataManager.kt
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage
import com.intellij.util.concurrency.SynchronizedClearableLazy
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.storage.SourceToOutputMapping
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import java.util.concurrent.ConcurrentHashMap
internal class ExperimentalBuildDataManager(
private val storageManager: StorageManager,
private val relativizer: PathRelativizerService,
) {
// only for new experimental storage
private val targetToMapManager = ConcurrentHashMap<BuildTarget<*>, PerTargetMapManager>()
/**
* A map not scoped to a target is problematic because we cannot transfer built target bytecode with JPS build data.
* Normally, a source file in a module isn't expected to compile into multiple output directories.
* We can address this later, but for now, let's support it as the old storage did.
*/
private val outputToTargetMapping = SynchronizedClearableLazy {
ExperimentalOutputToTargetMapping(storageManager)
}
fun clearCache() {
storageManager.clearCache()
}
fun removeStaleTarget(targetId: String, targetTypeId: String) {
storageManager.removeMaps(targetId, targetTypeId)
outputToTargetMapping.value.removeTarget(targetId, targetTypeId)
}
private fun getPerTargetMapManager(target: BuildTarget<*>): PerTargetMapManager {
return targetToMapManager.computeIfAbsent(target) {
PerTargetMapManager(storageManager, relativizer, it, outputToTargetMapping)
}
}
fun getFileStampStorage(target: BuildTarget<*>): StampsStorage<*> {
return getPerTargetMapManager(target).stamp
}
fun getSourceToOutputMapping(target: BuildTarget<*>): SourceToOutputMapping {
return getPerTargetMapManager(target).sourceToOutputMapping
}
fun getOutputToTargetMapping(): OutputToTargetMapping = outputToTargetMapping.value
fun getSourceToForm(target: BuildTarget<*>): ExperimentalOneToManyPathMapping {
return getPerTargetMapManager(target).sourceToForm
}
fun closeTargetMaps(target: BuildTarget<*>) {
targetToMapManager.remove(target)
}
fun removeAllMaps() {
outputToTargetMapping.drop()
targetToMapManager.clear()
storageManager.clean()
}
fun commit() {
storageManager.commit()
}
fun close() {
outputToTargetMapping.drop()
targetToMapManager.clear()
storageManager.close()
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt
+21 -13
View File
@@ -10,7 +10,7 @@ import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash
@ApiStatus.Internal
open class ExperimentalOneToManyPathMapping(
@JvmField protected val mapHandle: MapHandle<LongArray, Array<String>>,
@JvmField protected val relativizer: PathRelativizerService,
@JvmField internal val relativizer: PathRelativizerService,
private val valueOffset: Int = 0,
) : OneToManyPathMapping {
protected fun getKey(path: String): LongArray = stringTo128BitHash(relativizer.toRelative(path))
@@ -22,23 +22,31 @@ open class ExperimentalOneToManyPathMapping(
return Array<String>(list.size - valueOffset) { relativizer.toFull(list.get(it + valueOffset)) }.asList()
}
final override fun setOutputs(path: String, outPaths: List<String>) {
protected fun normalizeOutputPaths(outPaths: List<String>, relativeSourcePath: String?): Array<String>? {
return when {
outPaths.isEmpty() -> null
relativeSourcePath != null -> {
Array(outPaths.size + 1) {
if (it == 0) relativeSourcePath else relativizer.toRelative(outPaths.get(it - 1))
}
}
else -> {
Array(outPaths.size) {
relativizer.toRelative(outPaths.get(it))
}
}
}
}
override fun setOutputs(path: String, outPaths: List<String>) {
val relativeSourcePath = relativizer.toRelative(path)
val key = stringTo128BitHash(relativeSourcePath)
if (outPaths.isEmpty()) {
val normalizeOutputPaths = normalizeOutputPaths(outPaths, null)
if (normalizeOutputPaths == null) {
mapHandle.map.remove(key)
}
else if (valueOffset == 1) {
val listWithRelativePaths = Array(outPaths.size + 1) {
if (it == 0) relativeSourcePath else relativizer.toRelative(outPaths.get(it - 1))
}
mapHandle.map.put(key, listWithRelativePaths)
}
else {
val listWithRelativePaths = Array(outPaths.size) {
relativizer.toRelative(outPaths.get(it))
}
mapHandle.map.put(key, listWithRelativePaths)
mapHandle.map.put(key, normalizeOutputPaths)
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOutputToTargetMapping.kt
+163
View File
@@ -0,0 +1,163 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage
import com.dynatrace.hash4j.hashing.Hashing
import org.h2.mvstore.MVMap.Decision
import org.h2.mvstore.MVMap.DecisionMaker
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.jps.builders.storage.SourceToOutputMapping
import org.jetbrains.jps.incremental.storage.dataTypes.LongListKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash
@ApiStatus.Internal
class ExperimentalOutputToTargetMapping(
storageManager: StorageManager,
) : OutputToTargetMapping {
private val mapHandle = storageManager.openMap("out-to-target-v1", LongPairKeyDataType, LongListKeyDataType)
override fun removeTargetAndGetSafeToDeleteOutputs(
outputPaths: Collection<String>,
currentTargetId: Int,
srcToOut: SourceToOutputMapping,
): Collection<String> {
val size = outputPaths.size
if (size == 0) {
return emptyList()
}
srcToOut as ExperimentalSourceToOutputMapping
val decisionMaker = LongListRemoveItemDecisionMaker(srcToOut.targetHashId)
val relativizer = srcToOut.relativizer
val result = ArrayList<String>(size)
for (outPath in outputPaths) {
val key = stringTo128BitHash(relativizer.toRelative(outPath))
mapHandle.map.operate(key, null, decisionMaker)
if (!decisionMaker.outStillUsed) {
result.add(outPath)
}
}
return result
}
override fun removeMappings(outputPaths: Collection<String>, buildTargetId: Int, srcToOut: SourceToOutputMapping) {
srcToOut as ExperimentalSourceToOutputMapping
val relativizer = srcToOut.relativizer
val decisionMaker = LongListRemoveItemDecisionMaker(srcToOut.targetHashId)
for (outPath in outputPaths) {
val key = stringTo128BitHash(relativizer.toRelative(outPath))
mapHandle.map.operate(key, null, decisionMaker)
}
}
fun addMappings(normalizeOutputPaths: Array<String>, targetHashId: Long) {
val decisionMaker = LongListAddItemDecisionMaker(targetHashId)
for (outPath in normalizeOutputPaths) {
mapHandle.map.operate(stringTo128BitHash(outPath), null, decisionMaker)
}
}
fun addMapping(normalizeOutputPath: String, targetHashId: Long) {
val decisionMaker = LongListAddItemDecisionMaker(targetHashId)
mapHandle.map.operate(stringTo128BitHash(normalizeOutputPath), null, decisionMaker)
}
fun removeTarget(targetId: String, targetTypeId: String) {
val map = mapHandle.map
val iterator = map.cursor(null)
val decisionMaker = LongListRemoveItemDecisionMaker(targetToHash(targetId, targetTypeId))
while (iterator.hasNext()) {
mapHandle.map.operate(iterator.next(), null, decisionMaker)
}
}
}
private class LongListAddItemDecisionMaker(private val toAdd: Long) : DecisionMaker<LongArray>() {
override fun decide(existingValue: LongArray?, providedValue: LongArray?): Decision {
return when {
existingValue == null || existingValue.isEmpty() -> Decision.PUT
existingValue.contains(toAdd) -> Decision.ABORT
else -> Decision.PUT
}
}
// it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel
override fun <T : LongArray?> selectValue(existingValue: T?, ignore: T?): T? {
if (existingValue == null || existingValue.isEmpty()) {
@Suppress("UNCHECKED_CAST")
return longArrayOf(toAdd) as T
}
else {
// we checked `contains` in `decide`
@Suppress("UNCHECKED_CAST")
return addElementToEnd(existingValue, toAdd) as T?
}
}
}
private class LongListRemoveItemDecisionMaker(private val toRemove: Long) : DecisionMaker<LongArray>() {
private var indexToRemove: Int = -1
@JvmField
var outStillUsed = false
override fun reset() {
indexToRemove = -1
outStillUsed = false
}
override fun decide(existingValue: LongArray?, ignore: LongArray?): Decision {
return when {
existingValue == null -> Decision.ABORT
// empty value list is not normal, recover - just delete record
existingValue.isEmpty() -> Decision.REMOVE
else -> {
indexToRemove = existingValue.indexOf(toRemove)
when {
indexToRemove == -1 -> {
outStillUsed = true
Decision.ABORT
}
existingValue.size == 1 -> Decision.REMOVE
else -> {
outStillUsed = true
Decision.PUT
}
}
}
}
}
// it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel
override fun <T : LongArray?> selectValue(existingValue: T, providedValue: T?): T {
assert(indexToRemove != -1)
@Suppress("UNCHECKED_CAST")
return removeElementAtIndex(existingValue!!, indexToRemove) as T
}
}
private fun removeElementAtIndex(old: LongArray, index: Int): LongArray {
val newSize = old.size - 1
val result = LongArray(newSize)
System.arraycopy(old, 0, result, 0, index)
if (index < newSize) {
System.arraycopy(old, index + 1, result, index, newSize - index)
}
return result
}
private fun addElementToEnd(old: LongArray, element: Long): LongArray {
val result = LongArray(old.size + 1)
System.arraycopy(old, 0, result, 0, old.size)
result[old.size] = element
return result
}
internal fun targetToHash(targetId: String, targetTypeId: String): Long {
val b1 = targetId.toByteArray()
val b2 = targetTypeId.toByteArray()
return Hashing.xxh3_64().hashStream()
.putByteArray(b1)
.putByteArray(b2)
.asLong
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt
+31 -31
View File
@@ -7,7 +7,6 @@ import org.h2.mvstore.MVMap.DecisionMaker
import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.annotations.TestOnly
import org.jetbrains.annotations.VisibleForTesting
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.storage.SourceToOutputMapping
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
@@ -18,6 +17,8 @@ import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash
class ExperimentalSourceToOutputMapping private constructor(
mapHandle: MapHandle<LongArray, Array<String>>,
relativizer: PathRelativizerService,
private val outputToTargetMapping: ExperimentalOutputToTargetMapping?,
@JvmField internal val targetHashId: Long,
) : SourceToOutputMapping, ExperimentalOneToManyPathMapping(mapHandle = mapHandle, relativizer = relativizer, valueOffset = 1) {
companion object {
// we have a lot of targets - reduce GC and reuse map builder
@@ -26,19 +27,6 @@ class ExperimentalSourceToOutputMapping private constructor(
it.setValueType(StringListDataType)
}
fun createSourceToOutputMap(
storageManager: StorageManager,
relativizer: PathRelativizerService,
target: BuildTarget<*>,
): ExperimentalSourceToOutputMapping {
return createSourceToOutputMap(
storageManager = storageManager,
relativizer = relativizer,
targetId = target.id,
targetTypeId = target.targetType.typeId,
)
}
@VisibleForTesting
@Internal
fun createSourceToOutputMap(
@@ -46,6 +34,7 @@ class ExperimentalSourceToOutputMapping private constructor(
relativizer: PathRelativizerService,
targetId: String,
targetTypeId: String,
outputToTargetMapping: ExperimentalOutputToTargetMapping?,
): ExperimentalSourceToOutputMapping {
// we can use composite key and sort by target id, but as we compile targets in parallel:
// * avoid blocking - in-memory lock per map root,
@@ -54,32 +43,39 @@ class ExperimentalSourceToOutputMapping private constructor(
return ExperimentalSourceToOutputMapping(
mapHandle = storageManager.openMap(mapName, mapBuilder),
relativizer = relativizer,
outputToTargetMapping = outputToTargetMapping,
targetHashId = targetToHash(targetId, targetTypeId),
)
}
}
//override fun getKey(sourcePath: String): LongArray {
// val stringKey = relativizer.toRelative(sourcePath).toByteArray()
//
// // We should sort by target id for data locality -
// // the high integer (high) is shifted to the higher 32 bits of the long value,
// // while the low integer (low) occupies the lower 32 bits.
// // When we do compare two long values, the higher 32 bits (which contain the high integer) dictate the order.
// val low = Hashing.komihash5_0().hashBytesToInt(stringKey)
// val high = targetId
// return longArrayOf((high.toLong() shl 32) or (low.toLong() and 0xFFFFFFFFL), Hashing.xxh3_64().hashBytesToLong(stringKey))
//}
override fun setOutputs(path: String, outPaths: List<String>) {
val relativeSourcePath = super.relativizer.toRelative(path)
val key = stringTo128BitHash(relativeSourcePath)
val normalizeOutputPaths = super.normalizeOutputPaths(outPaths, relativeSourcePath)
if (normalizeOutputPaths == null) {
mapHandle.map.remove(key)
}
else {
mapHandle.map.put(key, normalizeOutputPaths)
outputToTargetMapping?.addMappings(normalizeOutputPaths, targetHashId)
}
}
override fun setOutput(sourcePath: String, outputPath: String) {
val relativeSourcePath = relativizer.toRelative(sourcePath)
mapHandle.map.put(stringTo128BitHash(relativeSourcePath), arrayOf(relativeSourcePath, relativizer.toRelative(outputPath)))
val relativeOutputPath = relativizer.toRelative(outputPath)
mapHandle.map.put(stringTo128BitHash(relativeSourcePath), arrayOf(relativeSourcePath, relativeOutputPath))
outputToTargetMapping?.addMapping(relativeOutputPath, targetHashId)
}
override fun appendOutput(sourcePath: String, outputPath: String) {
val relativeSourcePath = relativizer.toRelative(sourcePath)
val relativeOutputPath = relativizer.toRelative(outputPath)
mapHandle.map.operate(stringTo128BitHash(relativeSourcePath),
null,
AddItemDecisionMaker(sourcePath = relativeSourcePath, toAdd = relativizer.toRelative(outputPath)))
AddItemDecisionMaker(sourcePath = relativeSourcePath, toAdd = relativeOutputPath))
outputToTargetMapping?.addMapping(relativeOutputPath, targetHashId)
}
override fun removeOutput(sourcePath: String, outputPath: String) {
@@ -157,6 +153,10 @@ private class AddItemDecisionMaker(private val sourcePath: String, private val t
private class RemoveItemDecisionMaker(private val toRemove: String) : DecisionMaker<Array<String>>() {
private var indexToRemove: Int = -1
override fun reset() {
indexToRemove = -1
}
override fun decide(existingValue: Array<String>?, ignore: Array<String>?): Decision {
return when {
existingValue == null -> Decision.ABORT
@@ -197,8 +197,8 @@ private fun removeElementAtIndex(old: Array<String>, index: Int): Array<String?>
}
private fun addElementToEnd(old: Array<String>, element: String): Array<String?> {
val result = arrayOfNulls<String>(old.size + 1)
System.arraycopy(old, 0, result, 0, old.size)
result[old.size] = element
return result
val result = arrayOfNulls<String>(old.size + 1)
System.arraycopy(old, 0, result, 0, old.size)
result[old.size] = element
return result
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt
+14
View File
@@ -2,6 +2,7 @@
package org.jetbrains.jps.incremental.storage
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.jps.builders.storage.SourceToOutputMapping
import java.io.IOException
internal interface OneToManyPathMapping {
@@ -19,4 +20,17 @@ internal interface OneToManyPathMapping {
interface SourceToOutputMappingCursor : Iterator<String> {
/** [next] must be called beforehand */
val outputPaths: Array<String>
}
@ApiStatus.Internal
interface OutputToTargetMapping {
@Throws(IOException::class)
fun removeTargetAndGetSafeToDeleteOutputs(
outputPaths: Collection<String>,
currentTargetId: Int,
srcToOut: SourceToOutputMapping,
): Collection<String>
@Throws(IOException::class)
fun removeMappings(outputPaths: Collection<String>, buildTargetId: Int, srcToOut: SourceToOutputMapping)
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java
+30 -16
View File
@@ -10,7 +10,9 @@ import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.ints.IntSets;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.builders.storage.SourceToOutputMapping;
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
import java.io.DataInput;
@@ -21,10 +23,10 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
public final class OutputToTargetRegistry extends AbstractStateStorage<Integer, IntSet> {
@ApiStatus.Internal
public final class OutputToTargetRegistry extends AbstractStateStorage<Integer, IntSet> implements OutputToTargetMapping {
private static final DataExternalizer<IntSet> DATA_EXTERNALIZER = new DataExternalizer<>() {
@Override
public void save(@NotNull DataOutput out, IntSet value) throws IOException {
@@ -56,27 +58,25 @@ public final class OutputToTargetRegistry extends AbstractStateStorage<Integer,
appendData(pathHashCode(outputPath), IntSets.singleton(buildTargetId));
}
void addMapping(@NotNull Collection<String> outputPaths, int buildTargetId) throws IOException {
void addMappings(@NotNull Collection<String> outputPaths, int buildTargetId) throws IOException {
IntSet set = IntSets.singleton(buildTargetId);
for (String outputPath : outputPaths) {
appendData(pathHashCode(outputPath), set);
}
}
public void removeMapping(String outputPath, int buildTargetId) throws IOException {
removeMapping(Collections.singleton(outputPath), buildTargetId);
}
public void removeMapping(Collection<String> outputPaths, int buildTargetId) throws IOException {
@Override
public void removeMappings(@NotNull Collection<String> outputPaths, int buildTargetId, @NotNull SourceToOutputMapping srcToOut) throws IOException {
if (outputPaths.isEmpty()) {
return;
}
for (String outputPath : outputPaths) {
final int key = pathHashCode(outputPath);
int key = pathHashCode(outputPath);
synchronized (dataLock) {
final IntSet state = getState(key);
IntSet state = getState(key);
if (state != null) {
final boolean removed = state.remove(buildTargetId);
boolean removed = state.remove(buildTargetId);
if (state.isEmpty()) {
remove(key);
}
@@ -88,7 +88,10 @@ public final class OutputToTargetRegistry extends AbstractStateStorage<Integer,
}
}
public @NotNull Collection<String> getSafeToDeleteOutputs(Collection<String> outputPaths, int currentTargetId) throws IOException {
@Override
public @NotNull Collection<String> removeTargetAndGetSafeToDeleteOutputs(Collection<String> outputPaths,
int currentTargetId,
@NotNull SourceToOutputMapping srcToOut) throws IOException {
int size = outputPaths.size();
if (size == 0) {
return outputPaths;
@@ -98,11 +101,22 @@ public final class OutputToTargetRegistry extends AbstractStateStorage<Integer,
for (String outputPath : outputPaths) {
int key = pathHashCode(outputPath);
synchronized (dataLock) {
IntSet associatedTargets = getState(key);
if (associatedTargets == null || associatedTargets.size() != 1) {
continue;
IntSet state = getState(key);
boolean isSafeToDelete = false;
if (state == null) {
isSafeToDelete = true;
}
if (associatedTargets.contains(currentTargetId)) {
else {
boolean removed = state.remove(currentTargetId);
if (state.isEmpty()) {
remove(key);
isSafeToDelete = true;
}
else if (removed) {
update(key, state);
}
}
if (isSafeToDelete) {
result.add(outputPath);
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/PerTargetMapManager.kt
+7 -5
View File
@@ -6,13 +6,13 @@ import org.jetbrains.jps.builders.storage.SourceToOutputMapping
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType
import java.util.function.UnaryOperator
import java.util.function.Supplier
internal class PerTargetMapManager(
storageManager: StorageManager,
relativizer: PathRelativizerService,
target: BuildTarget<*>,
sourceToOutputMappingWrapper: UnaryOperator<SourceToOutputMapping>,
outputToTargetMapping: Supplier<ExperimentalOutputToTargetMapping>,
) {
@JvmField
val stamp: StampsStorage<*> = if (ProjectStamps.PORTABLE_CACHES) {
@@ -33,11 +33,13 @@ internal class PerTargetMapManager(
}
val sourceToOutputMapping: SourceToOutputMapping by lazy {
sourceToOutputMappingWrapper.apply(ExperimentalSourceToOutputMapping.createSourceToOutputMap(
ExperimentalSourceToOutputMapping.createSourceToOutputMap(
storageManager = storageManager,
relativizer = relativizer,
target = target,
))
targetId = target.id,
targetTypeId = target.targetType.typeId,
outputToTargetMapping = outputToTargetMapping.get(),
)
}
val sourceToForm: ExperimentalOneToManyPathMapping by lazy {
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt
+20 -11
View File
@@ -19,7 +19,7 @@ private val MV_STORE_CACHE_SIZE_IN_MB = System.getProperty("jps.new.storage.cach
private val LOG = logger<StorageManager>()
@ApiStatus.Internal
class StorageManager(@JvmField val file: Path, private val allowedCompactionTimeOnClose: Int) {
class StorageManager constructor(@JvmField val file: Path) {
private val storeValue = SynchronizedClearableLazy {
LOG.debug { "Opening storage $file" }
createOrResetMvStore(file = file, readOnly = false, logSupplier = { LOG })
@@ -61,8 +61,8 @@ class StorageManager(@JvmField val file: Path, private val allowedCompactionTime
storeValue.valueIfInitialized?.let {
storeValue.drop()
val isCompactOnClose = System.getProperty("jps.new.storage.compact.on.close", "true").toBoolean()
it.close(if (isCompactOnClose) 0 else allowedCompactionTimeOnClose)
val isCompactOnClose = System.getProperty("jps.new.storage.compact.on.close", "false").toBoolean()
it.close()
if (isCompactOnClose && Files.exists(file)) {
val time = measureTime {
MVStoreTool.compact(file.toString(), false)
@@ -94,10 +94,10 @@ class StorageManager(@JvmField val file: Path, private val allowedCompactionTime
}
}
fun removeMaps(targetId: String, typeId: String) {
fun removeMaps(targetId: String, targetTypeId: String) {
val store = storeValue.value
for (mapName in store.mapNames) {
if (mapName.startsWith(getMapName(targetId = targetId, targetTypeId = typeId, suffix = ""))) {
if (mapName.startsWith(getMapName(targetId = targetId, targetTypeId = targetTypeId, suffix = ""))) {
store.removeMap(mapName)
}
}
@@ -166,20 +166,29 @@ private fun tryOpenMvStore(file: Path?, readOnly: Boolean, logSupplier: () -> Lo
val store = MVStore.Builder()
.fileName(file?.toAbsolutePath()?.toString())
.backgroundExceptionHandler(storeErrorHandler)
// We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread.
// Additionally, we ensure that the write operation will not slow down any tasks,
// as the actual save will be done in a background thread.
// Use an 8MB threshold for auto-commit instead of the default 1MB -
// if writes are performed too often, do not save intermediate B-Tree pages to disk.
.autoCommitBufferSize(8192)
.cacheSize(MV_STORE_CACHE_SIZE_IN_MB)
.let {
if (readOnly) it.readOnly() else it
}
// We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread.
// Additionally, we ensure that the write operation will not slow down any tasks,
// as the actual save will be done in a background thread.
// Use a 16MB BUFFER for auto-commit instead of the default 1MB -
// if writes are performed too often, do not save intermediate B-Tree pages to disk.
// Or... just disable auto-commit based on the size of unsaved data and save once in 1 minute
.autoCommitBufferSize(0)
.open()
storeErrorHandler.isStoreOpened = true
// versioning isn't required, otherwise the file size will be larger than needed
store.setVersionsToKeep(0)
// We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread.
// Additionally, we ensure that the write operation will not slow down any tasks,
// as the actual save will be done in a background thread.
// Use a 16MB BUFFER for auto-commit instead of the default 1MB -
// if writes are performed too often, do not save intermediate B-Tree pages to disk.
// Or... just disable auto-commit based on the size of unsaved data and save once in 1 minute
store.autoCommitDelay = 60_000
return store
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongListKeyDataType.kt
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage.dataTypes
import org.h2.mvstore.DataUtils.readVarInt
import org.h2.mvstore.WriteBuffer
import org.h2.mvstore.type.DataType
import java.nio.ByteBuffer
internal object LongListKeyDataType : DataType<LongArray> {
override fun isMemoryEstimationAllowed() = true
// don't care about non-ASCII strings for memory estimation
override fun getMemory(obj: LongArray): Int = obj.size * Long.SIZE_BYTES
override fun createStorage(size: Int): Array<LongArray?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
for (value in (storage as Array<LongArray>)) {
buff.putVarInt(value.size)
for (item in value) {
buff.putLong(item)
}
}
}
override fun write(buff: WriteBuffer, obj: LongArray) = throw IllegalStateException("Must not be called")
override fun read(buff: ByteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<LongArray>
for (i in 0 until len) {
storage[i] = LongArray(readVarInt(buff)) {
buff.getLong()
}
}
}
override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called")
override fun binarySearch(key: LongArray, storage: Any, size: Int, initialGuess: Int): Int = throw IllegalStateException("Must not be called")
@Suppress("DuplicatedCode")
override fun compare(a: LongArray, b: LongArray): Int = throw IllegalStateException("Must not be called")
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt
+4 -4
View File
@@ -17,15 +17,15 @@ internal object LongPairKeyDataType : DataType<LongArray> {
override fun isMemoryEstimationAllowed() = true
// don't care about non-ASCII strings for memory estimation
override fun getMemory(obj: LongArray): Int = 16
override fun getMemory(obj: LongArray): Int = 2 * Long.SIZE_BYTES
override fun createStorage(size: Int): Array<LongArray?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
for (key in (storage as Array<LongArray>)) {
buff.putLong(key[0])
buff.putLong(key[1])
for (value in (storage as Array<LongArray>)) {
buff.putLong(value[0])
buff.putLong(value[1])
}
}
jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java
+11 -7
View File
@@ -6,7 +6,6 @@ import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import org.jetbrains.annotations.NotNull;
@@ -81,19 +80,24 @@ public final class BuildResult implements MessageHandler {
}
OutputToTargetRegistry registry = pd.dataManager.getOutputToTargetRegistry();
List<Integer> keys = new IntArrayList(registry.getKeysIterator());
Collections.sort(keys);
OutputToTargetRegistry registry = (OutputToTargetRegistry)pd.dataManager.getOutputToTargetMapping();
List<Integer> keys = registry.getAllKeys();
if (keys.size() > 1) {
keys.sort(null);
}
stream.println("Begin Of OutputToTarget");
for (Integer key : keys) {
IntSet targetsIds = registry.getState(key);
if (targetsIds == null) continue;
final List<String> targetsNames = new ArrayList<>();
if (targetsIds == null) {
continue;
}
List<String> targetsNames = new ArrayList<>();
targetsIds.forEach(value -> {
BuildTarget<?> target = id2Target.get(value);
targetsNames.add(target != null ? getTargetIdWithTypeId(target) : "<unknown " + value + ">");
});
Collections.sort(targetsNames);
targetsNames.sort(null);
stream.println(hashCodeToOutputPath.get(key.intValue()) + " -> " + targetsNames);
}
stream.println("End Of OutputToTarget");
jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java
+2 -2
View File
@@ -201,8 +201,8 @@ public abstract class JpsBuildTestCase extends UsefulTestCase {
BuildTargetsState targetsState = new BuildTargetsState(dataPaths, myModel, buildRootIndex);
PathRelativizerService relativizer = new PathRelativizerService(myModel.getProject());
ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState);
BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, projectStamps, null);
return new ProjectDescriptor(myModel, new BuildFSState(true), dataManager, buildLoggingManager, index,
BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, null);
return new ProjectDescriptor(myModel, new BuildFSState(true), projectStamps, dataManager, buildLoggingManager, index,
targetIndex, buildRootIndex, ignoredFileIndex);
}
catch (IOException e) {
jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt
+1 -1
View File
@@ -25,7 +25,7 @@ class HashStampStorageFuzzTest {
@BeforeProperty
fun setUp() {
file = Files.createTempFile("mvstore", ".db")
storageManager = StorageManager(file!!, 0)
storageManager = StorageManager(file!!)
hashStampStorage = HashStampStorage.createSourceToStampMap(
storageManager = storageManager,
relativizer = PathRelativizerService(),
jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt
+8 -2
View File
@@ -18,18 +18,21 @@ class SourceToOutputMappingFuzzTest {
}
private lateinit var mapping: ExperimentalSourceToOutputMapping
private lateinit var targetMapping: ExperimentalOutputToTargetMapping
private lateinit var storageManager: StorageManager
private var file: Path? = null
@BeforeProperty
fun setUp() {
file = Files.createTempFile("mvstore", ".db")
storageManager = StorageManager(file!!, 0)
storageManager = StorageManager(file!!)
targetMapping = ExperimentalOutputToTargetMapping(storageManager)
mapping = ExperimentalSourceToOutputMapping.createSourceToOutputMap(
storageManager = storageManager,
relativizer = PathRelativizerService(),
targetId = "test-module",
targetTypeId = "java"
targetTypeId = "java",
outputToTargetMapping = targetMapping,
)
}
@@ -118,6 +121,9 @@ class SourceToOutputMappingFuzzTest {
}
assertThat(actualMap).isEqualTo(expectedMap)
for (outputPaths in actualMap.values) {
assertThat(targetMapping.removeTargetAndGetSafeToDeleteOutputs(outputPaths, -1, mapping)).isEqualTo(outputPaths)
}
}
private fun checkCursorAndSourceIterator(source: String, outputs: List<String>) {
jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt
+4 -3
View File
@@ -33,19 +33,20 @@ class StorageManagerTest {
@Test
fun staleMap(@TempDir tempDir: Path) {
val file = tempDir.resolve("jps-cache.db")
val storageManager = StorageManager(file, 0)
val storageManager = StorageManager(file)
try {
val mapping = ExperimentalSourceToOutputMapping.createSourceToOutputMap(
storageManager = storageManager,
relativizer = PathRelativizerService(),
targetId = "test-module",
targetTypeId = "java"
targetTypeId = "java",
outputToTargetMapping = null,
)
mapping.appendOutput("foo/bar/Baz.java", "out/bar/Baz.class")
assertThat(mapping.getOutputs("foo/bar/Baz.java")).containsExactly("out/bar/Baz.class")
storageManager.removeMaps(targetId = "test-module", typeId = "java")
storageManager.removeMaps(targetId = "test-module", targetTypeId = "java")
assertThat(mapping.getOutputs("foo/bar/Baz.java")).isNull()
}
finally {
platform/util/base/api-dump.txt
+1
View File
@@ -420,6 +420,7 @@ f:com.intellij.util.containers.CollectionFactory
- s:createCustomHashingStrategySet(com.intellij.util.containers.HashingStrategy):java.util.Set
- s:createFilePathLinkedMap():java.util.Map
- s:createFilePathLinkedSet():java.util.Set
- s:createFilePathLinkedSet(java.util.Set):java.util.Set
- s:createFilePathMap():java.util.Map
- s:createFilePathMap(I):java.util.Map
- s:createFilePathMap(I,Z):java.util.Map
platform/util/base/src/com/intellij/util/containers/CollectionFactory.java
+8 -3
View File
@@ -279,17 +279,22 @@ public final class CollectionFactory {
public static @NotNull Set<String> createFilePathLinkedSet() {
return SystemInfoRt.isFileSystemCaseSensitive
? new ObjectLinkedOpenHashSet<>()
? new LinkedHashSet<>()
: new ObjectLinkedOpenCustomHashSet<>(FastUtilHashingStrategies.getCaseInsensitiveStringStrategy());
}
public static @NotNull Set<String> createFilePathLinkedSet(@NotNull Set<String> source) {
return SystemInfoRt.isFileSystemCaseSensitive
? new LinkedHashSet<>(source)
: new ObjectLinkedOpenCustomHashSet<>(source, FastUtilHashingStrategies.getCaseInsensitiveStringStrategy());
}
/**
* Create a linked map with key hash strategy according to file system path case sensitivity.
*/
public static @NotNull <V> Map<String, V> createFilePathLinkedMap() {
//noinspection SSBasedInspection
return SystemInfoRt.isFileSystemCaseSensitive
? new Object2ObjectLinkedOpenHashMap<>()
? new LinkedHashMap<>()
: new Object2ObjectLinkedOpenCustomHashMap<>(FastUtilHashingStrategies.getCaseInsensitiveStringStrategy());
}