MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

experimental compact storage for JPS Cache (part 4 - granular write lock for HashStampStorage: per-target)

Browse Source 
GitOrigin-RevId: 74363b526225420573cd23929198c1854c9ffc23
This commit is contained in:
Vladimir Krivosheev
2024-09-19 05:28:30 +00:00
committed by intellij-monorepo-bot
parent 002cb9d690
commit 3b63b96418
33 changed files with 952 additions and 716 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps/jps-builders/api-dump.txt
+8 -15
View File
@@ -1883,7 +1883,7 @@ f:org.jetbrains.jps.cmdline.PreloadedData
f:org.jetbrains.jps.cmdline.ProjectDescriptor
- f:dataManager:org.jetbrains.jps.incremental.storage.BuildDataManager
- f:fsState:org.jetbrains.jps.incremental.fs.BuildFSState
- <init>(org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.ProjectStamps,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V
- <init>(org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V
- getBuildRootIndex():org.jetbrains.jps.builders.BuildRootIndex
- getBuildTargetIndex():org.jetbrains.jps.builders.BuildTargetIndex
- getEncodingConfiguration():org.jetbrains.jps.incremental.CompilerEncodingConfiguration
@@ -2434,7 +2434,7 @@ f:org.jetbrains.jps.incremental.fs.BuildFSState
- isInitialScanPerformed(org.jetbrains.jps.builders.BuildTarget):Z
- isMarkedForRecompilation(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.incremental.fs.CompilationRound,org.jetbrains.jps.builders.BuildRootDescriptor,java.io.File):Z
- load(java.io.DataInputStream,org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.builders.BuildRootIndex):V
- markAllUpToDate(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage):Z
- markAllUpToDate(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage,J):Z
- markDirty(org.jetbrains.jps.incremental.CompileContext,java.io.File,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage,Z):Z
- markDirty(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.incremental.fs.CompilationRound,java.io.File,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage,Z):Z
- markDirtyIfNotDeleted(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.incremental.fs.CompilationRound,java.io.File,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage):Z
@@ -2619,14 +2619,16 @@ a:org.jetbrains.jps.incremental.storage.AbstractStateStorage
f:org.jetbrains.jps.incremental.storage.BuildDataManager
- sf:PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY:java.lang.String
- clean(java.util.function.Consumer):V
- cleanStaleTarget(org.jetbrains.jps.builders.BuildTargetType,java.lang.String):V
- cleanTargetStorages(org.jetbrains.jps.builders.BuildTarget):V
- close():V
- closeSourceToOutputStorages(java.util.Collection):V
- closeSourceToOutputStorages(org.jetbrains.jps.builders.impl.BuildTargetChunk):V
- createDependencyGraph(java.io.File,Z):V
- flush(Z):V
- s:getConcurrencyLevel():I
- getDataPaths():org.jetbrains.jps.builders.storage.BuildDataPaths
- getDependencyGraph():org.jetbrains.jps.dependency.GraphConfiguration
- getFileStampService():org.jetbrains.jps.incremental.storage.ProjectStamps
- getFileStampStorage(org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.incremental.storage.StampsStorage
- s:getMappingsRoot(java.io.File):java.io.File
- getOutputToTargetRegistry():org.jetbrains.jps.incremental.storage.OutputToTargetRegistry
- getRelativizer():org.jetbrains.jps.incremental.relativizer.PathRelativizerService
@@ -2659,7 +2661,6 @@ f:org.jetbrains.jps.incremental.storage.BuildTargetTypeState
f:org.jetbrains.jps.incremental.storage.BuildTargetsState
- <init>(org.jetbrains.jps.builders.storage.BuildDataPaths,org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.builders.BuildRootIndex):V
- clean():V
- cleanStaleTarget(org.jetbrains.jps.builders.BuildTargetType,java.lang.String):V
- getAverageBuildTime(org.jetbrains.jps.builders.BuildTargetType):J
- getBuildRootIndex():org.jetbrains.jps.builders.BuildRootIndex
- getBuildTargetId(org.jetbrains.jps.builders.BuildTarget):I
@@ -2683,7 +2684,6 @@ a:org.jetbrains.jps.incremental.storage.CompositeStorageOwner
- pa:getChildStorages():java.lang.Iterable
f:org.jetbrains.jps.incremental.storage.OneToManyPathsMapping
- org.jetbrains.jps.incremental.storage.AbstractStateStorage
- org.jetbrains.jps.incremental.storage.StorageOwner
- <init>(java.nio.file.Path,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):V
- appendData(java.lang.String,java.lang.String):V
- appendData(java.lang.String,java.util.Collection):V
@@ -2710,17 +2710,10 @@ f:org.jetbrains.jps.incremental.storage.ProjectStamps
- close():V
- getStampStorage():org.jetbrains.jps.incremental.storage.StampsStorage
org.jetbrains.jps.incremental.storage.StampsStorage
- a:clean():V
- a:close():V
- a:force():V
- a:getCurrentStamp(java.nio.file.Path):org.jetbrains.jps.incremental.storage.StampsStorage$Stamp
- a:getPreviousStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.incremental.storage.StampsStorage$Stamp
- a:getCurrentStampIfUpToDate(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget,java.nio.file.attribute.BasicFileAttributes):java.lang.Object
- a:getStorageRoot():java.nio.file.Path
- a:isDirtyStamp(org.jetbrains.jps.incremental.storage.StampsStorage$Stamp,java.nio.file.Path):Z
- a:isDirtyStamp(org.jetbrains.jps.incremental.storage.StampsStorage$Stamp,java.nio.file.Path,java.nio.file.attribute.BasicFileAttributes):Z
- a:removeStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget):V
- a:saveStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget,org.jetbrains.jps.incremental.storage.StampsStorage$Stamp):V
org.jetbrains.jps.incremental.storage.StampsStorage$Stamp
- a:updateStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget,J):V
org.jetbrains.jps.incremental.storage.StorageOwner
- a:clean():V
- a:close():V
jps/jps-builders/intellij.platform.jps.build.tests.iml
+1
View File
@@ -69,5 +69,6 @@
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="mvstore" level="project" />
<orderEntry type="library" scope="TEST" name="memoryfilesystem" level="project" />
</component>
</module>
jps/jps-builders/src/org/jetbrains/jps/backwardRefs/index/CompilerReferenceIndex.java
+89 -80
View File
@@ -4,8 +4,8 @@ package org.jetbrains.jps.backwardRefs.index;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.LowMemoryWatcher;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.openapi.util.io.NioFiles;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.CommonProcessors;
import com.intellij.util.ExceptionUtil;
@@ -52,7 +52,7 @@ public class CompilerReferenceIndex<Input> {
private final NameEnumerator myNameEnumerator;
private final PersistentStringEnumerator myFilePathEnumerator;
private final File myBuildDir;
private final File myIndicesDir;
private final Path myIndicesDir;
private final LowMemoryWatcher myLowMemoryWatcher = LowMemoryWatcher.register(() -> force());
private volatile Throwable myRebuildRequestCause;
@@ -71,15 +71,20 @@ public class CompilerReferenceIndex<Input> {
@Nullable PathRelativizerService relativizer, boolean readOnly, int version,
boolean isCaseSensitive) {
myBuildDir = buildDir;
myIndicesDir = getIndexDir(buildDir);
if (!myIndicesDir.exists() && !myIndicesDir.mkdirs()) {
throw new RuntimeException("Can't create dir: " + buildDir.getAbsolutePath());
myIndicesDir = getIndexDir(buildDir.toPath());
try {
Files.createDirectories(myIndicesDir);
}
catch (IOException e) {
throw new RuntimeException(e);
}
try {
if (versionDiffers(buildDir, version)) {
saveVersion(buildDir, version);
}
myFilePathEnumerator = new PersistentStringEnumerator(new File(myIndicesDir, FILE_ENUM_TAB).toPath()) {
myFilePathEnumerator = new PersistentStringEnumerator(myIndicesDir.resolve(FILE_ENUM_TAB)) {
@Override
public int enumerate(String path) throws IOException {
@@ -109,7 +114,7 @@ public class CompilerReferenceIndex<Input> {
myIndices.put(indexExtension.getName(), createCompilerIndex(indexExtension, readOnly));
}
myNameEnumerator = new NameEnumerator(new File(myIndicesDir, NAME_ENUM_TAB));
myNameEnumerator = new NameEnumerator(myIndicesDir.resolve(NAME_ENUM_TAB).toFile());
}
catch (IOException e) {
//IJPL-2855: must close all storages opened
@@ -209,51 +214,15 @@ public class CompilerReferenceIndex<Input> {
removeIndexFiles(buildDir, null);
}
public static void removeIndexFiles(File buildDir, Throwable cause) {
final File indexDir = getIndexDir(buildDir);
if (indexDir.exists()) {
try {
FileUtilRt.deleteRecursively(indexDir.toPath());
LOG.info("backward reference index deleted", cause != null ? cause : new Exception());
}
catch (Throwable e) {
LOG.info("failed to delete backward reference index", e);
}
}
}
private static File getIndexDir(@NotNull File buildDir) {
return new File(buildDir, "backward-refs");
}
public static boolean exists(@NotNull File buildDir) {
return getIndexDir(buildDir).exists();
}
public static boolean versionDiffers(@NotNull File buildDir, int expectedVersion) {
Path versionFile = getIndexDir(buildDir).toPath().resolve(VERSION_FILE);
try (DataInputStream is = new DataInputStream(Files.newInputStream(versionFile))) {
int currentIndexVersion = is.readInt();
boolean isDiffer = currentIndexVersion != expectedVersion;
if (isDiffer) {
LOG.info("backward reference index version differ, expected = " + expectedVersion + ", current = " + currentIndexVersion);
}
return isDiffer;
}
catch (NoSuchFileException ignore) {
LOG.info("backward reference index version doesn't exist");
public void saveVersion(@NotNull File buildDir, int version) {
Path versionFile = getIndexDir(buildDir.toPath()).resolve(VERSION_FILE);
try {
NioFiles.createParentDirectories(versionFile);
}
catch (IOException e) {
LOG.info("backward reference index version differ due to: " + e.getClass());
throw new UncheckedIOException(e);
}
return true;
}
public void saveVersion(@NotNull File buildDir, int version) {
File versionFile = new File(getIndexDir(buildDir), VERSION_FILE);
FileUtil.createIfDoesntExist(versionFile);
try (DataOutputStream os = new DataOutputStream(new FileOutputStream(versionFile))) {
try (DataOutputStream os = new DataOutputStream(Files.newOutputStream(versionFile))) {
os.writeInt(version);
}
catch (IOException ex) {
@@ -262,12 +231,67 @@ public class CompilerReferenceIndex<Input> {
}
}
public File getIndicesDir() {
return myIndicesDir.toFile();
}
private <Key, Value> @NotNull CompilerMapReduceIndex<Key, Value> createCompilerIndex(@NotNull IndexExtension<Key, Value, ? super Input> indexExtension,
boolean readOnly) throws IOException {
IndexStorage<Key, Value> indexStorage = createIndexStorage(
indexExtension.getKeyDescriptor(),
indexExtension.getValueExternalizer(),
indexExtension.getName(),
myIndicesDir.toFile(),
readOnly
);
try {
if (readOnly) {
//noinspection unchecked,rawtypes
return new CompilerMapReduceIndex(indexExtension, indexStorage, /* forwardIndex: */ null, /* forwardIndexAccessor: */ null);
}
else {
Path storagePath = myIndicesDir.resolve(indexExtension.getName().getName() + ".inputs");
ForwardIndex forwardIndex = new PersistentMapBasedForwardIndex(storagePath, /* readOnly: */ false);
try {
ForwardIndexAccessor<Key, Value> forwardIndexAccessor = new KeyCollectionForwardIndexAccessor<>(indexExtension);
//noinspection unchecked,rawtypes
return new CompilerMapReduceIndex(indexExtension, indexStorage, forwardIndex, forwardIndexAccessor);
}
catch (Throwable t) {//IJPL-2855: must close all storages opened
forwardIndex.close();
throw t;
}
}
}
catch (Throwable t) {//IJPL-2855: must close all storages opened
indexStorage.close();
throw t;
}
}
public static void removeIndexFiles(File buildDir, Throwable cause) {
Path indexDir = getIndexDir(buildDir.toPath());
if (Files.exists(indexDir)) {
try {
FileUtilRt.deleteRecursively(indexDir);
LOG.info("backward reference index deleted", cause != null ? cause : new Exception());
}
catch (Throwable e) {
LOG.info("failed to delete backward reference index", e);
}
}
}
private static @NotNull Path getIndexDir(@NotNull Path buildDir) {
return buildDir.resolve("backward-refs");
}
public Throwable getRebuildRequestCause() {
return myRebuildRequestCause;
}
public File getIndicesDir() {
return myIndicesDir;
public static boolean exists(@NotNull File buildDir) {
return Files.exists(getIndexDir(buildDir.toPath()));
}
public void setRebuildRequestCause(Throwable e) {
@@ -321,38 +345,23 @@ public class CompilerReferenceIndex<Input> {
}
}
private <Key, Value> @NotNull CompilerMapReduceIndex<Key, Value> createCompilerIndex(@NotNull IndexExtension<Key, Value, ? super Input> indexExtension,
boolean readOnly) throws IOException {
IndexStorage<Key, Value> indexStorage = createIndexStorage(
indexExtension.getKeyDescriptor(),
indexExtension.getValueExternalizer(),
indexExtension.getName(),
myIndicesDir,
readOnly
);
try {
if (readOnly) {
//noinspection unchecked,rawtypes
return new CompilerMapReduceIndex(indexExtension, indexStorage, /* forwardIndex: */ null, /* forwardIndexAccessor: */ null);
}
else {
Path storagePath = new File(myIndicesDir, indexExtension.getName().getName() + ".inputs").toPath();
ForwardIndex forwardIndex = new PersistentMapBasedForwardIndex(storagePath, /* readOnly: */ false);
try {
ForwardIndexAccessor<Key, Value> forwardIndexAccessor = new KeyCollectionForwardIndexAccessor<>(indexExtension);
//noinspection unchecked,rawtypes
return new CompilerMapReduceIndex(indexExtension, indexStorage, forwardIndex, forwardIndexAccessor);
}
catch (Throwable t) {//IJPL-2855: must close all storages opened
forwardIndex.close();
throw t;
}
public static boolean versionDiffers(@NotNull File buildDir, int expectedVersion) {
Path versionFile = getIndexDir(buildDir.toPath()).resolve(VERSION_FILE);
try (DataInputStream is = new DataInputStream(Files.newInputStream(versionFile))) {
int currentIndexVersion = is.readInt();
boolean isDiffer = currentIndexVersion != expectedVersion;
if (isDiffer) {
LOG.info("backward reference index version differ, expected = " + expectedVersion + ", current = " + currentIndexVersion);
}
return isDiffer;
}
catch (Throwable t) {//IJPL-2855: must close all storages opened
indexStorage.close();
throw t;
catch (NoSuchFileException ignore) {
LOG.info("backward reference index version doesn't exist");
}
catch (IOException e) {
LOG.info("backward reference index version differ due to: " + e.getClass());
}
return true;
}
private static <Key, Value> IndexStorage<Key, Value> createIndexStorage(@NotNull KeyDescriptor<Key> keyDescriptor,
jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java
+1 -1
View File
@@ -116,7 +116,7 @@ public class Mappings {
createImplementation();
}
public Mappings(final File rootDir, PathRelativizerService relativizer) throws IOException {
public Mappings(final File rootDir, @NotNull PathRelativizerService relativizer) throws IOException {
myLock = new Object();
myIsDelta = false;
myChangedClasses = null;
jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java
+28 -30
View File
@@ -72,16 +72,10 @@ public final class BuildRunner {
return load(msgHandler, dataStorageRoot.toPath(), fsState);
}
private static @NotNull ProjectStamps initProjectStampStorage(@NotNull Path dataStorageRoot,
@NotNull PathRelativizerService relativizer,
@NotNull BuildTargetsState targetsState,
@Nullable StorageManager storageManager)
throws IOException {
private static @Nullable ProjectStamps initProjectStampStorage(@NotNull Path dataStorageRoot,
@NotNull BuildTargetsState targetsState) throws IOException {
if (ProjectStamps.PORTABLE_CACHES) {
// allow compaction on close (not more than 10 seconds) to ensure minimal storage size
assert storageManager != null;
HashStampStorage stampStorage = new HashStampStorage(storageManager, relativizer, targetsState);
return new ProjectStamps(stampStorage);
return null;
}
else {
return new ProjectStamps(dataStorageRoot, targetsState);
@@ -100,14 +94,14 @@ public final class BuildRunner {
PathRelativizerService relativizer = new PathRelativizerService(jpsModel.getProject(), isCompilerReferenceFSCaseSensitive());
ProjectStamps projectStamps = null;
ProjectStamps fileStampService = null;
BuildDataManager dataManager = null;
StorageManager storageManager = null;
try {
storageManager = createStorageManager(dataStorageRoot);
projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager);
fileStampService = initProjectStampStorage(dataStorageRoot, targetsState);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager);
if (dataManager.versionDiffers()) {
myForceCleanCaches = true;
msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO,
@@ -121,26 +115,28 @@ public final class BuildRunner {
if (storageManager != null) {
storageManager.forceClose();
}
if (projectStamps != null) {
projectStamps.close();
}
if (dataManager != null) {
dataManager.close();
}
else if (fileStampService != null) {
fileStampService.close();
}
myForceCleanCaches = true;
FileUtilRt.deleteRecursively(dataStorageRoot);
storageManager = createStorageManager(dataStorageRoot);
targetsState = new BuildTargetsState(dataPaths, jpsModel, buildRootIndex);
projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager);
fileStampService = initProjectStampStorage(dataStorageRoot, targetsState);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager);
// the second attempt succeeded
msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO,
JpsBuildBundle.message("build.message.project.rebuild.forced.0", e.getMessage())));
}
return new ProjectDescriptor(
jpsModel, fsState, projectStamps, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex
jpsModel, fsState, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex
);
}
@@ -206,8 +202,11 @@ public final class BuildRunner {
return createCompilationScope(pd, scopes, myFilePaths, forceClean, false);
}
private static CompileScope createCompilationScope(ProjectDescriptor pd, List<TargetTypeBuildScope> scopes, Collection<String> paths,
final boolean forceClean, final boolean includeDependenciesToScope) throws Exception {
private static CompileScope createCompilationScope(@NotNull ProjectDescriptor projectDescriptor,
@NotNull List<TargetTypeBuildScope> scopes,
@NotNull Collection<String> paths,
boolean forceClean,
boolean includeDependenciesToScope) throws Exception {
Set<BuildTargetType<?>> targetTypes = new HashSet<>();
Set<BuildTargetType<?>> targetTypesToForceBuild = new HashSet<>();
Set<BuildTarget<?>> targets = new HashSet<>();
@@ -227,7 +226,7 @@ public final class BuildRunner {
targetTypes.add(targetType);
}
else {
BuildTargetLoader<?> loader = targetType.createLoader(pd.getModel());
BuildTargetLoader<?> loader = targetType.createLoader(projectDescriptor.getModel());
for (String targetId : scope.getTargetIdList()) {
BuildTarget<?> target = loader.createTarget(targetId);
if (target != null) {
@@ -240,11 +239,13 @@ public final class BuildRunner {
}
}
if (includeDependenciesToScope) {
includeDependenciesToScope(targetTypes, targets, targetTypesToForceBuild, pd);
includeDependenciesToScope(targetTypes, targets, targetTypesToForceBuild, projectDescriptor);
}
final StampsStorage<? extends StampsStorage.Stamp> stampsStorage = pd.getProjectStamps().getStampStorage();
if (!paths.isEmpty()) {
if (paths.isEmpty()) {
files = Collections.emptyMap();
}
else {
boolean forceBuildAllModuleBasedTargets = false;
for (BuildTargetType<?> type : targetTypesToForceBuild) {
if (type instanceof JavaModuleBuildTargetType) {
@@ -255,7 +256,7 @@ public final class BuildRunner {
files = new HashMap<>();
for (String path : paths) {
final File file = new File(path);
final Collection<BuildRootDescriptor> descriptors = pd.getBuildRootIndex().findAllParentDescriptors(file, null);
final Collection<BuildRootDescriptor> descriptors = projectDescriptor.getBuildRootIndex().findAllParentDescriptors(file, null);
for (BuildRootDescriptor descriptor : descriptors) {
Set<File> fileSet = files.get(descriptor.getTarget());
if (fileSet == null) {
@@ -266,16 +267,13 @@ public final class BuildRunner {
if (added) {
final BuildTargetType<?> targetType = descriptor.getTarget().getTargetType();
if (targetTypesToForceBuild.contains(targetType) || (forceBuildAllModuleBasedTargets && targetType instanceof ModuleBasedBuildTargetType)) {
pd.fsState.markDirty(null, file, descriptor, stampsStorage, false);
StampsStorage<?> stampStorage = projectDescriptor.dataManager.getFileStampStorage(descriptor.getTarget());
projectDescriptor.fsState.markDirty(null, file, descriptor, stampStorage, false);
}
}
}
}
}
else {
files = Collections.emptyMap();
}
return new CompileScopeImpl(targetTypes, targetTypesToForceBuild, targets, files);
}
jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java
+17 -9
View File
@@ -15,7 +15,10 @@ import com.intellij.util.concurrency.SequentialTaskExecutor;
import com.intellij.util.io.DataOutputStream;
import com.intellij.util.io.StorageLockContext;
import io.netty.channel.Channel;
import org.jetbrains.annotations.*;
import org.jetbrains.annotations.Nls;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.api.*;
import org.jetbrains.jps.builders.*;
import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType;
@@ -26,6 +29,7 @@ import org.jetbrains.jps.incremental.TargetTypeRegistry;
import org.jetbrains.jps.incremental.Utils;
import org.jetbrains.jps.incremental.fs.BuildFSState;
import org.jetbrains.jps.incremental.messages.*;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.incremental.storage.ProjectStamps;
import org.jetbrains.jps.incremental.storage.StampsStorage;
import org.jetbrains.jps.model.module.JpsModule;
@@ -457,7 +461,9 @@ final class BuildSession implements Runnable, CanceledStatus {
});
}
private static void applyFSEvent(ProjectDescriptor pd, @Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent event, final boolean saveEventStamp) throws IOException {
private static void applyFSEvent(@NotNull ProjectDescriptor projectDescriptor,
@Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent event,
boolean saveEventStamp) throws IOException {
if (event == null) {
return;
}
@@ -466,8 +472,8 @@ final class BuildSession implements Runnable, CanceledStatus {
LOG.debug("applyFSEvent ordinal=" + event.getOrdinal());
}
final BuildRootIndex buildRootIndex = pd.getBuildRootIndex();
final StampsStorage<? extends StampsStorage.Stamp> stampStorage = pd.getProjectStamps().getStampStorage();
final BuildRootIndex buildRootIndex = projectDescriptor.getBuildRootIndex();
BuildDataManager dataManager = projectDescriptor.dataManager;
for (String deleted : event.getDeletedPathsList()) {
final File file = new File(deleted);
Collection<BuildRootDescriptor> descriptor = buildRootIndex.findAllParentDescriptors(file, null, null);
@@ -476,7 +482,8 @@ final class BuildSession implements Runnable, CanceledStatus {
LOG.debug("Applying deleted path from fs event: " + file.getPath());
}
for (BuildRootDescriptor rootDescriptor : descriptor) {
pd.fsState.registerDeleted(null, rootDescriptor.getTarget(), file, stampStorage);
StampsStorage<?> stampStorage = dataManager.getFileStampStorage(rootDescriptor.getTarget());
projectDescriptor.fsState.registerDeleted(null, rootDescriptor.getTarget(), file, stampStorage);
}
}
else if (LOG.isDebugEnabled()) {
@@ -505,12 +512,13 @@ final class BuildSession implements Runnable, CanceledStatus {
LOG.debug("Applying dirty path from fs event: " + file);
}
for (BuildRootDescriptor descriptor : descriptors) {
StampsStorage.Stamp stamp = stampStorage.getPreviousStamp(file, descriptor.getTarget());
if (stamp == null || stampStorage.isDirtyStamp(stamp, file)) {
pd.fsState.markDirty(null, file.toFile(), descriptor, stampStorage, saveEventStamp);
StampsStorage<?> stampStorage = dataManager.getFileStampStorage(descriptor.getTarget());
Object currentUpToDateStamp = stampStorage.getCurrentStampIfUpToDate(file, descriptor.getTarget(), null);
if (currentUpToDateStamp == null) {
projectDescriptor.fsState.markDirty(null, file.toFile(), descriptor, stampStorage, saveEventStamp);
}
else if (LOG.isDebugEnabled()) {
LOG.debug(descriptor.getTarget() + ": Path considered up-to-date: " + file + "; stamp= " + stamp);
LOG.debug(descriptor.getTarget() + ": Path considered up-to-date: " + file + "; stamp= " + currentUpToDateStamp);
}
}
}
jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java
+40 -24
View File
@@ -1,7 +1,10 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.cmdline;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.TestOnly;
import org.jetbrains.jps.builders.BuildRootIndex;
import org.jetbrains.jps.builders.BuildTarget;
import org.jetbrains.jps.builders.BuildTargetIndex;
import org.jetbrains.jps.builders.logging.BuildLoggingManager;
import org.jetbrains.jps.incremental.CompilerEncodingConfiguration;
@@ -27,8 +30,9 @@ import java.util.Set;
public final class ProjectDescriptor {
private final JpsProject myProject;
private final JpsModel myModel;
@TestOnly
private ProjectStamps deprecatedStamps;
public final BuildFSState fsState;
private final ProjectStamps myProjectStamps;
public final BuildDataManager dataManager;
private final BuildLoggingManager myLoggingManager;
private final ModuleExcludeIndex myModuleExcludeIndex;
@@ -39,18 +43,36 @@ public final class ProjectDescriptor {
private final BuildTargetIndex myBuildTargetIndex;
private final IgnoredFileIndex myIgnoredFileIndex;
/**
* @deprecated Use {@link ProjectDescriptor#ProjectDescriptor(JpsModel, BuildFSState, BuildDataManager, BuildLoggingManager, ModuleExcludeIndex, BuildTargetIndex, BuildRootIndex, IgnoredFileIndex)}
*/
@Deprecated(forRemoval = true)
@ApiStatus.Internal
public ProjectDescriptor(JpsModel model,
BuildFSState fsState,
ProjectStamps projectStamps,
BuildDataManager dataManager,
BuildLoggingManager loggingManager,
final ModuleExcludeIndex moduleExcludeIndex,
final BuildTargetIndex buildTargetIndex, final BuildRootIndex buildRootIndex, IgnoredFileIndex ignoredFileIndex) {
ModuleExcludeIndex moduleExcludeIndex,
BuildTargetIndex buildTargetIndex,
BuildRootIndex buildRootIndex,
IgnoredFileIndex ignoredFileIndex) {
this(model, fsState, dataManager, loggingManager, moduleExcludeIndex, buildTargetIndex, buildRootIndex, ignoredFileIndex);
deprecatedStamps = projectStamps;
}
public ProjectDescriptor(JpsModel model,
BuildFSState fsState,
BuildDataManager dataManager,
BuildLoggingManager loggingManager,
ModuleExcludeIndex moduleExcludeIndex,
BuildTargetIndex buildTargetIndex,
BuildRootIndex buildRootIndex,
IgnoredFileIndex ignoredFileIndex) {
myModel = model;
myIgnoredFileIndex = ignoredFileIndex;
myProject = model.getProject();
this.fsState = fsState;
myProjectStamps = projectStamps;
this.dataManager = dataManager;
myBuildTargetIndex = buildTargetIndex;
myBuildRootIndex = buildRootIndex;
@@ -106,26 +128,11 @@ public final class ProjectDescriptor {
shouldClose = myUseCounter == 0;
}
if (shouldClose) {
if (dataManager.getStorageManager() == null) {
try {
myProjectStamps.close();
}
finally {
try {
dataManager.close();
}
catch (IOException e) {
e.printStackTrace(System.err);
}
}
try {
dataManager.close();
}
else {
try {
dataManager.close();
}
catch (IOException e) {
e.printStackTrace(System.err);
}
catch (IOException e) {
e.printStackTrace(System.err);
}
}
}
@@ -142,7 +149,16 @@ public final class ProjectDescriptor {
return myProject;
}
/**
* @deprecated Use {@link BuildDataManager#getFileStampStorage(BuildTarget)}.
*/
@Deprecated(forRemoval = true)
public ProjectStamps getProjectStamps() {
return myProjectStamps;
//noinspection TestOnlyProblems
if (deprecatedStamps != null) {
return deprecatedStamps;
}
//noinspection removal
return dataManager.getFileStampService();
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java
+18 -17
View File
@@ -33,20 +33,20 @@ import java.util.*;
public final class BuildOperations {
private BuildOperations() { }
public static void ensureFSStateInitialized(CompileContext context, BuildTarget<?> target, boolean readOnly) throws IOException {
final ProjectDescriptor pd = context.getProjectDescriptor();
final StampsStorage<? extends StampsStorage.Stamp> stampsStorage = pd.getProjectStamps().getStampStorage();
final BuildTargetConfiguration configuration = pd.getTargetsState().getTargetConfiguration(target);
public static void ensureFSStateInitialized(@NotNull CompileContext context, @NotNull BuildTarget<?> target, boolean readOnly) throws IOException {
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
BuildTargetConfiguration configuration = projectDescriptor.getTargetsState().getTargetConfiguration(target);
if (JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)) {
FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampsStorage, true, null, null);
pd.fsState.markInitialScanPerformed(target);
StampsStorage<?> stampStorage = projectDescriptor.dataManager.getFileStampStorage(target);
FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampStorage, true, null, null);
projectDescriptor.fsState.markInitialScanPerformed(target);
if (!readOnly) {
configuration.save(context);
}
}
else {
boolean isTargetDirty = false;
if (context.getScope().isBuildForced(target) || (isTargetDirty = configuration.isTargetDirty(context.getProjectDescriptor())) || (!pd.getBuildRootIndex().getTargetRoots(target, context).isEmpty() && configuration.outputRootWasDeleted(context))) {
if (context.getScope().isBuildForced(target) || (isTargetDirty = configuration.isTargetDirty(context.getProjectDescriptor())) || (!projectDescriptor.getBuildRootIndex().getTargetRoots(target, context).isEmpty() && configuration.outputRootWasDeleted(context))) {
if (isTargetDirty) {
configuration.logDiagnostics(context);
}
@@ -56,34 +56,34 @@ public final class BuildOperations {
// case when target build is forced, is handled separately
IncProjectBuilder.clearOutputFiles(context, target);
}
pd.dataManager.cleanTargetStorages(target);
projectDescriptor.dataManager.cleanTargetStorages(target);
configuration.save(context);
}
}
else if (!pd.fsState.isInitialScanPerformed(target)) {
else if (!projectDescriptor.fsState.isInitialScanPerformed(target)) {
initTargetFSState(context, target, false);
}
}
}
private static void initTargetFSState(CompileContext context, BuildTarget<?> target, final boolean forceMarkDirty) throws IOException {
final ProjectDescriptor pd = context.getProjectDescriptor();
final StampsStorage<? extends StampsStorage.Stamp> stampsStorage = pd.getProjectStamps().getStampStorage();
final ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
StampsStorage<?> stampStorage = projectDescriptor.dataManager.getFileStampStorage(target);
Set<File> currentFiles = FileCollectionFactory.createCanonicalFileSet();
FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampsStorage, forceMarkDirty, currentFiles, null);
FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampStorage, forceMarkDirty, currentFiles, null);
// handle deleted paths
final BuildFSState fsState = pd.fsState;
final SourceToOutputMapping sourceToOutputMap = pd.dataManager.getSourceToOutputMap(target);
final BuildFSState fsState = projectDescriptor.fsState;
final SourceToOutputMapping sourceToOutputMap = projectDescriptor.dataManager.getSourceToOutputMap(target);
for (final Iterator<String> it = sourceToOutputMap.getSourcesIterator(); it.hasNext(); ) {
final String path = it.next();
// can check if the file exists
final File file = new File(path);
if (!currentFiles.contains(file)) {
fsState.registerDeleted(context, target, file, stampsStorage);
fsState.registerDeleted(context, target, file, stampStorage);
}
}
pd.fsState.markInitialScanPerformed(target);
projectDescriptor.fsState.markInitialScanPerformed(target);
}
public static void markTargetsUpToDate(CompileContext context, BuildTargetChunk chunk) throws IOException {
@@ -102,7 +102,8 @@ public final class BuildOperations {
if (target instanceof ModuleBuildTarget) {
context.clearNonIncrementalMark((ModuleBuildTarget)target);
}
StampsStorage<? extends StampsStorage.Stamp> stampStorage = projectDescriptor.getProjectStamps().getStampStorage();
StampsStorage<?> stampStorage = projectDescriptor.dataManager.getFileStampStorage(target);
long targetBuildStartStamp = context.getCompilationStartStamp(target);
for (BuildRootDescriptor buildRootDescriptor : projectDescriptor.getBuildRootIndex().getTargetRoots(target, context)) {
marked |= fsState.markAllUpToDate(context, buildRootDescriptor, stampStorage, targetBuildStartStamp);
jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java
+10 -8
View File
@@ -1,4 +1,4 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental;
import com.intellij.openapi.util.Pair;
@@ -6,6 +6,7 @@ import com.intellij.openapi.util.io.FileUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.builders.BuildRootDescriptor;
import org.jetbrains.jps.builders.BuildRootIndex;
import org.jetbrains.jps.builders.BuildTarget;
import org.jetbrains.jps.cmdline.ProjectDescriptor;
import org.jetbrains.jps.incremental.fs.BuildFSState;
import org.jetbrains.jps.incremental.messages.FileDeletedEvent;
@@ -26,17 +27,18 @@ final class ChainedTargetsBuildListener implements BuildListener {
@Override
public void filesGenerated(@NotNull FileGeneratedEvent event) {
final ProjectDescriptor pd = myContext.getProjectDescriptor();
final BuildFSState fsState = pd.fsState;
final ProjectDescriptor projectDescriptor = myContext.getProjectDescriptor();
final BuildFSState fsState = projectDescriptor.fsState;
for (Pair<String, String> pair : event.getPaths()) {
final String relativePath = pair.getSecond();
final File file = relativePath.equals(".") ? new File(pair.getFirst()) : new File(pair.getFirst(), relativePath);
for (BuildRootDescriptor desc : pd.getBuildRootIndex().findAllParentDescriptors(file, myContext)) {
if (!event.getSourceTarget().equals(desc.getTarget())) {
// do not mark files belonging to the target that originated the event
// It is assumed that those files will be explicitly marked dirty by particular builder, if needed.
for (BuildRootDescriptor buildRootDescriptor : projectDescriptor.getBuildRootIndex().findAllParentDescriptors(file, myContext)) {
BuildTarget<?> target = buildRootDescriptor.getTarget();
if (!event.getSourceTarget().equals(target)) {
// Do not mark files belonging to the target that originated the event;
// It is assumed that those files will be explicitly marked dirty by a particular builder, if needed.
try {
fsState.markDirty(myContext, file, desc, pd.getProjectStamps().getStampStorage(), false);
fsState.markDirty(myContext, file, buildRootDescriptor, projectDescriptor.dataManager.getFileStampStorage(target), false);
}
catch (IOException ignored) {
}
jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java
+60 -42
View File
@@ -15,6 +15,7 @@ import org.jetbrains.jps.builders.java.JavaBuilderUtil;
import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor;
import org.jetbrains.jps.cmdline.ProjectDescriptor;
import org.jetbrains.jps.incremental.fs.CompilationRound;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.incremental.storage.StampsStorage;
import org.jetbrains.jps.model.java.JpsJavaClasspathKind;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
@@ -59,18 +60,27 @@ public final class FSOperations {
}
public static void markDirty(CompileContext context, final CompilationRound round, final File file) throws IOException {
final JavaSourceRootDescriptor rd = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file);
if (rd != null) {
final ProjectDescriptor pd = context.getProjectDescriptor();
pd.fsState.markDirty(context, round, file, rd, pd.getProjectStamps().getStampStorage(), false);
JavaSourceRootDescriptor rootDescriptor = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file);
if (rootDescriptor != null) {
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
projectDescriptor.fsState.markDirty(context,
round,
file,
rootDescriptor,
projectDescriptor.dataManager.getFileStampStorage(rootDescriptor.target),
false);
}
}
public static void markDirtyIfNotDeleted(CompileContext context, final CompilationRound round, final File file) throws IOException {
final JavaSourceRootDescriptor rd = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file);
if (rd != null) {
final ProjectDescriptor pd = context.getProjectDescriptor();
pd.fsState.markDirtyIfNotDeleted(context, round, file, rd, pd.getProjectStamps().getStampStorage());
JavaSourceRootDescriptor rootDescriptor = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file);
if (rootDescriptor != null) {
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
projectDescriptor.fsState.markDirtyIfNotDeleted(context,
round,
file,
rootDescriptor,
projectDescriptor.dataManager.getFileStampStorage(rootDescriptor.target));
}
}
@@ -101,23 +111,25 @@ public final class FSOperations {
@Override
public DirtyFilesHolderBuilder<R, T> markDirtyFile(T target, File file) throws IOException {
final ProjectDescriptor pd = context.getProjectDescriptor();
final R rd = pd.getBuildRootIndex().findParentDescriptor(file, Collections.singleton(target.getTargetType()), context);
if (rd != null) {
if (pd.fsState.markDirtyIfNotDeleted(context, round, file, rd, pd.getProjectStamps().getStampStorage()) ||
pd.fsState.isMarkedForRecompilation(context, round, rd, file)) {
Map<R, Set<File>> targetFiles = dirtyFiles.get(target);
if (targetFiles == null) {
targetFiles = new HashMap<>();
dirtyFiles.put(target, targetFiles);
}
Set<File> rootFiles = targetFiles.get(rd);
if (rootFiles == null) {
rootFiles = FileCollectionFactory.createCanonicalFileSet();
targetFiles.put(rd, rootFiles);
}
rootFiles.add(file);
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
R rootDescriptor = projectDescriptor.getBuildRootIndex().findParentDescriptor(file, List.of(target.getTargetType()), context);
if (rootDescriptor == null) {
return this;
}
if (projectDescriptor.fsState.markDirtyIfNotDeleted(context, round, file, rootDescriptor, projectDescriptor.dataManager.getFileStampStorage(target)) ||
projectDescriptor.fsState.isMarkedForRecompilation(context, round, rootDescriptor, file)) {
Map<R, Set<File>> targetFiles = dirtyFiles.get(target);
if (targetFiles == null) {
targetFiles = new HashMap<>();
dirtyFiles.put(target, targetFiles);
}
Set<File> rootFiles = targetFiles.get(rootDescriptor);
if (rootFiles == null) {
rootFiles = FileCollectionFactory.createCanonicalFileSet();
targetFiles.put(rootDescriptor, rootFiles);
}
rootFiles.add(file);
}
return this;
}
@@ -158,10 +170,13 @@ public final class FSOperations {
}
public static void markDeleted(CompileContext context, File file) throws IOException {
final JavaSourceRootDescriptor rd = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file);
if (rd != null) {
final ProjectDescriptor pd = context.getProjectDescriptor();
pd.fsState.registerDeleted(context, rd.target, file, pd.getProjectStamps().getStampStorage());
JavaSourceRootDescriptor rootDescriptor = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file);
if (rootDescriptor != null) {
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
projectDescriptor.fsState.registerDeleted(context,
rootDescriptor.target,
file,
projectDescriptor.dataManager.getFileStampStorage(rootDescriptor.target));
}
}
@@ -171,9 +186,15 @@ public final class FSOperations {
}
}
public static void markDirty(CompileContext context, final CompilationRound round, final ModuleBuildTarget target, @Nullable FileFilter filter) throws IOException {
final ProjectDescriptor pd = context.getProjectDescriptor();
markDirtyFiles(context, target, round, pd.getProjectStamps().getStampStorage(), true, null, filter);
public static void markDirty(CompileContext context, CompilationRound round, ModuleBuildTarget target, @Nullable FileFilter filter) throws IOException {
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
markDirtyFiles(context,
target,
round,
projectDescriptor.dataManager.getFileStampStorage(target),
true,
null,
filter);
}
public static void markDirtyRecursively(CompileContext context, final CompilationRound round, ModuleChunk chunk) throws IOException {
@@ -224,11 +245,11 @@ public final class FSOperations {
removeTargetsAlreadyMarkedDirty(context, dirtyTargets);
final StampsStorage<? extends StampsStorage.Stamp> stampsStorage = context.getProjectDescriptor().getProjectStamps().getStampStorage();
BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
for (ModuleBuildTarget target : dirtyTargets) {
markDirtyFiles(context, target, round, stampsStorage, true, null, filter);
StampsStorage<?> stampStorage = dataManager.getFileStampStorage(target);
markDirtyFiles(context, target, round, stampStorage, true, null, filter);
}
}
private static Set<JpsModule> getDependentModulesRecursively(final JpsModule module, final JpsJavaClasspathKind kind) {
@@ -248,7 +269,7 @@ public final class FSOperations {
static void markDirtyFiles(CompileContext context,
BuildTarget<?> target,
final CompilationRound round,
StampsStorage<? extends StampsStorage.Stamp> stampsStorage,
@Nullable StampsStorage<?> stampStorage,
boolean forceMarkDirty,
@Nullable Set<? super File> currentFiles,
@Nullable FileFilter filter) throws IOException {
@@ -262,7 +283,7 @@ public final class FSOperations {
if (filter == null) {
context.getProjectDescriptor().fsState.clearRecompile(rd);
}
completelyMarkedDirty &= traverseRecursively(context, rd, round, rd.getRootFile(), stampsStorage, forceMarkDirty, currentFiles, filter);
completelyMarkedDirty &= traverseRecursively(context, rd, round, rd.getRootFile(), stampStorage, forceMarkDirty, currentFiles, filter);
}
if (completelyMarkedDirty) {
@@ -279,7 +300,7 @@ public final class FSOperations {
final BuildRootDescriptor rd,
final CompilationRound round,
final File file,
final @NotNull StampsStorage<? extends StampsStorage.Stamp> stampStorage,
@Nullable StampsStorage<?> stampStorage,
final boolean forceDirty,
@Nullable Set<? super File> currentFiles, @Nullable FileFilter filter) throws IOException {
@@ -294,15 +315,12 @@ public final class FSOperations {
boolean markDirty = forceDirty;
if (!markDirty) {
Path nioFile = file.toPath();
StampsStorage.Stamp previousStamp = stampStorage.getPreviousStamp(nioFile, rd.getTarget());
markDirty = previousStamp == null || (attrs == null
? stampStorage.isDirtyStamp(previousStamp, nioFile)
: stampStorage.isDirtyStamp(previousStamp, nioFile, attrs));
markDirty = stampStorage == null || stampStorage.getCurrentStampIfUpToDate(nioFile, rd.getTarget(), attrs) == null;
}
if (markDirty) {
// if it is a full project rebuild, all storages are already completely cleared;
// so passing null because there is no need to access the storage to clear non-existing data
final StampsStorage<? extends StampsStorage.Stamp> marker = context.isProjectRebuild()? null : stampStorage;
StampsStorage<?> marker = context.isProjectRebuild()? null : stampStorage;
context.getProjectDescriptor().fsState.markDirty(context, round, file, rd, marker, false);
}
if (currentFiles != null) {
jps/jps-builders/src/org/jetbrains/jps/incremental/FileHashUtil.java
+2 -2
View File
@@ -6,6 +6,7 @@ import com.dynatrace.hash4j.hashing.Hashing;
import com.intellij.openapi.util.SystemInfoRt;
import com.intellij.openapi.util.text.StringUtilRt;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.incremental.storage.ProjectStamps;
@@ -17,12 +18,11 @@ import java.nio.file.StandardOpenOption;
@ApiStatus.Internal
public final class FileHashUtil {
/**
* @param path a normalized system-independent path, possibly relative, but without "." and ".." relative references
* @param hash hash sink to be updated
*/
public static void computePathHashCode(@Nullable String path, HashSink hash) {
public static void computePathHashCode(@Nullable String path, @NotNull HashSink hash) {
int length = path == null? 0 : path.length();
if (length == 0) {
hash.putInt(0);
jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java
+76 -88
View File
@@ -1,6 +1,7 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental;
import com.intellij.concurrency.ContextAwareRunnable;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.*;
import com.intellij.openapi.util.io.FileUtil;
@@ -210,16 +211,11 @@ public final class IncProjectBuilder {
checkRebuildRequired(scope);
rebuildRequiredSpan.complete();
BuildDataManager dataManager = myProjectDescriptor.dataManager;
final LowMemoryWatcher memWatcher = LowMemoryWatcher.register(() -> {
JavacMain.clearCompilerZipFileCache();
StorageManager storageManager = myProjectDescriptor.dataManager.getStorageManager();
if (storageManager == null) {
((StorageOwner)myProjectDescriptor.getProjectStamps().getStampStorage()).flush(false);
}
myProjectDescriptor.dataManager.flush(false);
if (storageManager != null) {
storageManager.clearCache();
}
dataManager.flush(false);
dataManager.clearCache();
});
final CleanupTempDirectoryExtension cleaner = CleanupTempDirectoryExtension.getInstance();
@@ -240,8 +236,8 @@ public final class IncProjectBuilder {
Tracer.Span buildSpan = Tracer.start("IncProjectBuilder.runBuild");
runBuild(context, forceCleanCaches);
buildSpan.complete();
myProjectDescriptor.dataManager.saveVersion();
myProjectDescriptor.dataManager.reportUnhandledRelativizerPaths();
dataManager.saveVersion();
dataManager.reportUnhandledRelativizerPaths();
sourcesState.reportSourcesState();
reportRebuiltModules(context);
reportUnprocessedChanges(context);
@@ -305,10 +301,11 @@ public final class IncProjectBuilder {
}
private void checkRebuildRequired(final CompileScope scope) throws RebuildRequestedException {
boolean isDebugEnabled = LOG.isDebugEnabled();
if (myIsTestMode || isAutoBuild()) {
// do not use the heuristic in tests in order to properly test all cases
// automatic builds should not cause start full project rebuilds to avoid situations when rebuild is not expected by user
if (LOG.isDebugEnabled()) {
if (isDebugEnabled) {
LOG.debug("Rebuild heuristic: skipping the check; isTestMode = " + myIsTestMode + "; isAutoBuild = " + isAutoBuild());
}
return;
@@ -316,7 +313,7 @@ public final class IncProjectBuilder {
final BuildTargetsState targetsState = myProjectDescriptor.getTargetsState();
final long timeThreshold = targetsState.getLastSuccessfulRebuildDuration() * 95 / 100; // 95% of last registered clean rebuild time
if (timeThreshold <= 0) {
if (LOG.isDebugEnabled()) {
if (isDebugEnabled) {
LOG.debug("Rebuild heuristic: no stats available");
}
return;
@@ -325,13 +322,13 @@ public final class IncProjectBuilder {
// checking only JavaModuleBuildTargetType because these target types directly correspond to project modules
for (BuildTargetType<?> type : JavaModuleBuildTargetType.ALL_TYPES) {
if (!scope.isBuildIncrementally(type)) {
if (LOG.isDebugEnabled()) {
if (isDebugEnabled) {
LOG.debug("Rebuild heuristic: skipping the check because rebuild is forced for targets of type " + type.getTypeId());
}
return;
}
if (!scope.isAllTargetsOfTypeAffected(type)) {
if (LOG.isDebugEnabled()) {
if (isDebugEnabled) {
LOG.debug("Rebuild heuristic: skipping the check because some targets are excluded from compilation scope, e.g. targets of type " + type.getTypeId());
}
return;
@@ -346,7 +343,7 @@ public final class IncProjectBuilder {
return allTargetsAffected.contains(target.getTargetType()) || scope.isAffected(target);
}
});
if (LOG.isDebugEnabled()) {
if (isDebugEnabled) {
LOG.debug("Rebuild heuristic: estimated build time / timeThreshold : " + estimatedWorkTime + " / " + timeThreshold);
}
@@ -436,11 +433,7 @@ public final class IncProjectBuilder {
private static void flushContext(CompileContext context) {
if (context != null) {
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
if (projectDescriptor.dataManager.getStorageManager() == null) {
((StorageOwner)projectDescriptor.getProjectStamps().getStampStorage()).flush(false);
}
projectDescriptor.dataManager.flush(false);
context.getProjectDescriptor().dataManager.flush(false);
}
ExternalJavacManager server = ExternalJavacManager.KEY.get(context);
@@ -558,7 +551,6 @@ public final class IncProjectBuilder {
}
context.processMessage(new ProgressMessage(JpsBuildBundle.message("progress.message.finished.saving.caches")));
}
}
private void sendElapsedTimeMessages(CompileContext context) {
@@ -653,35 +645,20 @@ public final class IncProjectBuilder {
LOG.info("Cleaned output directories in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - cleanStart) + " ms");
if (cleanCaches) {
try {
if (projectDescriptor.dataManager.getStorageManager() == null) {
((StorageOwner)projectDescriptor.getProjectStamps().getStampStorage()).clean();
}
projectDescriptor.dataManager.clean(myAsyncTasks::add);
}
catch (IOException e) {
if (ex == null) {
ex = new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.timestamps.storage"), e);
ex = new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.compiler.storages"), e);
}
else {
LOG.info("Error cleaning timestamps storage", e);
LOG.info("Error cleaning compiler storages", e);
}
}
finally {
try {
projectDescriptor.dataManager.clean(myAsyncTasks::add);
}
catch (IOException e) {
if (ex == null) {
ex = new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.compiler.storages"), e);
}
else {
LOG.info("Error cleaning compiler storages", e);
}
}
finally {
projectDescriptor.fsState.clearAll();
if (ex != null) {
throw ex;
}
projectDescriptor.fsState.clearAll();
if (ex != null) {
throw ex;
}
}
}
@@ -955,15 +932,14 @@ public final class IncProjectBuilder {
}
}
private static void runTasks(CompileContext context, final List<? extends BuildTask> tasks) throws ProjectBuildException {
private static void runTasks(@NotNull CompileContext context, @NotNull List<? extends BuildTask> tasks) throws ProjectBuildException {
for (BuildTask task : tasks) {
task.build(context);
}
}
private void buildChunks(final CompileContextImpl context, BuildProgress buildProgress) throws ProjectBuildException {
private void buildChunks(@NotNull CompileContextImpl context, @NotNull BuildProgress buildProgress) throws ProjectBuildException {
try {
boolean compileInParallel = isParallelBuild();
if (compileInParallel && MAX_BUILDER_THREADS <= 1) {
LOG.info("Switched off parallel compilation because maximum number of builder threads is less than 2. Set '"
@@ -977,14 +953,15 @@ public final class IncProjectBuilder {
}
else {
// non-parallel build
final ProjectDescriptor pd = context.getProjectDescriptor();
final Runnable flushCommand = Utils.asCountedRunnable(FLUSH_INVOCATIONS_TO_SKIP, () -> pd.dataManager.flush(true));
for (BuildTargetChunk chunk : pd.getBuildTargetIndex().getSortedTargetChunks(context)) {
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
BuildDataManager dataManager = projectDescriptor.dataManager;
final Runnable flushCommand = Utils.asCountedRunnable(FLUSH_INVOCATIONS_TO_SKIP, () -> dataManager.flush(true));
for (BuildTargetChunk chunk : projectDescriptor.getBuildTargetIndex().getSortedTargetChunks(context)) {
try {
buildChunkIfAffected(context, context.getScope(), chunk, buildProgress);
}
finally {
pd.dataManager.closeSourceToOutputStorages(Collections.singleton(chunk));
dataManager.closeSourceToOutputStorages(chunk);
flushCommand.run();
}
}
@@ -1028,12 +1005,14 @@ public final class IncProjectBuilder {
}
}
public List<BuildChunkTask> markAsFinishedAndGetNextReadyTasks() {
List<BuildChunkTask> nextTasks = new SmartList<>();
@Nullable List<BuildChunkTask> getNextReadyTasks() {
List<BuildChunkTask> nextTasks = null;
for (BuildChunkTask task : myTasksDependsOnThis) {
int dependenciesCount = task.myNotBuildDependenciesCount.decrementAndGet();
if (dependenciesCount == 0) {
if (nextTasks == null) {
nextTasks = new SmartList<>();
}
nextTasks.add(task);
}
}
@@ -1042,16 +1021,10 @@ public final class IncProjectBuilder {
}
private final class BuildParallelizer {
private final ExecutorService myParallelBuildExecutor = AppExecutorUtil.createCustomPriorityQueueBoundedApplicationPoolExecutor(
"IncProjectBuilder Executor Pool", SharedThreadPool.getInstance(), MAX_BUILDER_THREADS, (o1, o2) -> {
int p1 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o1).priority : 1;
int p2 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o2).priority : 1;
return Integer.compare(p2, p1);
});
private final CompileContext myContext;
private final BuildProgress myBuildProgress;
private final AtomicReference<Throwable> myException = new AtomicReference<>();
private final CountDownLatch myTasksCountDown;
private final CountDownLatch taskCountDown;
private final List<BuildChunkTask> myTasks;
private final Runnable myFlushCommand;
@@ -1091,7 +1064,6 @@ public final class IncProjectBuilder {
}
collectTaskDependantsSpan.complete();
Tracer.Span prioritisationSpan = Tracer.start("IncProjectBuilder.prioritisation");
// bitset stores indexes of transitively dependant tasks
HashMap<BuildChunkTask, BitSet> chunkToTransitive = new HashMap<>();
@@ -1112,7 +1084,7 @@ public final class IncProjectBuilder {
}
prioritisationSpan.complete();
myTasksCountDown = new CountDownLatch(myTasks.size());
taskCountDown = new CountDownLatch(myTasks.size());
span.complete();
}
@@ -1123,10 +1095,21 @@ public final class IncProjectBuilder {
initialTasks.add(task);
}
}
queueTasks(initialTasks);
@NotNull ExecutorService parallelBuildExecutor =
AppExecutorUtil.createCustomPriorityQueueBoundedApplicationPoolExecutor(
"IncProjectBuilder Executor Pool",
SharedThreadPool.getInstance(),
MAX_BUILDER_THREADS,
(o1, o2) -> {
int p1 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o1).priority : 1;
int p2 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o2).priority : 1;
return Integer.compare(p2, p1);
});
queueTasks(initialTasks, LOG.isDebugEnabled(), parallelBuildExecutor);
try {
myTasksCountDown.await();
taskCountDown.await();
}
catch (InterruptedException e) {
LOG.info(e);
@@ -1141,29 +1124,29 @@ public final class IncProjectBuilder {
}
}
private void queueTasks(List<BuildChunkTask> tasks) {
if (tasks.isEmpty()) return;
ArrayList<BuildChunkTask> sorted = new ArrayList<>(tasks);
sorted.sort(Comparator.comparingLong(BuildChunkTask::getScore).reversed());
private void queueTasks(List<BuildChunkTask> tasks, boolean isDebugLogEnabled, @NotNull ExecutorService parallelBuildExecutor) {
BuildChunkTask[] sorted = tasks.toArray(new BuildChunkTask[0]);
Arrays.sort(sorted, Comparator.comparingLong(BuildChunkTask::getScore).reversed());
if (LOG.isDebugEnabled()) {
final List<BuildTargetChunk> chunksToLog = new ArrayList<>();
if (isDebugLogEnabled) {
List<BuildTargetChunk> chunksToLog = new ArrayList<>(sorted.length);
for (BuildChunkTask task : sorted) {
chunksToLog.add(task.getChunk());
}
final StringBuilder logBuilder = new StringBuilder("Queuing " + chunksToLog.size() + " chunks in parallel: ");
StringBuilder logBuilder = new StringBuilder().append("Queuing ").append(chunksToLog.size()).append(" chunks in parallel: ");
chunksToLog.sort(Comparator.comparing(BuildTargetChunk::toString));
for (BuildTargetChunk chunk : chunksToLog) {
logBuilder.append(chunk.toString()).append("; ");
}
LOG.debug(logBuilder.toString());
}
for (BuildChunkTask task : sorted) {
queueTask(task);
queueTask(task, isDebugLogEnabled, parallelBuildExecutor);
}
}
private abstract class RunnableWithPriority implements Runnable {
private abstract class RunnableWithPriority implements Runnable, ContextAwareRunnable {
public final int priority;
RunnableWithPriority(int priority) {
@@ -1171,9 +1154,9 @@ public final class IncProjectBuilder {
}
}
private void queueTask(final BuildChunkTask task) {
final CompileContext chunkLocalContext = createContextWrapper(myContext);
myParallelBuildExecutor.execute(new RunnableWithPriority(task.getScore()) {
private void queueTask(@NotNull BuildChunkTask task, boolean isDebugLogEnabled, @NotNull ExecutorService parallelBuildExecutor) {
CompileContext chunkLocalContext = createContextWrapper(myContext);
parallelBuildExecutor.execute(new RunnableWithPriority(task.getScore()) {
@Override
public void run() {
try {
@@ -1183,10 +1166,8 @@ public final class IncProjectBuilder {
}
}
finally {
Tracer.Span flush = Tracer.start("flushing");
myProjectDescriptor.dataManager.closeSourceToOutputStorages(Collections.singletonList(task.getChunk()));
myProjectDescriptor.dataManager.closeSourceToOutputStorages(task.getChunk());
myFlushCommand.run();
flush.complete();
}
}
catch (Throwable e) {
@@ -1194,12 +1175,18 @@ public final class IncProjectBuilder {
LOG.info(e);
}
finally {
LOG.debug("Finished compilation of " + task.getChunk().toString());
myTasksCountDown.countDown();
List<BuildChunkTask> nextTasks;
nextTasks = task.markAsFinishedAndGetNextReadyTasks();
if (!nextTasks.isEmpty()) {
queueTasks(nextTasks);
try {
if (isDebugLogEnabled) {
LOG.debug("Finished compilation of " + task.getChunk().toString());
}
List<BuildChunkTask> nextTasks = task.getNextReadyTasks();
if (nextTasks != null && !nextTasks.isEmpty()) {
queueTasks(nextTasks, isDebugLogEnabled, parallelBuildExecutor);
}
}
finally {
taskCountDown.countDown();
}
}
}
@@ -1489,7 +1476,8 @@ public final class IncProjectBuilder {
}
final int buildTargetId = context.getProjectDescriptor().getTargetsState().getBuildTargetId(target);
final boolean shouldPruneEmptyDirs = target instanceof ModuleBasedTarget;
final SourceToOutputMapping sourceToOutputStorage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
final SourceToOutputMapping sourceToOutputStorage = dataManager.getSourceToOutputMap(target);
final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger();
// actually delete outputs associated with removed paths
final Collection<String> pathsForIteration;
@@ -1506,7 +1494,7 @@ public final class IncProjectBuilder {
final Collection<String> outputs = sourceToOutputStorage.getOutputs(deletedSource);
if (outputs != null && !outputs.isEmpty()) {
List<String> deletedOutputPaths = new ArrayList<>();
final OutputToTargetRegistry outputToSourceRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetRegistry();
final OutputToTargetRegistry outputToSourceRegistry = dataManager.getOutputToTargetRegistry();
for (String output : outputToSourceRegistry.getSafeToDeleteOutputs(outputs, buildTargetId)) {
final boolean deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, shouldPruneEmptyDirs ? dirsToDelete : null);
if (deleted) {
@@ -1526,8 +1514,8 @@ public final class IncProjectBuilder {
if (target instanceof ModuleBuildTarget) {
// check if the deleted source was associated with a form
final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
final Collection<String> boundForms = sourceToFormMap.getOutputs(deletedSource);
OneToManyPathMapping sourceToFormMap = dataManager.getSourceToFormMap(target);
Collection<String> boundForms = sourceToFormMap.getOutputs(deletedSource);
if (boundForms != null) {
for (String formPath : boundForms) {
final File formFile = new File(formPath);
jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java
+14 -15
View File
@@ -167,7 +167,7 @@ public final class BuildFSState {
public void registerDeleted(@Nullable CompileContext context,
BuildTarget<?> target,
File file,
@Nullable StampsStorage<? extends StampsStorage.Stamp> stampStorage) throws IOException {
@Nullable StampsStorage<?> stampStorage) throws IOException {
registerDeleted(context, target, file);
if (stampStorage != null) {
stampStorage.removeStamp(file.toPath(), target);
@@ -239,7 +239,7 @@ public final class BuildFSState {
public boolean markDirty(@Nullable CompileContext context,
File file,
final BuildRootDescriptor rd,
@Nullable StampsStorage<? extends StampsStorage.Stamp> stampStorage,
@Nullable StampsStorage<?> stampStorage,
boolean saveEventStamp) throws IOException {
return markDirty(context, CompilationRound.NEXT, file, rd, stampStorage, saveEventStamp);
}
@@ -247,21 +247,21 @@ public final class BuildFSState {
public boolean markDirty(@Nullable CompileContext context,
CompilationRound round,
File file,
final BuildRootDescriptor rd,
@Nullable StampsStorage<? extends StampsStorage.Stamp> stampStorage,
@NotNull BuildRootDescriptor buildRootDescriptor,
@Nullable StampsStorage<?> stampStorage,
boolean saveEventStamp) throws IOException {
final FilesDelta roundDelta = getRoundDelta(round == CompilationRound.NEXT? NEXT_ROUND_DELTA_KEY : CURRENT_ROUND_DELTA_KEY, context);
if (roundDelta != null && isInCurrentContextTargets(context, rd)) {
roundDelta.markRecompile(rd, file);
if (roundDelta != null && isInCurrentContextTargets(context, buildRootDescriptor)) {
roundDelta.markRecompile(buildRootDescriptor, file);
}
final FilesDelta filesDelta = getDelta(rd.getTarget());
final FilesDelta filesDelta = getDelta(buildRootDescriptor.getTarget());
filesDelta.lockData();
try {
final boolean marked = filesDelta.markRecompile(rd, file);
final boolean marked = filesDelta.markRecompile(buildRootDescriptor, file);
if (marked) {
if (LOG.isDebugEnabled()) {
LOG.debug(rd.getTarget() + ": MARKED DIRTY: " + file.getPath());
LOG.debug(buildRootDescriptor.getTarget() + ": MARKED DIRTY: " + file.getPath());
}
if (saveEventStamp) {
final long eventStamp = System.currentTimeMillis();
@@ -270,12 +270,12 @@ public final class BuildFSState {
}
}
if (stampStorage != null) {
stampStorage.removeStamp(file.toPath(), rd.getTarget());
stampStorage.removeStamp(file.toPath(), buildRootDescriptor.getTarget());
}
}
else {
if (LOG.isDebugEnabled()) {
LOG.debug(rd.getTarget() + ": NOT MARKED DIRTY: " + file.getPath());
LOG.debug(buildRootDescriptor.getTarget() + ": NOT MARKED DIRTY: " + file.getPath());
}
}
return marked;
@@ -297,7 +297,7 @@ public final class BuildFSState {
CompilationRound round,
File file,
final BuildRootDescriptor rd,
@Nullable StampsStorage<? extends StampsStorage.Stamp> stampStorage) throws IOException {
@Nullable StampsStorage<?> stampStorage) throws IOException {
final boolean marked = getDelta(rd.getTarget()).markRecompileIfNotDeleted(rd, file);
if (marked && stampStorage != null) {
stampStorage.removeStamp(file.toPath(), rd.getTarget());
@@ -384,7 +384,7 @@ public final class BuildFSState {
*/
public boolean markAllUpToDate(@NotNull CompileContext context,
@NotNull BuildRootDescriptor buildRootDescriptor,
@NotNull StampsStorage stampsStorage,
@NotNull StampsStorage<?> stampStorage,
long targetBuildStartStamp) throws IOException {
boolean marked = false;
final BuildTarget<?> target = buildRootDescriptor.getTarget();
@@ -402,7 +402,6 @@ public final class BuildFSState {
if (scope.isAffected(target, file)) {
Path nioFile = file.toPath();
long currentFileTimestamp = FSOperations.lastModified(nioFile);
StampsStorage.Stamp stamp = stampsStorage.getCurrentStamp(nioFile, currentFileTimestamp);
if (!buildRootDescriptor.isGenerated() && (currentFileTimestamp > targetBuildStartStamp || getEventRegistrationStamp(file) > targetBuildStartStamp)) {
// if the file was modified after the compilation had started,
// do not save the stamp considering a file dirty
@@ -416,7 +415,7 @@ public final class BuildFSState {
}
else {
marked = true;
stampsStorage.saveStamp(nioFile, target, stamp); // todo: ask jeka
stampStorage.updateStamp(nioFile, target, currentFileTimestamp);
}
}
else {
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java
+152 -73
View File
@@ -5,12 +5,15 @@ import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.openapi.util.io.NioFiles;
import com.intellij.tracing.Tracer;
import com.intellij.util.io.PersistentHashMapValueStorage;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import org.jetbrains.jps.builders.BuildTarget;
import org.jetbrains.jps.builders.BuildTargetType;
import org.jetbrains.jps.builders.JpsBuildBundle;
import org.jetbrains.jps.builders.impl.BuildTargetChunk;
import org.jetbrains.jps.builders.impl.storage.BuildTargetStorages;
import org.jetbrains.jps.builders.java.JavaBuilderUtil;
@@ -24,6 +27,7 @@ import org.jetbrains.jps.dependency.impl.Containers;
import org.jetbrains.jps.dependency.impl.DependencyGraphImpl;
import org.jetbrains.jps.dependency.impl.LoggingDependencyGraph;
import org.jetbrains.jps.dependency.impl.PathSourceMapper;
import org.jetbrains.jps.incremental.ProjectBuildException;
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
import java.io.*;
@@ -31,6 +35,7 @@ import java.nio.file.Path;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
@@ -53,8 +58,15 @@ public final class BuildDataManager {
private static final String OUT_TARGET_STORAGE = "out-target";
private static final String MAPPINGS_STORAGE = "mappings";
private static final String SRC_TO_OUTPUT_FILE_NAME = "data";
private final ConcurrentMap<BuildTarget<?>, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>();
private final OneToManyPathMapping mySrcToFormMap;
private final @NotNull ConcurrentMap<BuildTarget<?>, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>();
private final @NotNull ConcurrentMap<BuildTarget<?>, SourceToOutputMappingWrapper> buildTargetToSourceToOutputMapping = new ConcurrentHashMap<>();
// only for new experimental storage
private final @NotNull ConcurrentMap<BuildTarget<?>, PerTargetMapManager> targetToMapManager = new ConcurrentHashMap<>();
private final @Nullable ProjectStamps fileStampService;
private final @Nullable OneToManyPathsMapping sourceToFormMap;
private final Mappings myMappings;
private final Object myGraphManagementLock = new Object();
private DependencyGraph myDepGraph;
@@ -67,29 +79,33 @@ public final class BuildDataManager {
private final PathRelativizerService myRelativizer;
private boolean myProcessConstantsIncrementally = !Boolean.parseBoolean(System.getProperty(PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY, "false"));
private final @NotNull ConcurrentMap<BuildTarget<?>, SourceToOutputMappingWrapper> buildTargetToSourceToOutputMapping = new ConcurrentHashMap<>();
@ApiStatus.Internal
@TestOnly
public BuildDataManager(BuildDataPaths dataPaths,
BuildTargetsState targetsState,
@NotNull PathRelativizerService relativizer) throws IOException {
this(dataPaths, targetsState, relativizer, null, null);
}
@ApiStatus.Internal
public BuildDataManager(BuildDataPaths dataPaths,
BuildTargetsState targetsState,
PathRelativizerService relativizer,
@NotNull PathRelativizerService relativizer,
@Nullable ProjectStamps fileStampService,
@Nullable StorageManager storageManager) throws IOException {
this.fileStampService = fileStampService;
myDataPaths = dataPaths;
myTargetsState = targetsState;
this.storageManager = storageManager;
try {
if (storageManager == null) {
mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer);
}
else {
mySrcToFormMap = new ExperimentalOneToManyPathMapping("source-to-form", storageManager, relativizer);
}
sourceToFormMap = storageManager == null ? new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer) : null;
myOutputToTargetRegistry = new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer);
File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot());
if (JavaBuilderUtil.isDepGraphEnabled()) {
myMappings = null;
createDependencyGraph(mappingsRoot, false);
FileUtil.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), false)); // delete older mappings data if available
FileUtilRt.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), false)); // delete older mappings data if available
LOG.info("Using DependencyGraph-based build incremental analysis");
}
else {
@@ -107,12 +123,15 @@ public final class BuildDataManager {
throw e;
}
myVersionFile = new File(myDataPaths.getDataStorageRoot(), "version.dat");
myDepGraphPathMapper = relativizer != null? new PathSourceMapper(relativizer::toFull, relativizer::toRelative) : new PathSourceMapper();
myDepGraphPathMapper = new PathSourceMapper(relativizer::toFull, relativizer::toRelative);
myRelativizer = relativizer;
}
public @Nullable StorageManager getStorageManager() {
return storageManager;
@ApiStatus.Internal
public void clearCache() {
if (storageManager != null) {
storageManager.clearCache();
}
}
public void setProcessConstantsIncrementally(boolean processInc) {
@@ -135,7 +154,7 @@ public final class BuildDataManager {
try {
FileUtilRt.deleteRecursively(getDataPaths().getTargetDataRoot(targetType, targetId));
if (storageManager != null) {
storageManager.removeStaleMaps(targetId, targetType.getTypeId());
storageManager.removeMaps(targetId, targetType.getTypeId());
}
}
finally {
@@ -148,30 +167,53 @@ public final class BuildDataManager {
}
public @NotNull SourceToOutputMapping getSourceToOutputMap(@NotNull BuildTarget<?> target) throws IOException {
int targetId = myTargetsState.getBuildTargetId(target);
if (storageManager == null) {
try {
return buildTargetToSourceToOutputMapping.computeIfAbsent(target, this::createSourceToOutputMap);
}
catch (BuildDataCorruptedException e) {
LOG.info(e);
throw e.getCause();
}
}
else {
return getPerTargetMapManager(target).getSourceToOutputMapping();
}
}
private @NotNull SourceToOutputMappingWrapper createSourceToOutputMap(@NotNull BuildTarget<?> target) {
SourceToOutputMapping map;
try {
return buildTargetToSourceToOutputMapping.computeIfAbsent(target, t -> {
SourceToOutputMapping map;
if (storageManager == null) {
try {
Path file = myDataPaths.getTargetDataRootDir(t).resolve(SRC_TO_OUTPUT_STORAGE).resolve(SRC_TO_OUTPUT_FILE_NAME);
map = new SourceToOutputMappingImpl(file, myRelativizer);
}
catch (IOException e) {
LOG.info(e);
throw new BuildDataCorruptedException(e);
}
}
else {
map = ExperimentalSourceToOutputMapping.createSourceToOutputMap(storageManager, myRelativizer, target);
}
return new SourceToOutputMappingWrapper(map, targetId);
});
Path file = myDataPaths.getTargetDataRootDir(target).resolve(SRC_TO_OUTPUT_STORAGE).resolve(SRC_TO_OUTPUT_FILE_NAME);
map = new SourceToOutputMappingImpl(file, myRelativizer);
}
catch (BuildDataCorruptedException e) {
catch (IOException e) {
LOG.info(e);
throw e.getCause();
throw new BuildDataCorruptedException(e);
}
return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target));
}
public @Nullable StampsStorage<?> getFileStampStorage(@NotNull BuildTarget<?> target) {
if (storageManager == null) {
return fileStampService == null ? null : fileStampService.getStampStorage();
}
return getPerTargetMapManager(target).stamp;
}
private @NotNull PerTargetMapManager getPerTargetMapManager(@NotNull BuildTarget<?> target) {
return targetToMapManager.computeIfAbsent(target, it -> {
assert storageManager != null;
return new PerTargetMapManager(storageManager, myRelativizer, it, mapping -> new SourceToOutputMappingWrapper(mapping, myTargetsState.getBuildTargetId(it)));
});
}
/**
* @deprecated Use {@link BuildDataManager#getFileStampStorage(BuildTarget)}.
*/
@Deprecated(forRemoval = true)
public @NotNull ProjectStamps getFileStampService() {
return Objects.requireNonNull(fileStampService);
}
@ApiStatus.Internal
@@ -185,8 +227,8 @@ public final class BuildDataManager {
}
@ApiStatus.Internal
public OneToManyPathMapping getSourceToFormMap() {
return mySrcToFormMap;
public @NotNull OneToManyPathMapping getSourceToFormMap(@NotNull BuildTarget<?> target) {
return sourceToFormMap == null ? getPerTargetMapManager(target).getSourceToForm() : sourceToFormMap;
}
@ApiStatus.Internal
@@ -224,6 +266,8 @@ public final class BuildDataManager {
if (sourceToOutput != null && sourceToOutput.myDelegate instanceof StorageOwner) {
((StorageOwner)sourceToOutput.myDelegate).close();
}
targetToMapManager.remove(target);
}
}
finally {
@@ -240,10 +284,20 @@ public final class BuildDataManager {
}
}
public void clean(Consumer<Future<?>> asyncTaskCollector) throws IOException {
public void clean(@NotNull Consumer<Future<?>> asyncTaskCollector) throws IOException {
if (fileStampService != null) {
try {
((StorageOwner)fileStampService.getStampStorage()).clean();
}
catch (Throwable e) {
LOG.error(new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.timestamps.storage"), e));
}
}
try {
allTargetStorages(asyncTaskCollector).clean();
buildTargetToSourceToOutputMapping.clear();
targetToMapManager.clear();
myTargetStorages.clear();
if (storageManager != null) {
storageManager.clean();
@@ -251,8 +305,8 @@ public final class BuildDataManager {
}
finally {
try {
if (mySrcToFormMap instanceof StorageOwner) {
wipeStorage(getSourceToFormsRoot(), (StorageOwner)mySrcToFormMap);
if (sourceToFormMap != null) {
wipeStorage(getSourceToFormsRoot(), sourceToFormMap);
}
}
finally {
@@ -314,14 +368,19 @@ public final class BuildDataManager {
}
public void flush(boolean memoryCachesOnly) {
if (storageManager != null) {
if (storageManager == null) {
if (fileStampService != null) {
((StorageOwner)fileStampService.getStampStorage()).flush(false);
}
}
else if (!memoryCachesOnly) {
storageManager.commit();
}
allTargetStorages().flush(memoryCachesOnly);
myOutputToTargetRegistry.flush(memoryCachesOnly);
if (mySrcToFormMap instanceof StorageOwner) {
((StorageOwner)mySrcToFormMap).flush(memoryCachesOnly);
if (sourceToFormMap != null) {
sourceToFormMap.flush(memoryCachesOnly);
}
Mappings mappings = myMappings;
@@ -334,15 +393,6 @@ public final class BuildDataManager {
public void close() throws IOException {
try {
if (storageManager != null) {
try {
storageManager.close();
}
catch (Throwable e) {
LOG.error(e);
}
}
myTargetsState.save();
try {
allTargetStorages().close();
@@ -351,6 +401,24 @@ public final class BuildDataManager {
myTargetStorages.clear();
buildTargetToSourceToOutputMapping.clear();
}
if (storageManager != null) {
targetToMapManager.clear();
try {
storageManager.close();
}
catch (Throwable e) {
LOG.error(e);
}
}
else if (fileStampService != null) {
try {
fileStampService.close();
}
catch (Throwable e) {
LOG.error(e);
}
}
}
finally {
try {
@@ -358,9 +426,9 @@ public final class BuildDataManager {
}
finally {
try {
if (mySrcToFormMap instanceof StorageOwner) {
synchronized (mySrcToFormMap) {
((StorageOwner)mySrcToFormMap).close();
if (sourceToFormMap != null) {
synchronized (sourceToFormMap) {
sourceToFormMap.close();
}
}
}
@@ -392,27 +460,38 @@ public final class BuildDataManager {
}
}
public void closeSourceToOutputStorages(@NotNull Collection<BuildTargetChunk> chunks) throws IOException {
IOException ex = null;
for (BuildTargetChunk chunk : chunks) {
public void closeSourceToOutputStorages(@NotNull BuildTargetChunk chunk) throws IOException {
if (storageManager != null) {
for (BuildTarget<?> target : chunk.getTargets()) {
try {
SourceToOutputMappingWrapper sourceToOutputMapping = buildTargetToSourceToOutputMapping.remove(target);
if (sourceToOutputMapping != null) {
(((StorageOwner)sourceToOutputMapping.myDelegate)).close();
}
}
catch (IOException e) {
LOG.info(e);
if (ex == null) {
ex = e;
}
buildTargetToSourceToOutputMapping.remove(target);
}
return;
}
Tracer.Span flush = Tracer.start("flushing");
IOException error = null;
for (BuildTarget<?> target : chunk.getTargets()) {
SourceToOutputMappingWrapper sourceToOutputMapping = buildTargetToSourceToOutputMapping.remove(target);
if (sourceToOutputMapping == null) {
continue;
}
StorageOwner delegate = (StorageOwner)sourceToOutputMapping.myDelegate;
try {
delegate.close();
}
catch (IOException e) {
LOG.info(e);
if (error == null) {
error = e;
}
}
}
if (ex != null) {
throw ex;
if (error != null) {
throw error;
}
flush.complete();
}
private @NotNull Path getSourceToOutputMapRoot(BuildTarget<?> target) {
@@ -449,6 +528,7 @@ public final class BuildDataManager {
private static void wipeStorage(@NotNull Path root, @Nullable StorageOwner storage) {
if (storage != null) {
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (storage) {
try {
storage.clean();
@@ -507,7 +587,7 @@ public final class BuildDataManager {
myRelativizer.reportUnhandledPaths();
}
private final class SourceToOutputMappingWrapper implements SourceToOutputMapping, Supplier<@Nullable StorageOwner> {
final class SourceToOutputMappingWrapper implements SourceToOutputMapping, Supplier<@Nullable StorageOwner> {
private final SourceToOutputMapping myDelegate;
private final int myBuildTargetId;
@@ -701,5 +781,4 @@ public final class BuildDataManager {
}
};
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetConfiguration.java
+7 -8
View File
@@ -54,7 +54,7 @@ public final class BuildTargetConfiguration {
catch (NoSuchFileException ignore) {
}
catch (IOException e) {
LOG.info("Cannot load configuration of " + target);
LOG.warn("Cannot load configuration of " + target, e);
}
return "";
}
@@ -77,12 +77,11 @@ public final class BuildTargetConfiguration {
if (LOG.isDebugEnabled()) {
LOG.debug(target + " configuration was changed:");
LOG.debug("Old:");
LOG.debug(configuration);
LOG.debug("New:");
LOG.debug(currentState);
LOG.debug("Old: " + configuration);
LOG.debug("New: " + currentState);
LOG.debug(target + " will be recompiled");
}
if (target instanceof ModuleBuildTarget) {
final JpsModule module = ((ModuleBuildTarget)target).getModule();
synchronized (MODULES_WITH_TARGET_CONFIG_CHANGED_KEY) {
@@ -124,7 +123,7 @@ public final class BuildTargetConfiguration {
return myTargetsState.getDataPaths().getTargetDataRootDir(target).resolve("nonexistent-outputs.dat");
}
private @NotNull String getCurrentState(@NotNull ProjectDescriptor pd) {
private @NotNull String getCurrentState(@NotNull ProjectDescriptor projectDescriptor) {
String state = currentState;
if (state != null) {
return state;
@@ -132,12 +131,12 @@ public final class BuildTargetConfiguration {
if (target instanceof BuildTargetHashSupplier) {
HashStream64 hash = Hashing.komihash5_0().hashStream();
((BuildTargetHashSupplier)target).computeConfigurationDigest(pd, hash);
((BuildTargetHashSupplier)target).computeConfigurationDigest(projectDescriptor, hash);
state = Long.toUnsignedString(hash.getAsLong(), Character.MAX_RADIX);
}
else {
StringWriter out = new StringWriter();
target.writeConfiguration(pd, new PrintWriter(out));
target.writeConfiguration(projectDescriptor, new PrintWriter(out));
state = out.toString();
}
currentState = state;
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java
+21 -23
View File
@@ -68,10 +68,9 @@ public final class BuildTargetSourcesState implements BuildListener {
// Some modules can have same out folder for different BuildTarget's to avoid an extra hash calculation collection will be used
// There are no pre-calculated hashes for entries from this collection in FileStampStorage
private final Map<String, Long> calculatedHashes = new ConcurrentHashMap<>();
private final PathRelativizerService relativizer;
private final BuildTargetIndex buildTargetIndex;
private final BuildRootIndex buildRootIndex;
private final ProjectStamps projectStamps;
private final BuildDataManager dataManager;
private final CompileContext context;
private final String outputFolderPath;
private final Path targetStateStorage;
@@ -79,14 +78,13 @@ public final class BuildTargetSourcesState implements BuildListener {
public BuildTargetSourcesState(@NotNull CompileContext context) {
this.context = context;
ProjectDescriptor pd = context.getProjectDescriptor();
projectStamps = pd.getProjectStamps();
buildRootIndex = pd.getBuildRootIndex();
buildTargetIndex = pd.getBuildTargetIndex();
relativizer = pd.dataManager.getRelativizer();
outputFolderPath = getOutputFolderPath(pd.getProject());
ProjectDescriptor projectDescriptor = context.getProjectDescriptor();
dataManager = projectDescriptor.dataManager;
buildRootIndex = projectDescriptor.getBuildRootIndex();
buildTargetIndex = projectDescriptor.getBuildTargetIndex();
outputFolderPath = getOutputFolderPath(projectDescriptor.getProject());
BuildDataPaths dataPaths = pd.getTargetsState().getDataPaths();
BuildDataPaths dataPaths = projectDescriptor.getTargetsState().getDataPaths();
targetStateStorage = dataPaths.getDataStorageRoot().toPath().resolve(TARGET_SOURCES_STATE_FILE_NAME);
// subscribe to events for reporting only changed build targets
@@ -116,6 +114,7 @@ public final class BuildTargetSourcesState implements BuildListener {
result = Collections.emptyList();
}
else {
PathRelativizerService relativizer = dataManager.getRelativizer();
List<Future<?>> list = new ArrayList<>(buildTargets.size());
for (BuildTarget<?> t : buildTargets) {
list.add(parallelBuildExecutor.submit(() -> {
@@ -265,6 +264,8 @@ public final class BuildTargetSourcesState implements BuildListener {
return;
}
HashStampStorage stampStorage = (HashStampStorage)dataManager.getFileStampStorage(target);
Files.walkFileTree(rootFile, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
@@ -276,7 +277,7 @@ public final class BuildTargetSourcesState implements BuildListener {
if (!buildRootIndex.isFileAccepted(path.toFile(), rootDescriptor)) {
return FileVisitResult.CONTINUE;
}
getFileHash(target, path, rootFile, hash, hashToReuse);
getFileHash(path, rootFile, hash, hashToReuse, stampStorage);
return FileVisitResult.CONTINUE;
}
});
@@ -305,16 +306,13 @@ public final class BuildTargetSourcesState implements BuildListener {
.getAsLong();
}
private void getFileHash(@NotNull BuildTarget<?> target,
@NotNull Path path,
@NotNull Path rootFile,
@NotNull LongArrayList hash,
@NotNull HashStream64 hashToReuse) {
StampsStorage<? extends StampsStorage.Stamp> storage = projectStamps.getStampStorage();
assert storage instanceof HashStampStorage;
HashStampStorage fileStampStorage = (HashStampStorage)storage;
Long fileHash = fileStampStorage.getStoredFileHash(path, target);
if (fileHash == null) {
private static void getFileHash(@NotNull Path path,
@NotNull Path rootFile,
@NotNull LongArrayList hash,
@NotNull HashStream64 hashToReuse,
@NotNull HashStampStorage stampStorage) {
HashStamp stamp = stampStorage.getStoredFileStamp(path);
if (stamp == null) {
return;
}
@@ -325,7 +323,7 @@ public final class BuildTargetSourcesState implements BuildListener {
hash.add(hashToReuse
.reset()
.putLong(fileHash)
.putLong(stamp.hash)
.putString(relativePath)
.getAsLong());
}
@@ -353,8 +351,8 @@ public final class BuildTargetSourcesState implements BuildListener {
return new HashMap<>();
}
private boolean reportStateUnavailable() {
return !PORTABLE_CACHES || projectStamps == null;
private static boolean reportStateUnavailable() {
return !PORTABLE_CACHES;
}
private static @NotNull String toRelative(@NotNull Path target, @NotNull Path rootPath) {
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt
+3 -16
View File
@@ -3,24 +3,16 @@
package org.jetbrains.jps.incremental.storage
import com.dynatrace.hash4j.hashing.Hashing
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType
import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash
@ApiStatus.Internal
open class ExperimentalOneToManyPathMapping protected constructor(
open class ExperimentalOneToManyPathMapping(
@JvmField protected val mapHandle: MapHandle<LongArray, Array<String>>,
@JvmField protected val relativizer: PathRelativizerService,
private val valueOffset: Int = 0,
) : OneToManyPathMapping{
constructor(
mapName: String,
storageManager: StorageManager,
relativizer: PathRelativizerService,
) : this(storageManager.openMap(mapName, LongPairKeyDataType, StringListDataType), relativizer)
) : OneToManyPathMapping {
protected fun getKey(path: String): LongArray = stringTo128BitHash(relativizer.toRelative(path))
@Suppress("ReplaceGetOrSet")
@@ -53,9 +45,4 @@ open class ExperimentalOneToManyPathMapping protected constructor(
final override fun remove(path: String) {
mapHandle.map.remove(getKey(path))
}
}
internal fun stringTo128BitHash(string: String): LongArray {
val bytes = string.toByteArray()
return longArrayOf(Hashing.xxh3_64().hashBytesToLong(bytes), Hashing.komihash5_0().hashBytesToLong(bytes))
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt
+2 -2
View File
@@ -12,6 +12,7 @@ import org.jetbrains.jps.builders.storage.SourceToOutputMapping
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType
import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash
@Internal
class ExperimentalSourceToOutputMapping private constructor(
@@ -25,7 +26,6 @@ class ExperimentalSourceToOutputMapping private constructor(
it.setValueType(StringListDataType)
}
@JvmStatic
fun createSourceToOutputMap(
storageManager: StorageManager,
relativizer: PathRelativizerService,
@@ -50,7 +50,7 @@ class ExperimentalSourceToOutputMapping private constructor(
// we can use composite key and sort by target id, but as we compile targets in parallel:
// * avoid blocking - in-memory lock per map root,
// * avoid a huge B-tree and reduce rebalancing time due to contention.
val mapName = storageManager.getMapName(targetId = targetId, typeId = targetTypeId, suffix = "src-to-out-v1")
val mapName = storageManager.getMapName(targetId = targetId, targetTypeId = targetTypeId, suffix = "src-to-out-v1")
return ExperimentalSourceToOutputMapping(
mapHandle = storageManager.openMap(mapName, mapBuilder),
relativizer = relativizer,
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalTimeStampStorage.kt
+89
View File
@@ -0,0 +1,89 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.jps.incremental.storage
import org.h2.mvstore.DataUtils.readVarLong
import org.h2.mvstore.WriteBuffer
import org.h2.mvstore.type.DataType
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.VisibleForTesting
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.incremental.FSOperations
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash
import java.nio.ByteBuffer
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
@ApiStatus.Internal
class ExperimentalTimeStampStorage private constructor(
private val mapHandle: MapHandle<LongArray, Long>,
private val relativizer: PathRelativizerService,
) : StampsStorage<Long> {
companion object {
@VisibleForTesting
fun createSourceToStampMap(
storageManager: StorageManager,
relativizer: PathRelativizerService,
targetId: String,
targetTypeId: String,
): ExperimentalTimeStampStorage {
val mapName = storageManager.getMapName(targetId, targetTypeId, "file-hash-and-mtime-v1")
return ExperimentalTimeStampStorage(
mapHandle = storageManager.openMap(mapName, LongPairKeyDataType, LongStorageValueType),
relativizer = relativizer,
)
}
}
override fun getStorageRoot(): Path? = null
override fun updateStamp(file: Path, buildTarget: BuildTarget<*>?, currentFileTimestamp: Long) {
mapHandle.map.put(createKey(file), currentFileTimestamp)
}
private fun createKey(file: Path): LongArray = stringTo128BitHash(relativizer.toRelative(file))
override fun removeStamp(file: Path, target: BuildTarget<*>?) {
mapHandle.map.remove(createKey(file))
}
override fun getCurrentStampIfUpToDate(file: Path, target: BuildTarget<*>?, attrs: BasicFileAttributes?): Long? {
return mapHandle.map.get(createKey(file))?.takeIf {
it == (if (attrs == null || !attrs.isRegularFile) FSOperations.lastModified(file) else attrs.lastModifiedTime().toMillis())
}
}
}
private object LongStorageValueType : DataType<Long> {
override fun isMemoryEstimationAllowed() = true
override fun getMemory(obj: Long): Int = Long.SIZE_BYTES
override fun createStorage(size: Int): Array<Long?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
for (value in (storage as Array<Long>)) {
buff.putVarLong(value)
}
}
override fun write(buff: WriteBuffer, obj: Long) = throw IllegalStateException("Must not be called")
override fun read(buff: ByteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<Long>
for (i in 0 until len) {
storage[i] = readVarLong(buff)
}
}
override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called")
override fun compare(a: Long, b: Long) = throw IllegalStateException("Must not be called")
override fun binarySearch(key: Long?, storage: Any?, size: Int, initialGuess: Int) = throw IllegalStateException("Must not be called")
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java
+15 -35
View File
@@ -39,42 +39,28 @@ final class FileTimestampStorage extends AbstractStateStorage<File, TimestampPer
}
@Override
public @Nullable FileTimestamp getPreviousStamp(@NotNull Path file, BuildTarget<?> target) throws IOException {
public FileTimestamp getCurrentStampIfUpToDate(@NotNull Path file, @NotNull BuildTarget<?> target, @Nullable BasicFileAttributes attrs) throws IOException {
TimestampPerTarget[] state = getState(file.toFile());
if (state != null) {
int targetId = myTargetsState.getBuildTargetId(target);
for (TimestampPerTarget timestampPerTarget : state) {
if (timestampPerTarget.targetId == targetId) {
return FileTimestamp.fromLong(timestampPerTarget.timestamp);
}
if (state == null) {
return null;
}
int targetId = myTargetsState.getBuildTargetId(target);
for (TimestampPerTarget timestampPerTarget : state) {
if (timestampPerTarget.targetId == targetId) {
long current = timestampPerTarget.timestamp;
long timestamp = (attrs == null || !attrs.isRegularFile()) ? FSOperations.lastModified(file) : attrs.lastModifiedTime().toMillis();
return current == timestamp ? FileTimestamp.fromLong(current) : null;
}
}
return null;
}
@Override
public @NotNull FileTimestamp getCurrentStamp(@NotNull Path file, long currentFileTimestamp) {
return FileTimestamp.fromLong(currentFileTimestamp);
}
@Override
public boolean isDirtyStamp(@NotNull Stamp stamp, @NotNull Path file) {
return !(stamp instanceof FileTimestamp) || ((FileTimestamp)stamp).myTimestamp != FSOperations.lastModified(file);
}
@Override
public boolean isDirtyStamp(@Nullable Stamp stamp, @NotNull Path file, @NotNull BasicFileAttributes attrs) {
if (!(stamp instanceof FileTimestamp)) return true;
FileTimestamp timestamp = (FileTimestamp) stamp;
// for symlinks, the attr structure reflects the symlink's timestamp and not symlink's target timestamp
return attrs.isRegularFile() ? attrs.lastModifiedTime().toMillis() != timestamp.myTimestamp : isDirtyStamp(timestamp, file);
}
@Override
public void saveStamp(@NotNull Path file, BuildTarget<?> buildTarget, @NotNull FileTimestamp stamp) throws IOException {
public void updateStamp(@NotNull Path file, BuildTarget<?> buildTarget, long currentFileTimestamp) throws IOException {
int targetId = myTargetsState.getBuildTargetId(buildTarget);
File ioFile = file.toFile();
update(ioFile, updateTimestamp(getState(ioFile), targetId, stamp.asLong()));
update(ioFile, updateTimestamp(getState(ioFile), targetId, currentFileTimestamp));
}
private static TimestampPerTarget @NotNull [] updateTimestamp(TimestampPerTarget[] oldState, final int targetId, long timestamp) {
@@ -146,26 +132,20 @@ final class FileTimestampStorage extends AbstractStateStorage<File, TimestampPer
}
}
static final class FileTimestamp implements StampsStorage.Stamp {
static final class FileTimestamp {
private final long myTimestamp;
FileTimestamp(long timestamp) {
myTimestamp = timestamp;
}
long asLong() {
return myTimestamp;
}
static @NotNull FileTimestamp fromLong(long l) {
return new FileTimestamp(l);
}
@Override
public String toString() {
return "Timestamp{" +
"myTimestamp=" + myTimestamp +
'}';
return "Timestamp{myTimestamp=" + myTimestamp + '}';
}
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/HashStampStorage.kt
+38 -126
View File
@@ -3,158 +3,70 @@
package org.jetbrains.jps.incremental.storage
import com.dynatrace.hash4j.hashing.Hashing
import org.h2.mvstore.DataUtils.readVarInt
import org.h2.mvstore.DataUtils.readVarLong
import org.h2.mvstore.WriteBuffer
import org.h2.mvstore.type.DataType
import org.jetbrains.annotations.ApiStatus
import org.jetbrains.annotations.VisibleForTesting
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.incremental.FSOperations
import org.jetbrains.jps.incremental.FileHashUtil
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash
import java.nio.ByteBuffer
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
internal class HashStampStorage(
private val storageManager: StorageManager,
@ApiStatus.Internal
class HashStampStorage private constructor(
private val mapHandle: MapHandle<LongArray, HashStamp>,
private val relativizer: PathRelativizerService,
private val targetState: BuildTargetsState,
) : StampsStorage<HashStamp> {
private val mapHandle = storageManager.openMap("file-hash-and-mtime-v1", HashStampStorageKeyType, HashStampStorageValueType)
override fun getStorageRoot(): Path = storageManager.file
override fun saveStamp(file: Path, buildTarget: BuildTarget<*>, stamp: HashStamp) {
mapHandle.map.put(createKey(buildTarget, file), stamp)
}
private fun createKey(target: BuildTarget<*>, file: Path): HashStampStorageKey {
return HashStampStorageKey(
targetId = targetState.getBuildTargetId(target),
// getBytes is faster (70k op/s vs. 50 op/s)
// use xxh3_64 as it is more proven hash algo than komihash
pathHash = Hashing.xxh3_64().hashBytesToLong(relativizer.toRelative(file).toByteArray()),
)
}
override fun removeStamp(file: Path, target: BuildTarget<*>) {
mapHandle.map.remove(createKey(target, file))
}
override fun getPreviousStamp(file: Path, target: BuildTarget<*>): HashStamp? {
return mapHandle.map.get(createKey(target, file))
}
fun getStoredFileHash(file: Path, target: BuildTarget<*>): Long? {
return mapHandle.map.get(createKey(target, file))?.hash
}
override fun getCurrentStamp(file: Path, currentFileTimestamp: Long): HashStamp {
return HashStamp(hash = FileHashUtil.getFileHash(file), timestamp = currentFileTimestamp)
}
override fun isDirtyStamp(stamp: StampsStorage.Stamp, file: Path): Boolean {
if (stamp !is HashStamp) {
return true
}
if (stamp.timestamp == FSOperations.lastModified(file)) {
return false
}
return stamp.hash != FileHashUtil.getFileHash(file)
}
override fun isDirtyStamp(stamp: StampsStorage.Stamp?, file: Path, attrs: BasicFileAttributes): Boolean {
if (stamp !is HashStamp) {
return true
}
// If equal, then non-dirty.
// If not equal, then we check the hash to avoid marking the file as `dirty` only because of a different timestamp.
// We cannot rely solely on the hash, as getting the last-modified timestamp is much cheaper than computing the file hash.
if ((if (attrs.isRegularFile) attrs.lastModifiedTime().toMillis() else FSOperations.lastModified(file)) == stamp.timestamp) {
return false
}
return stamp.hash != FileHashUtil.getFileHash(file)
}
}
internal class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp
private class HashStampStorageKey(@JvmField val targetId: Int, @JvmField val pathHash: Long)
private object HashStampStorageKeyType : DataType<HashStampStorageKey> {
override fun isMemoryEstimationAllowed() = true
override fun getMemory(obj: HashStampStorageKey): Int = Int.SIZE_BYTES + Long.SIZE_BYTES
override fun createStorage(size: Int): Array<HashStampStorageKey?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
for (key in (storage as Array<HashStampStorageKey>)) {
buff.putVarInt(key.targetId)
// not var long - maybe negative number
buff.putLong(key.pathHash)
companion object {
@VisibleForTesting
fun createSourceToStampMap(
storageManager: StorageManager,
relativizer: PathRelativizerService,
targetId: String,
targetTypeId: String,
): HashStampStorage {
val mapName = storageManager.getMapName(targetId = targetId, targetTypeId = targetTypeId, suffix = "file-hash-and-mtime-v1")
return HashStampStorage(
mapHandle = storageManager.openMap(mapName, LongPairKeyDataType, HashStampStorageValueType),
relativizer = relativizer,
)
}
}
override fun write(buff: WriteBuffer, obj: HashStampStorageKey) = throw IllegalStateException("Must not be called")
override fun getStorageRoot(): Path? = null
override fun read(buff: ByteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<HashStampStorageKey>
for (i in 0 until len) {
storage[i] = HashStampStorageKey(targetId = readVarInt(buff), pathHash = buff.getLong())
}
override fun updateStamp(file: Path, buildTarget: BuildTarget<*>?, currentFileTimestamp: Long) {
mapHandle.map.put(createKey(file), HashStamp(hash = FileHashUtil.getFileHash(file), timestamp = currentFileTimestamp))
}
override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called")
private fun createKey(file: Path): LongArray = stringTo128BitHash(relativizer.toRelative(file))
override fun binarySearch(key: HashStampStorageKey, storage: Any, size: Int, initialGuess: Int): Int {
@Suppress("UNCHECKED_CAST")
storage as Array<HashStampStorageKey>
var low = 0
var high = size - 1
// the cached index minus one, so that for the first time (when cachedCompare is 0), the default value is used
var x = initialGuess - 1
if (x < 0 || x > high) {
x = high ushr 1
}
while (low <= high) {
val b = storage[x]
val compare = when {
key.targetId > b.targetId -> 1
key.targetId < b.targetId -> -1
key.pathHash > b.pathHash -> 1
key.pathHash < b.pathHash -> -1
else -> 0
}
when {
compare > 0 -> low = x + 1
compare < 0 -> high = x - 1
else -> return x
}
x = (low + high) ushr 1
}
return low.inv()
override fun removeStamp(file: Path, target: BuildTarget<*>?) {
mapHandle.map.remove(createKey(file))
}
@Suppress("DuplicatedCode")
override fun compare(a: HashStampStorageKey, b: HashStampStorageKey): Int {
return when {
a.targetId > b.targetId -> 1
a.targetId < b.targetId -> -1
a.pathHash > b.pathHash -> 1
a.pathHash < b.pathHash -> -1
else -> 0
fun getStoredFileStamp(file: Path): HashStamp? {
return mapHandle.map.get(createKey(file))
}
override fun getCurrentStampIfUpToDate(file: Path, target: BuildTarget<*>?, attrs: BasicFileAttributes?): HashStamp? {
return mapHandle.map.get(createKey(file))?.takeIf {
val timestamp = if (attrs == null || !attrs.isRegularFile) FSOperations.lastModified(file) else attrs.lastModifiedTime().toMillis()
timestamp == it.timestamp || it.hash == FileHashUtil.getFileHash(file)
}
}
}
@ApiStatus.Internal
@VisibleForTesting
class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long)
private object HashStampStorageValueType : DataType<HashStamp> {
override fun isMemoryEstimationAllowed() = true
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java
+29 -27
View File
@@ -1,14 +1,15 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.dynatrace.hash4j.hashing.Hashing;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorIntegerDescriptor;
import com.intellij.util.io.PersistentMapBuilder;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.ints.IntSets;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
@@ -16,17 +17,17 @@ import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
public final class OutputToTargetRegistry extends AbstractStateStorage<Integer, IntSet> {
private final PathRelativizerService myRelativizer;
private static final DataExternalizer<IntSet> DATA_EXTERNALIZER = new DataExternalizer<IntSet>() {
private static final DataExternalizer<IntSet> DATA_EXTERNALIZER = new DataExternalizer<>() {
@Override
public void save(final @NotNull DataOutput out, IntSet value) throws IOException {
public void save(@NotNull DataOutput out, IntSet value) throws IOException {
IntIterator iterator = value.iterator();
while (iterator.hasNext()) {
out.writeInt(iterator.nextInt());
@@ -35,27 +36,28 @@ public final class OutputToTargetRegistry extends AbstractStateStorage<Integer,
@Override
public IntSet read(@NotNull DataInput in) throws IOException {
final IntSet result = new IntOpenHashSet();
final DataInputStream stream = (DataInputStream)in;
IntSet result = new IntOpenHashSet();
DataInputStream stream = (DataInputStream)in;
while (stream.available() > 0) {
result.add(in.readInt());
}
return result;
}
};
private final PathRelativizerService relativizer;
OutputToTargetRegistry(@NotNull Path storePath, PathRelativizerService relativizer) throws IOException {
super(PersistentMapBuilder.newBuilder(storePath, EnumeratorIntegerDescriptor.INSTANCE, DATA_EXTERNALIZER));
myRelativizer = relativizer;
this.relativizer = relativizer;
}
void addMapping(String outputPath, int buildTargetId) throws IOException {
addMapping(Collections.singleton(outputPath), buildTargetId);
void addMapping(@NotNull String outputPath, int buildTargetId) throws IOException {
appendData(pathHashCode(outputPath), IntSets.singleton(buildTargetId));
}
void addMapping(Collection<String> outputPaths, int buildTargetId) throws IOException {
final IntSet set = new IntOpenHashSet();
set.add(buildTargetId);
void addMapping(@NotNull Collection<String> outputPaths, int buildTargetId) throws IOException {
IntSet set = IntSets.singleton(buildTargetId);
for (String outputPath : outputPaths) {
appendData(pathHashCode(outputPath), set);
}
@@ -78,26 +80,25 @@ public final class OutputToTargetRegistry extends AbstractStateStorage<Integer,
if (state.isEmpty()) {
remove(key);
}
else {
if (removed) {
update(key, state);
}
else if (removed) {
update(key, state);
}
}
}
}
}
public Collection<String> getSafeToDeleteOutputs(Collection<String> outputPaths, int currentTargetId) throws IOException {
final int size = outputPaths.size();
public @NotNull Collection<String> getSafeToDeleteOutputs(Collection<String> outputPaths, int currentTargetId) throws IOException {
int size = outputPaths.size();
if (size == 0) {
return outputPaths;
}
final Collection<String> result = new ArrayList<>(size);
List<String> result = new ArrayList<>(size);
for (String outputPath : outputPaths) {
final int key = pathHashCode(outputPath);
int key = pathHashCode(outputPath);
synchronized (dataLock) {
final IntSet associatedTargets = getState(key);
IntSet associatedTargets = getState(key);
if (associatedTargets == null || associatedTargets.size() != 1) {
continue;
}
@@ -110,11 +111,12 @@ public final class OutputToTargetRegistry extends AbstractStateStorage<Integer,
}
private int pathHashCode(@NotNull String path) {
String relativePath = myRelativizer.toRelative(path);
// On case-insensitive OS hash calculated from path converted to lower case
String relativePath = relativizer.toRelative(path);
if (ProjectStamps.PORTABLE_CACHES) {
return StringUtil.isEmpty(relativePath) ? 0 : FileUtil.toCanonicalPath(relativePath).hashCode();
return Hashing.xxh3_64().hashBytesToInt(relativePath.getBytes(StandardCharsets.UTF_8));
}
else {
return FileUtilRt.pathHashCode(relativePath);
}
return FileUtil.pathHashCode(relativePath); // todo: better hash function?
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/PerTargetMapManager.kt
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.builders.storage.SourceToOutputMapping
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType
import java.util.function.UnaryOperator
internal class PerTargetMapManager(
storageManager: StorageManager,
relativizer: PathRelativizerService,
target: BuildTarget<*>,
sourceToOutputMappingWrapper: UnaryOperator<SourceToOutputMapping>,
) {
@JvmField
val stamp: StampsStorage<*> = if (ProjectStamps.PORTABLE_CACHES) {
HashStampStorage.createSourceToStampMap(
storageManager = storageManager,
relativizer = relativizer,
targetId = target.id,
targetTypeId = target.targetType.typeId,
)
}
else {
ExperimentalTimeStampStorage.createSourceToStampMap(
storageManager = storageManager,
relativizer = relativizer,
targetId = target.id,
targetTypeId = target.targetType.typeId,
)
}
val sourceToOutputMapping: SourceToOutputMapping by lazy {
sourceToOutputMappingWrapper.apply(ExperimentalSourceToOutputMapping.createSourceToOutputMap(
storageManager = storageManager,
relativizer = relativizer,
target = target,
))
}
val sourceToForm: ExperimentalOneToManyPathMapping by lazy {
ExperimentalOneToManyPathMapping(
mapHandle = storageManager.openMap(
name = storageManager.getMapName(targetId = target.id, targetTypeId = target.targetType.typeId, suffix = "source-to-form-v1"),
keyType = LongPairKeyDataType,
valueType = StringListDataType,
),
relativizer = relativizer,
)
}
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java
+7 -10
View File
@@ -3,7 +3,6 @@ package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.NioFiles;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
@@ -23,15 +22,10 @@ public final class ProjectStamps {
private static final Logger LOG = Logger.getInstance(ProjectStamps.class);
private final StampsStorage<? extends StampsStorage.Stamp> stampStorage;
@ApiStatus.Internal
public ProjectStamps(@NotNull StampsStorage<? extends StampsStorage.Stamp> stampStorage) throws IOException {
this.stampStorage = stampStorage;
}
private final StampsStorage<?> stampStorage;
public ProjectStamps(@NotNull Path dataStorageRoot, @NotNull BuildTargetsState targetsState) throws IOException {
this(new FileTimestampStorage(dataStorageRoot, targetsState));
this.stampStorage = new FileTimestampStorage(dataStorageRoot, targetsState);
}
/**
@@ -43,7 +37,7 @@ public final class ProjectStamps {
this(dataStorageRoot.toPath(), targetsState);
}
public @NotNull StampsStorage<? extends StampsStorage.Stamp> getStampStorage() {
public @NotNull StampsStorage<?> getStampStorage() {
return stampStorage;
}
@@ -56,7 +50,10 @@ public final class ProjectStamps {
catch (IOException e) {
LOG.error(e);
try {
NioFiles.deleteRecursively(stampStorage.getStorageRoot());
Path root = stampStorage.getStorageRoot();
if (root != null) {
NioFiles.deleteRecursively(root);
}
}
catch (IOException ignore) {
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java
+4 -15
View File
@@ -9,27 +9,16 @@ import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import static org.jetbrains.jps.incremental.storage.StampsStorage.Stamp;
/**
* @author Eugene Zhuravlev
*/
public interface StampsStorage<T extends Stamp> {
Path getStorageRoot();
public interface StampsStorage<T> {
@Nullable Path getStorageRoot();
void saveStamp(@NotNull Path file, BuildTarget<?> buildTarget, @NotNull T stamp) throws IOException;
void updateStamp(@NotNull Path file, BuildTarget<?> buildTarget, long currentFileTimestamp) throws IOException;
void removeStamp(@NotNull Path file, BuildTarget<?> buildTarget) throws IOException;
@Nullable
T getPreviousStamp(@NotNull Path file, BuildTarget<?> target) throws IOException;
@NotNull
T getCurrentStamp(@NotNull Path file, long currentFileTimestamp) throws IOException;
boolean isDirtyStamp(@NotNull Stamp stamp, @NotNull Path file) throws IOException;
boolean isDirtyStamp(@Nullable Stamp stamp, @NotNull Path file, @NotNull BasicFileAttributes attrs) throws IOException;
interface Stamp { }
T getCurrentStampIfUpToDate(@NotNull Path file, BuildTarget<?> target, @Nullable BasicFileAttributes attrs) throws IOException;
}
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt
+50 -23
View File
@@ -2,7 +2,8 @@
package org.jetbrains.jps.incremental.storage
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.diagnostic.thisLogger
import com.intellij.openapi.diagnostic.debug
import com.intellij.openapi.diagnostic.logger
import com.intellij.util.concurrency.SynchronizedClearableLazy
import org.h2.mvstore.MVMap
import org.h2.mvstore.MVStore
@@ -15,15 +16,22 @@ import kotlin.time.measureTime
private val MV_STORE_CACHE_SIZE_IN_MB = System.getProperty("jps.new.storage.cache.size.mb", "64").toInt()
private val LOG = logger<StorageManager>()
@ApiStatus.Internal
class StorageManager(@JvmField val file: Path, private val allowedCompactionTimeOnClose: Int) {
private val store = SynchronizedClearableLazy { createOrResetMvStore(file = file, readOnly = false, ::thisLogger) }
private val storeValue = SynchronizedClearableLazy {
LOG.debug { "Opening storage $file" }
createOrResetMvStore(file = file, readOnly = false, logSupplier = { LOG })
}
fun open() {
store.value
storeValue.value
}
fun <K : Any, V : Any> openMap(name: String, keyType: DataType<K>, valueType: DataType<V>): MapHandle<K, V> {
LOG.debug { "Open map $name" }
val mapBuilder = MVMap.Builder<K, V>()
mapBuilder.setKeyType(keyType)
mapBuilder.setValueType(valueType)
@@ -31,55 +39,71 @@ class StorageManager(@JvmField val file: Path, private val allowedCompactionTime
}
fun <K : Any, V : Any> openMap(name: String, mapBuilder: MVMap.Builder<K, V>): MapHandle<K, V> {
return MapHandle(openOrResetMap(store = store.value, name = name, mapBuilder = mapBuilder, logSupplier = ::thisLogger))
return MapHandle(openOrResetMap(store = storeValue.value, name = name, mapBuilder = mapBuilder, logSupplier = { LOG }))
}
/** Only if error occurred */
fun forceClose() {
store.valueIfInitialized?.closeImmediately()
if (LOG.isDebugEnabled) {
LOG.debug("Force closing storage $file", Throwable())
}
storeValue.valueIfInitialized?.let {
storeValue.drop()
it.closeImmediately()
}
}
fun close() {
store.valueIfInitialized?.let {
store.drop()
if (LOG.isDebugEnabled) {
LOG.debug("Closing storage $file", Throwable())
}
storeValue.valueIfInitialized?.let {
storeValue.drop()
val isCompactOnClose = System.getProperty("jps.new.storage.compact.on.close", "true").toBoolean()
it.close(if (isCompactOnClose) 0 else allowedCompactionTimeOnClose)
if (isCompactOnClose && Files.exists(file)) {
val time = measureTime {
MVStoreTool.compact(file.toString(), false)
}
thisLogger().info("Compacted storage in $time")
LOG.info("Compacted storage in $time")
}
}
}
fun commit() {
store.valueIfInitialized?.tryCommit()
storeValue.valueIfInitialized?.tryCommit()
}
fun clearCache() {
// set again to force to clear the cache (in kb)
store.valueIfInitialized?.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024
storeValue.valueIfInitialized?.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024
}
fun clean() {
store.valueIfInitialized?.let {
store.drop()
it.closeImmediately()
val store = storeValue.valueIfInitialized
if (store == null) {
Files.deleteIfExists(file)
}
}
fun removeStaleMaps(targetId: String, typeId: String) {
val store = store.value
for (mapName in store.mapNames) {
if (mapName.startsWith(getMapName(targetId = targetId, typeId = typeId, suffix = ""))) {
else {
// we cannot recreate the store if reference to map is already acquired - so, remove all maps
for (mapName in store.mapNames) {
store.removeMap(mapName)
}
}
}
fun getMapName(targetId: String, typeId: String, suffix: String): String = "$targetId|$typeId|$suffix"
fun removeMaps(targetId: String, typeId: String) {
val store = storeValue.value
for (mapName in store.mapNames) {
if (mapName.startsWith(getMapName(targetId = targetId, targetTypeId = typeId, suffix = ""))) {
store.removeMap(mapName)
}
}
}
fun getMapName(targetId: String, targetTypeId: String, suffix: String): String = "$targetId|$targetTypeId|$suffix"
}
@ApiStatus.Internal
@@ -142,9 +166,12 @@ private fun tryOpenMvStore(file: Path?, readOnly: Boolean, logSupplier: () -> Lo
val store = MVStore.Builder()
.fileName(file?.toAbsolutePath()?.toString())
.backgroundExceptionHandler(storeErrorHandler)
// avoid extra thread - db maintainer should use coroutines
.autoCommitDisabled()
.autoCommitBufferSize(4096)
// We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread.
// Additionally, we ensure that the write operation will not slow down any tasks,
// as the actual save will be done in a background thread.
// Use an 8MB threshold for auto-commit instead of the default 1MB -
// if writes are performed too often, do not save intermediate B-Tree pages to disk.
.autoCommitBufferSize(8192)
.cacheSize(MV_STORE_CACHE_SIZE_IN_MB)
.let {
if (readOnly) it.readOnly() else it
jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt
+8
View File
@@ -1,10 +1,18 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage.dataTypes
import com.dynatrace.hash4j.hashing.Hashing
import org.h2.mvstore.WriteBuffer
import org.h2.mvstore.type.DataType
import java.nio.ByteBuffer
// getBytes is faster (70k op/s vs. 50 op/s)
// use xxh3_64 as first as it is more proven hash algo than komihash
internal fun stringTo128BitHash(string: String): LongArray {
val bytes = string.toByteArray()
return longArrayOf(Hashing.xxh3_64().hashBytesToLong(bytes), Hashing.komihash5_0().hashBytesToLong(bytes))
}
internal object LongPairKeyDataType : DataType<LongArray> {
override fun isMemoryEstimationAllowed() = true
jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java
+2 -2
View File
@@ -201,8 +201,8 @@ public abstract class JpsBuildTestCase extends UsefulTestCase {
BuildTargetsState targetsState = new BuildTargetsState(dataPaths, myModel, buildRootIndex);
PathRelativizerService relativizer = new PathRelativizerService(myModel.getProject());
ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState);
BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, null);
return new ProjectDescriptor(myModel, new BuildFSState(true), projectStamps, dataManager, buildLoggingManager, index,
BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, projectStamps, null);
return new ProjectDescriptor(myModel, new BuildFSState(true), dataManager, buildLoggingManager, index,
targetIndex, buildRootIndex, ignoredFileIndex);
}
catch (IOException e) {
jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt
+85
View File
@@ -0,0 +1,85 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage
import com.github.marschall.memoryfilesystem.MemoryFileSystemBuilder
import com.intellij.util.io.write
import net.jqwik.api.*
import net.jqwik.api.lifecycle.AfterProperty
import net.jqwik.api.lifecycle.BeforeProperty
import org.assertj.core.api.Assertions.assertThat
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import java.nio.file.FileSystem
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.ExperimentalPathApi
import kotlin.io.path.deleteIfExists
import kotlin.io.path.deleteRecursively
class HashStampStorageFuzzTest {
private lateinit var hashStampStorage: HashStampStorage
private lateinit var storageManager: StorageManager
private var file: Path? = null
private lateinit var fs: FileSystem
@BeforeProperty
fun setUp() {
file = Files.createTempFile("mvstore", ".db")
storageManager = StorageManager(file!!, 0)
hashStampStorage = HashStampStorage.createSourceToStampMap(
storageManager = storageManager,
relativizer = PathRelativizerService(),
targetId = "test-module",
targetTypeId = "java"
)
fs = MemoryFileSystemBuilder.newLinux().setCurrentWorkingDirectory("/").build()
}
@AfterProperty
fun tearDown() {
storageManager.close()
file?.deleteIfExists()
fs.close()
}
@Provide
fun paths(): Arbitrary<String> {
return Arbitraries.strings().alpha().numeric().withChars('/').ofMinLength(2).ofMaxLength(255)
}
@Provide
fun hashStamps(): Arbitrary<HashStamp> {
return Arbitraries.longs().map { HashStamp(hash = it, timestamp = System.currentTimeMillis()) }
}
@Property
fun saveAndRetrieveStamp(@ForAll("paths") pathStr: String, @ForAll("hashStamps") stamp: HashStamp) {
val file = prepareAndGetFile(pathStr)
hashStampStorage.updateStamp(file, null, stamp.timestamp)
val retrievedStamp = hashStampStorage.getStoredFileStamp(file)
assertThat(retrievedStamp).isNotNull()
assertThat(retrievedStamp!!.timestamp).isEqualTo(stamp.timestamp)
}
@OptIn(ExperimentalPathApi::class)
private fun prepareAndGetFile(pathStr: String): Path {
val file = fs.getPath("/test", pathStr.replace("//", "/").let { if (it.length == 1) "/test" else it })
file.parent.deleteRecursively()
file.write("")
return file
}
@Property
fun removeAndCheckStamp(@ForAll("paths") pathStr: String) {
val file = prepareAndGetFile(pathStr)
val stamp = HashStamp(hash = 12345L, timestamp = System.currentTimeMillis())
hashStampStorage.updateStamp(file, null, stamp.timestamp)
hashStampStorage.removeStamp(file, null)
val retrievedStamp = hashStampStorage.getStoredFileStamp(file)
assertThat(retrievedStamp).isNull()
}
}
jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt
+1 -1
View File
@@ -45,7 +45,7 @@ class StorageManagerTest {
mapping.appendOutput("foo/bar/Baz.java", "out/bar/Baz.class")
assertThat(mapping.getOutputs("foo/bar/Baz.java")).containsExactly("out/bar/Baz.class")
storageManager.removeStaleMaps(targetId = "test-module", typeId = "java")
storageManager.removeMaps(targetId = "test-module", typeId = "java")
assertThat(mapping.getOutputs("foo/bar/Baz.java")).isNull()
}
finally {
platform/build-scripts/src/org/jetbrains/intellij/build/impl/logging/jps/JpsFileLoggerFactory.kt
+3 -5
View File
@@ -1,4 +1,4 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.intellij.build.impl.logging.jps;
import com.intellij.openapi.diagnostic.IdeaLogRecordFormatter
@@ -13,12 +13,10 @@ import java.util.logging.Level
@ApiStatus.Internal
class JpsFileLoggerFactory(logFile: Path, categoriesWithDebugLevel: String) : Factory {
private val appender: RollingFileHandler
private val categoriesWithDebugLevel: List<String>
private val appender = RollingFileHandler(logFile, 20_000_000L, 10, true)
private val categoriesWithDebugLevel = if (categoriesWithDebugLevel.isEmpty()) emptyList() else categoriesWithDebugLevel.split(',')
init {
this.categoriesWithDebugLevel = if (categoriesWithDebugLevel.isEmpty()) emptyList() else categoriesWithDebugLevel.split(",")
this.appender = RollingFileHandler(logFile, 20_000_000L, 10, true)
this.appender.setFormatter(IdeaLogRecordFormatter())
}
plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java
+17 -15
View File
@@ -22,7 +22,7 @@ import org.jetbrains.jps.incremental.java.CopyResourcesUtil;
import org.jetbrains.jps.incremental.java.FormsParsing;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.storage.OneToManyPathMapping;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.model.JpsProject;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.java.compiler.JpsCompilerExcludes;
@@ -158,22 +158,24 @@ public final class FormsBindingManager extends FormsBuilder {
formsToCompile.keySet().removeAll(alienForms);
// form should be considered dirty if the class it is bound to is dirty
final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
BuildDataManager dataManager = context.getProjectDescriptor().dataManager;
for (Map.Entry<File, ModuleBuildTarget> entry : filesToCompile.entrySet()) {
final File srcFile = entry.getKey();
final ModuleBuildTarget target = entry.getValue();
final Collection<String> boundForms = sourceToFormMap.getOutputs(srcFile.getPath());
if (boundForms != null) {
for (String formPath : boundForms) {
final File formFile = new File(formPath);
if (!excludes.isExcluded(formFile) && formFile.exists()) {
addBinding(srcFile, formFile, srcToForms);
holderBuilder.markDirtyFile(target, formFile);
File srcFile = entry.getKey();
ModuleBuildTarget target = entry.getValue();
Collection<String> boundForms = dataManager.getSourceToFormMap(target).getOutputs(srcFile.getPath());
if (boundForms == null) {
continue;
}
context.getScope().markIndirectlyAffected(target, formFile);
formsToCompile.put(formFile, target);
exitCode = ExitCode.OK;
}
for (String formPath : boundForms) {
File formFile = new File(formPath);
if (!excludes.isExcluded(formFile) && formFile.exists()) {
addBinding(srcFile, formFile, srcToForms);
holderBuilder.markDirtyFile(target, formFile);
context.getScope().markIndirectlyAffected(target, formFile);
formsToCompile.put(formFile, target);
exitCode = ExitCode.OK;
}
}
}
plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java
+2 -4
View File
@@ -86,10 +86,8 @@ public final class FormsInstrumenter extends FormsBuilder {
final InstrumentationClassFinder finder = ClassProcessingBuilder.createInstrumentationClassFinder(sdk, platformCp, classpath, outputConsumer);
try {
final Map<File, Collection<File>> processed = instrumentForms(context, chunk, chunkSourcePath, finder, formsToCompile, outputConsumer, config.isUseDynamicBundles());
final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
Map<File, Collection<File>> processed = instrumentForms(context, chunk, chunkSourcePath, finder, formsToCompile, outputConsumer, config.isUseDynamicBundles());
OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(chunk.representativeTarget());
for (Map.Entry<File, Collection<File>> entry : processed.entrySet()) {
final File src = entry.getKey();
final Collection<File> forms = entry.getValue();