From 3b63b96418b70d43fba07ff7b4c19efd0f14fd6a Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Sun, 15 Sep 2024 18:25:24 +0200 Subject: [PATCH] experimental compact storage for JPS Cache (part 4 - granular write lock for HashStampStorage: per-target) GitOrigin-RevId: 74363b526225420573cd23929198c1854c9ffc23 --- jps/jps-builders/api-dump.txt | 23 +- .../intellij.platform.jps.build.tests.iml | 1 + .../index/CompilerReferenceIndex.java | 169 ++++++------- .../java/dependencyView/Mappings.java | 2 +- .../jetbrains/jps/cmdline/BuildRunner.java | 58 +++-- .../jetbrains/jps/cmdline/BuildSession.java | 26 +- .../jps/cmdline/ProjectDescriptor.java | 64 +++-- .../jps/incremental/BuildOperations.java | 35 +-- .../ChainedTargetsBuildListener.java | 18 +- .../jps/incremental/FSOperations.java | 102 ++++---- .../jps/incremental/FileHashUtil.java | 4 +- .../jps/incremental/IncProjectBuilder.java | 164 ++++++------- .../jps/incremental/fs/BuildFSState.java | 29 ++- .../incremental/storage/BuildDataManager.java | 225 ++++++++++++------ .../storage/BuildTargetConfiguration.java | 15 +- .../storage/BuildTargetSourcesState.java | 44 ++-- .../ExperimentalOneToManyPathMapping.kt | 19 +- .../ExperimentalSourceToOutputMapping.kt | 4 +- .../storage/ExperimentalTimeStampStorage.kt | 89 +++++++ .../storage/FileTimestampStorage.java | 50 ++-- .../incremental/storage/HashStampStorage.kt | 164 +++---------- .../storage/OutputToTargetRegistry.java | 56 ++--- .../storage/PerTargetMapManager.kt | 53 +++++ .../incremental/storage/ProjectStamps.java | 17 +- .../incremental/storage/StampsStorage.java | 19 +- .../jps/incremental/storage/StorageManager.kt | 73 ++++-- .../storage/dataTypes/LongPairKeyDataType.kt | 8 + .../jps/builders/JpsBuildTestCase.java | 4 +- .../storage/HashStampStorageFuzzTest.kt | 85 +++++++ .../incremental/storage/StorageManagerTest.kt | 2 +- .../impl/logging/jps/JpsFileLoggerFactory.kt | 8 +- .../compiler/FormsBindingManager.java | 32 +-- .../compiler/FormsInstrumenter.java | 6 +- 33 files changed, 952 insertions(+), 716 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalTimeStampStorage.kt create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/PerTargetMapManager.kt create mode 100644 jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt diff --git a/jps/jps-builders/api-dump.txt b/jps/jps-builders/api-dump.txt index fa585d41e71a..3601d1e015f6 100644 --- a/jps/jps-builders/api-dump.txt +++ b/jps/jps-builders/api-dump.txt @@ -1883,7 +1883,7 @@ f:org.jetbrains.jps.cmdline.PreloadedData f:org.jetbrains.jps.cmdline.ProjectDescriptor - f:dataManager:org.jetbrains.jps.incremental.storage.BuildDataManager - f:fsState:org.jetbrains.jps.incremental.fs.BuildFSState -- (org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.ProjectStamps,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V +- (org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.incremental.fs.BuildFSState,org.jetbrains.jps.incremental.storage.BuildDataManager,org.jetbrains.jps.builders.logging.BuildLoggingManager,org.jetbrains.jps.indices.ModuleExcludeIndex,org.jetbrains.jps.builders.BuildTargetIndex,org.jetbrains.jps.builders.BuildRootIndex,org.jetbrains.jps.indices.IgnoredFileIndex):V - getBuildRootIndex():org.jetbrains.jps.builders.BuildRootIndex - getBuildTargetIndex():org.jetbrains.jps.builders.BuildTargetIndex - getEncodingConfiguration():org.jetbrains.jps.incremental.CompilerEncodingConfiguration @@ -2434,7 +2434,7 @@ f:org.jetbrains.jps.incremental.fs.BuildFSState - isInitialScanPerformed(org.jetbrains.jps.builders.BuildTarget):Z - isMarkedForRecompilation(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.incremental.fs.CompilationRound,org.jetbrains.jps.builders.BuildRootDescriptor,java.io.File):Z - load(java.io.DataInputStream,org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.builders.BuildRootIndex):V -- markAllUpToDate(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage):Z +- markAllUpToDate(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage,J):Z - markDirty(org.jetbrains.jps.incremental.CompileContext,java.io.File,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage,Z):Z - markDirty(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.incremental.fs.CompilationRound,java.io.File,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage,Z):Z - markDirtyIfNotDeleted(org.jetbrains.jps.incremental.CompileContext,org.jetbrains.jps.incremental.fs.CompilationRound,java.io.File,org.jetbrains.jps.builders.BuildRootDescriptor,org.jetbrains.jps.incremental.storage.StampsStorage):Z @@ -2619,14 +2619,16 @@ a:org.jetbrains.jps.incremental.storage.AbstractStateStorage f:org.jetbrains.jps.incremental.storage.BuildDataManager - sf:PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY:java.lang.String - clean(java.util.function.Consumer):V +- cleanStaleTarget(org.jetbrains.jps.builders.BuildTargetType,java.lang.String):V - cleanTargetStorages(org.jetbrains.jps.builders.BuildTarget):V - close():V -- closeSourceToOutputStorages(java.util.Collection):V +- closeSourceToOutputStorages(org.jetbrains.jps.builders.impl.BuildTargetChunk):V - createDependencyGraph(java.io.File,Z):V - flush(Z):V -- s:getConcurrencyLevel():I - getDataPaths():org.jetbrains.jps.builders.storage.BuildDataPaths - getDependencyGraph():org.jetbrains.jps.dependency.GraphConfiguration +- getFileStampService():org.jetbrains.jps.incremental.storage.ProjectStamps +- getFileStampStorage(org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.incremental.storage.StampsStorage - s:getMappingsRoot(java.io.File):java.io.File - getOutputToTargetRegistry():org.jetbrains.jps.incremental.storage.OutputToTargetRegistry - getRelativizer():org.jetbrains.jps.incremental.relativizer.PathRelativizerService @@ -2659,7 +2661,6 @@ f:org.jetbrains.jps.incremental.storage.BuildTargetTypeState f:org.jetbrains.jps.incremental.storage.BuildTargetsState - (org.jetbrains.jps.builders.storage.BuildDataPaths,org.jetbrains.jps.model.JpsModel,org.jetbrains.jps.builders.BuildRootIndex):V - clean():V -- cleanStaleTarget(org.jetbrains.jps.builders.BuildTargetType,java.lang.String):V - getAverageBuildTime(org.jetbrains.jps.builders.BuildTargetType):J - getBuildRootIndex():org.jetbrains.jps.builders.BuildRootIndex - getBuildTargetId(org.jetbrains.jps.builders.BuildTarget):I @@ -2683,7 +2684,6 @@ a:org.jetbrains.jps.incremental.storage.CompositeStorageOwner - pa:getChildStorages():java.lang.Iterable f:org.jetbrains.jps.incremental.storage.OneToManyPathsMapping - org.jetbrains.jps.incremental.storage.AbstractStateStorage -- org.jetbrains.jps.incremental.storage.StorageOwner - (java.nio.file.Path,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):V - appendData(java.lang.String,java.lang.String):V - appendData(java.lang.String,java.util.Collection):V @@ -2710,17 +2710,10 @@ f:org.jetbrains.jps.incremental.storage.ProjectStamps - close():V - getStampStorage():org.jetbrains.jps.incremental.storage.StampsStorage org.jetbrains.jps.incremental.storage.StampsStorage -- a:clean():V -- a:close():V -- a:force():V -- a:getCurrentStamp(java.nio.file.Path):org.jetbrains.jps.incremental.storage.StampsStorage$Stamp -- a:getPreviousStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget):org.jetbrains.jps.incremental.storage.StampsStorage$Stamp +- a:getCurrentStampIfUpToDate(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget,java.nio.file.attribute.BasicFileAttributes):java.lang.Object - a:getStorageRoot():java.nio.file.Path -- a:isDirtyStamp(org.jetbrains.jps.incremental.storage.StampsStorage$Stamp,java.nio.file.Path):Z -- a:isDirtyStamp(org.jetbrains.jps.incremental.storage.StampsStorage$Stamp,java.nio.file.Path,java.nio.file.attribute.BasicFileAttributes):Z - a:removeStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget):V -- a:saveStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget,org.jetbrains.jps.incremental.storage.StampsStorage$Stamp):V -org.jetbrains.jps.incremental.storage.StampsStorage$Stamp +- a:updateStamp(java.nio.file.Path,org.jetbrains.jps.builders.BuildTarget,J):V org.jetbrains.jps.incremental.storage.StorageOwner - a:clean():V - a:close():V diff --git a/jps/jps-builders/intellij.platform.jps.build.tests.iml b/jps/jps-builders/intellij.platform.jps.build.tests.iml index ea678fbc894c..a74c01e25c2c 100644 --- a/jps/jps-builders/intellij.platform.jps.build.tests.iml +++ b/jps/jps-builders/intellij.platform.jps.build.tests.iml @@ -69,5 +69,6 @@ + \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/index/CompilerReferenceIndex.java b/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/index/CompilerReferenceIndex.java index 163f99ea7747..e4df27f58b7c 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/index/CompilerReferenceIndex.java +++ b/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/index/CompilerReferenceIndex.java @@ -4,8 +4,8 @@ package org.jetbrains.jps.backwardRefs.index; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.LowMemoryWatcher; import com.intellij.openapi.util.SystemInfo; -import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.io.FileUtilRt; +import com.intellij.openapi.util.io.NioFiles; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.CommonProcessors; import com.intellij.util.ExceptionUtil; @@ -52,7 +52,7 @@ public class CompilerReferenceIndex { private final NameEnumerator myNameEnumerator; private final PersistentStringEnumerator myFilePathEnumerator; private final File myBuildDir; - private final File myIndicesDir; + private final Path myIndicesDir; private final LowMemoryWatcher myLowMemoryWatcher = LowMemoryWatcher.register(() -> force()); private volatile Throwable myRebuildRequestCause; @@ -71,15 +71,20 @@ public class CompilerReferenceIndex { @Nullable PathRelativizerService relativizer, boolean readOnly, int version, boolean isCaseSensitive) { myBuildDir = buildDir; - myIndicesDir = getIndexDir(buildDir); - if (!myIndicesDir.exists() && !myIndicesDir.mkdirs()) { - throw new RuntimeException("Can't create dir: " + buildDir.getAbsolutePath()); + myIndicesDir = getIndexDir(buildDir.toPath()); + + try { + Files.createDirectories(myIndicesDir); } + catch (IOException e) { + throw new RuntimeException(e); + } + try { if (versionDiffers(buildDir, version)) { saveVersion(buildDir, version); } - myFilePathEnumerator = new PersistentStringEnumerator(new File(myIndicesDir, FILE_ENUM_TAB).toPath()) { + myFilePathEnumerator = new PersistentStringEnumerator(myIndicesDir.resolve(FILE_ENUM_TAB)) { @Override public int enumerate(String path) throws IOException { @@ -109,7 +114,7 @@ public class CompilerReferenceIndex { myIndices.put(indexExtension.getName(), createCompilerIndex(indexExtension, readOnly)); } - myNameEnumerator = new NameEnumerator(new File(myIndicesDir, NAME_ENUM_TAB)); + myNameEnumerator = new NameEnumerator(myIndicesDir.resolve(NAME_ENUM_TAB).toFile()); } catch (IOException e) { //IJPL-2855: must close all storages opened @@ -209,51 +214,15 @@ public class CompilerReferenceIndex { removeIndexFiles(buildDir, null); } - public static void removeIndexFiles(File buildDir, Throwable cause) { - final File indexDir = getIndexDir(buildDir); - if (indexDir.exists()) { - try { - FileUtilRt.deleteRecursively(indexDir.toPath()); - LOG.info("backward reference index deleted", cause != null ? cause : new Exception()); - } - catch (Throwable e) { - LOG.info("failed to delete backward reference index", e); - } - } - } - - private static File getIndexDir(@NotNull File buildDir) { - return new File(buildDir, "backward-refs"); - } - - public static boolean exists(@NotNull File buildDir) { - return getIndexDir(buildDir).exists(); - } - - public static boolean versionDiffers(@NotNull File buildDir, int expectedVersion) { - Path versionFile = getIndexDir(buildDir).toPath().resolve(VERSION_FILE); - try (DataInputStream is = new DataInputStream(Files.newInputStream(versionFile))) { - int currentIndexVersion = is.readInt(); - boolean isDiffer = currentIndexVersion != expectedVersion; - if (isDiffer) { - LOG.info("backward reference index version differ, expected = " + expectedVersion + ", current = " + currentIndexVersion); - } - return isDiffer; - } - catch (NoSuchFileException ignore) { - LOG.info("backward reference index version doesn't exist"); + public void saveVersion(@NotNull File buildDir, int version) { + Path versionFile = getIndexDir(buildDir.toPath()).resolve(VERSION_FILE); + try { + NioFiles.createParentDirectories(versionFile); } catch (IOException e) { - LOG.info("backward reference index version differ due to: " + e.getClass()); + throw new UncheckedIOException(e); } - return true; - } - - public void saveVersion(@NotNull File buildDir, int version) { - File versionFile = new File(getIndexDir(buildDir), VERSION_FILE); - - FileUtil.createIfDoesntExist(versionFile); - try (DataOutputStream os = new DataOutputStream(new FileOutputStream(versionFile))) { + try (DataOutputStream os = new DataOutputStream(Files.newOutputStream(versionFile))) { os.writeInt(version); } catch (IOException ex) { @@ -262,12 +231,67 @@ public class CompilerReferenceIndex { } } + public File getIndicesDir() { + return myIndicesDir.toFile(); + } + + private @NotNull CompilerMapReduceIndex createCompilerIndex(@NotNull IndexExtension indexExtension, + boolean readOnly) throws IOException { + IndexStorage indexStorage = createIndexStorage( + indexExtension.getKeyDescriptor(), + indexExtension.getValueExternalizer(), + indexExtension.getName(), + myIndicesDir.toFile(), + readOnly + ); + try { + if (readOnly) { + //noinspection unchecked,rawtypes + return new CompilerMapReduceIndex(indexExtension, indexStorage, /* forwardIndex: */ null, /* forwardIndexAccessor: */ null); + } + else { + Path storagePath = myIndicesDir.resolve(indexExtension.getName().getName() + ".inputs"); + ForwardIndex forwardIndex = new PersistentMapBasedForwardIndex(storagePath, /* readOnly: */ false); + try { + ForwardIndexAccessor forwardIndexAccessor = new KeyCollectionForwardIndexAccessor<>(indexExtension); + //noinspection unchecked,rawtypes + return new CompilerMapReduceIndex(indexExtension, indexStorage, forwardIndex, forwardIndexAccessor); + } + catch (Throwable t) {//IJPL-2855: must close all storages opened + forwardIndex.close(); + throw t; + } + } + } + catch (Throwable t) {//IJPL-2855: must close all storages opened + indexStorage.close(); + throw t; + } + } + + public static void removeIndexFiles(File buildDir, Throwable cause) { + Path indexDir = getIndexDir(buildDir.toPath()); + if (Files.exists(indexDir)) { + try { + FileUtilRt.deleteRecursively(indexDir); + LOG.info("backward reference index deleted", cause != null ? cause : new Exception()); + } + catch (Throwable e) { + LOG.info("failed to delete backward reference index", e); + } + } + } + + private static @NotNull Path getIndexDir(@NotNull Path buildDir) { + return buildDir.resolve("backward-refs"); + } + public Throwable getRebuildRequestCause() { return myRebuildRequestCause; } - public File getIndicesDir() { - return myIndicesDir; + public static boolean exists(@NotNull File buildDir) { + return Files.exists(getIndexDir(buildDir.toPath())); } public void setRebuildRequestCause(Throwable e) { @@ -321,38 +345,23 @@ public class CompilerReferenceIndex { } } - private @NotNull CompilerMapReduceIndex createCompilerIndex(@NotNull IndexExtension indexExtension, - boolean readOnly) throws IOException { - IndexStorage indexStorage = createIndexStorage( - indexExtension.getKeyDescriptor(), - indexExtension.getValueExternalizer(), - indexExtension.getName(), - myIndicesDir, - readOnly - ); - try { - if (readOnly) { - //noinspection unchecked,rawtypes - return new CompilerMapReduceIndex(indexExtension, indexStorage, /* forwardIndex: */ null, /* forwardIndexAccessor: */ null); - } - else { - Path storagePath = new File(myIndicesDir, indexExtension.getName().getName() + ".inputs").toPath(); - ForwardIndex forwardIndex = new PersistentMapBasedForwardIndex(storagePath, /* readOnly: */ false); - try { - ForwardIndexAccessor forwardIndexAccessor = new KeyCollectionForwardIndexAccessor<>(indexExtension); - //noinspection unchecked,rawtypes - return new CompilerMapReduceIndex(indexExtension, indexStorage, forwardIndex, forwardIndexAccessor); - } - catch (Throwable t) {//IJPL-2855: must close all storages opened - forwardIndex.close(); - throw t; - } + public static boolean versionDiffers(@NotNull File buildDir, int expectedVersion) { + Path versionFile = getIndexDir(buildDir.toPath()).resolve(VERSION_FILE); + try (DataInputStream is = new DataInputStream(Files.newInputStream(versionFile))) { + int currentIndexVersion = is.readInt(); + boolean isDiffer = currentIndexVersion != expectedVersion; + if (isDiffer) { + LOG.info("backward reference index version differ, expected = " + expectedVersion + ", current = " + currentIndexVersion); } + return isDiffer; } - catch (Throwable t) {//IJPL-2855: must close all storages opened - indexStorage.close(); - throw t; + catch (NoSuchFileException ignore) { + LOG.info("backward reference index version doesn't exist"); } + catch (IOException e) { + LOG.info("backward reference index version differ due to: " + e.getClass()); + } + return true; } private static IndexStorage createIndexStorage(@NotNull KeyDescriptor keyDescriptor, diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java index 2948e69a6e99..5e8423dce4dd 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/Mappings.java @@ -116,7 +116,7 @@ public class Mappings { createImplementation(); } - public Mappings(final File rootDir, PathRelativizerService relativizer) throws IOException { + public Mappings(final File rootDir, @NotNull PathRelativizerService relativizer) throws IOException { myLock = new Object(); myIsDelta = false; myChangedClasses = null; diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java index 6b1764b9b409..99babe6f0505 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java @@ -72,16 +72,10 @@ public final class BuildRunner { return load(msgHandler, dataStorageRoot.toPath(), fsState); } - private static @NotNull ProjectStamps initProjectStampStorage(@NotNull Path dataStorageRoot, - @NotNull PathRelativizerService relativizer, - @NotNull BuildTargetsState targetsState, - @Nullable StorageManager storageManager) - throws IOException { + private static @Nullable ProjectStamps initProjectStampStorage(@NotNull Path dataStorageRoot, + @NotNull BuildTargetsState targetsState) throws IOException { if (ProjectStamps.PORTABLE_CACHES) { - // allow compaction on close (not more than 10 seconds) to ensure minimal storage size - assert storageManager != null; - HashStampStorage stampStorage = new HashStampStorage(storageManager, relativizer, targetsState); - return new ProjectStamps(stampStorage); + return null; } else { return new ProjectStamps(dataStorageRoot, targetsState); @@ -100,14 +94,14 @@ public final class BuildRunner { PathRelativizerService relativizer = new PathRelativizerService(jpsModel.getProject(), isCompilerReferenceFSCaseSensitive()); - ProjectStamps projectStamps = null; + ProjectStamps fileStampService = null; BuildDataManager dataManager = null; StorageManager storageManager = null; try { storageManager = createStorageManager(dataStorageRoot); - projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager); + fileStampService = initProjectStampStorage(dataStorageRoot, targetsState); - dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager); + dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager); if (dataManager.versionDiffers()) { myForceCleanCaches = true; msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO, @@ -121,26 +115,28 @@ public final class BuildRunner { if (storageManager != null) { storageManager.forceClose(); } - if (projectStamps != null) { - projectStamps.close(); - } + if (dataManager != null) { dataManager.close(); } + else if (fileStampService != null) { + fileStampService.close(); + } + myForceCleanCaches = true; FileUtilRt.deleteRecursively(dataStorageRoot); storageManager = createStorageManager(dataStorageRoot); targetsState = new BuildTargetsState(dataPaths, jpsModel, buildRootIndex); - projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager); - dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager); + fileStampService = initProjectStampStorage(dataStorageRoot, targetsState); + dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, fileStampService, storageManager); // the second attempt succeeded msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO, JpsBuildBundle.message("build.message.project.rebuild.forced.0", e.getMessage()))); } return new ProjectDescriptor( - jpsModel, fsState, projectStamps, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex + jpsModel, fsState, dataManager, BuildLoggingManager.DEFAULT, index, targetIndex, buildRootIndex, ignoredFileIndex ); } @@ -206,8 +202,11 @@ public final class BuildRunner { return createCompilationScope(pd, scopes, myFilePaths, forceClean, false); } - private static CompileScope createCompilationScope(ProjectDescriptor pd, List scopes, Collection paths, - final boolean forceClean, final boolean includeDependenciesToScope) throws Exception { + private static CompileScope createCompilationScope(@NotNull ProjectDescriptor projectDescriptor, + @NotNull List scopes, + @NotNull Collection paths, + boolean forceClean, + boolean includeDependenciesToScope) throws Exception { Set> targetTypes = new HashSet<>(); Set> targetTypesToForceBuild = new HashSet<>(); Set> targets = new HashSet<>(); @@ -227,7 +226,7 @@ public final class BuildRunner { targetTypes.add(targetType); } else { - BuildTargetLoader loader = targetType.createLoader(pd.getModel()); + BuildTargetLoader loader = targetType.createLoader(projectDescriptor.getModel()); for (String targetId : scope.getTargetIdList()) { BuildTarget target = loader.createTarget(targetId); if (target != null) { @@ -240,11 +239,13 @@ public final class BuildRunner { } } if (includeDependenciesToScope) { - includeDependenciesToScope(targetTypes, targets, targetTypesToForceBuild, pd); + includeDependenciesToScope(targetTypes, targets, targetTypesToForceBuild, projectDescriptor); } - final StampsStorage stampsStorage = pd.getProjectStamps().getStampStorage(); - if (!paths.isEmpty()) { + if (paths.isEmpty()) { + files = Collections.emptyMap(); + } + else { boolean forceBuildAllModuleBasedTargets = false; for (BuildTargetType type : targetTypesToForceBuild) { if (type instanceof JavaModuleBuildTargetType) { @@ -255,7 +256,7 @@ public final class BuildRunner { files = new HashMap<>(); for (String path : paths) { final File file = new File(path); - final Collection descriptors = pd.getBuildRootIndex().findAllParentDescriptors(file, null); + final Collection descriptors = projectDescriptor.getBuildRootIndex().findAllParentDescriptors(file, null); for (BuildRootDescriptor descriptor : descriptors) { Set fileSet = files.get(descriptor.getTarget()); if (fileSet == null) { @@ -266,16 +267,13 @@ public final class BuildRunner { if (added) { final BuildTargetType targetType = descriptor.getTarget().getTargetType(); if (targetTypesToForceBuild.contains(targetType) || (forceBuildAllModuleBasedTargets && targetType instanceof ModuleBasedBuildTargetType)) { - pd.fsState.markDirty(null, file, descriptor, stampsStorage, false); + StampsStorage stampStorage = projectDescriptor.dataManager.getFileStampStorage(descriptor.getTarget()); + projectDescriptor.fsState.markDirty(null, file, descriptor, stampStorage, false); } } } } } - else { - files = Collections.emptyMap(); - } - return new CompileScopeImpl(targetTypes, targetTypesToForceBuild, targets, files); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java index 441305cafd78..67c59e18051c 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildSession.java @@ -15,7 +15,10 @@ import com.intellij.util.concurrency.SequentialTaskExecutor; import com.intellij.util.io.DataOutputStream; import com.intellij.util.io.StorageLockContext; import io.netty.channel.Channel; -import org.jetbrains.annotations.*; +import org.jetbrains.annotations.Nls; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.api.*; import org.jetbrains.jps.builders.*; import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType; @@ -26,6 +29,7 @@ import org.jetbrains.jps.incremental.TargetTypeRegistry; import org.jetbrains.jps.incremental.Utils; import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.messages.*; +import org.jetbrains.jps.incremental.storage.BuildDataManager; import org.jetbrains.jps.incremental.storage.ProjectStamps; import org.jetbrains.jps.incremental.storage.StampsStorage; import org.jetbrains.jps.model.module.JpsModule; @@ -457,7 +461,9 @@ final class BuildSession implements Runnable, CanceledStatus { }); } - private static void applyFSEvent(ProjectDescriptor pd, @Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent event, final boolean saveEventStamp) throws IOException { + private static void applyFSEvent(@NotNull ProjectDescriptor projectDescriptor, + @Nullable CmdlineRemoteProto.Message.ControllerMessage.FSEvent event, + boolean saveEventStamp) throws IOException { if (event == null) { return; } @@ -466,8 +472,8 @@ final class BuildSession implements Runnable, CanceledStatus { LOG.debug("applyFSEvent ordinal=" + event.getOrdinal()); } - final BuildRootIndex buildRootIndex = pd.getBuildRootIndex(); - final StampsStorage stampStorage = pd.getProjectStamps().getStampStorage(); + final BuildRootIndex buildRootIndex = projectDescriptor.getBuildRootIndex(); + BuildDataManager dataManager = projectDescriptor.dataManager; for (String deleted : event.getDeletedPathsList()) { final File file = new File(deleted); Collection descriptor = buildRootIndex.findAllParentDescriptors(file, null, null); @@ -476,7 +482,8 @@ final class BuildSession implements Runnable, CanceledStatus { LOG.debug("Applying deleted path from fs event: " + file.getPath()); } for (BuildRootDescriptor rootDescriptor : descriptor) { - pd.fsState.registerDeleted(null, rootDescriptor.getTarget(), file, stampStorage); + StampsStorage stampStorage = dataManager.getFileStampStorage(rootDescriptor.getTarget()); + projectDescriptor.fsState.registerDeleted(null, rootDescriptor.getTarget(), file, stampStorage); } } else if (LOG.isDebugEnabled()) { @@ -505,12 +512,13 @@ final class BuildSession implements Runnable, CanceledStatus { LOG.debug("Applying dirty path from fs event: " + file); } for (BuildRootDescriptor descriptor : descriptors) { - StampsStorage.Stamp stamp = stampStorage.getPreviousStamp(file, descriptor.getTarget()); - if (stamp == null || stampStorage.isDirtyStamp(stamp, file)) { - pd.fsState.markDirty(null, file.toFile(), descriptor, stampStorage, saveEventStamp); + StampsStorage stampStorage = dataManager.getFileStampStorage(descriptor.getTarget()); + Object currentUpToDateStamp = stampStorage.getCurrentStampIfUpToDate(file, descriptor.getTarget(), null); + if (currentUpToDateStamp == null) { + projectDescriptor.fsState.markDirty(null, file.toFile(), descriptor, stampStorage, saveEventStamp); } else if (LOG.isDebugEnabled()) { - LOG.debug(descriptor.getTarget() + ": Path considered up-to-date: " + file + "; stamp= " + stamp); + LOG.debug(descriptor.getTarget() + ": Path considered up-to-date: " + file + "; stamp= " + currentUpToDateStamp); } } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java index bd93e3665ca6..a1c00bfa44ac 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/ProjectDescriptor.java @@ -1,7 +1,10 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.cmdline; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.TestOnly; import org.jetbrains.jps.builders.BuildRootIndex; +import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.BuildTargetIndex; import org.jetbrains.jps.builders.logging.BuildLoggingManager; import org.jetbrains.jps.incremental.CompilerEncodingConfiguration; @@ -27,8 +30,9 @@ import java.util.Set; public final class ProjectDescriptor { private final JpsProject myProject; private final JpsModel myModel; + @TestOnly + private ProjectStamps deprecatedStamps; public final BuildFSState fsState; - private final ProjectStamps myProjectStamps; public final BuildDataManager dataManager; private final BuildLoggingManager myLoggingManager; private final ModuleExcludeIndex myModuleExcludeIndex; @@ -39,18 +43,36 @@ public final class ProjectDescriptor { private final BuildTargetIndex myBuildTargetIndex; private final IgnoredFileIndex myIgnoredFileIndex; + /** + * @deprecated Use {@link ProjectDescriptor#ProjectDescriptor(JpsModel, BuildFSState, BuildDataManager, BuildLoggingManager, ModuleExcludeIndex, BuildTargetIndex, BuildRootIndex, IgnoredFileIndex)} + */ + @Deprecated(forRemoval = true) + @ApiStatus.Internal public ProjectDescriptor(JpsModel model, BuildFSState fsState, ProjectStamps projectStamps, BuildDataManager dataManager, BuildLoggingManager loggingManager, - final ModuleExcludeIndex moduleExcludeIndex, - final BuildTargetIndex buildTargetIndex, final BuildRootIndex buildRootIndex, IgnoredFileIndex ignoredFileIndex) { + ModuleExcludeIndex moduleExcludeIndex, + BuildTargetIndex buildTargetIndex, + BuildRootIndex buildRootIndex, + IgnoredFileIndex ignoredFileIndex) { + this(model, fsState, dataManager, loggingManager, moduleExcludeIndex, buildTargetIndex, buildRootIndex, ignoredFileIndex); + deprecatedStamps = projectStamps; + } + + public ProjectDescriptor(JpsModel model, + BuildFSState fsState, + BuildDataManager dataManager, + BuildLoggingManager loggingManager, + ModuleExcludeIndex moduleExcludeIndex, + BuildTargetIndex buildTargetIndex, + BuildRootIndex buildRootIndex, + IgnoredFileIndex ignoredFileIndex) { myModel = model; myIgnoredFileIndex = ignoredFileIndex; myProject = model.getProject(); this.fsState = fsState; - myProjectStamps = projectStamps; this.dataManager = dataManager; myBuildTargetIndex = buildTargetIndex; myBuildRootIndex = buildRootIndex; @@ -106,26 +128,11 @@ public final class ProjectDescriptor { shouldClose = myUseCounter == 0; } if (shouldClose) { - if (dataManager.getStorageManager() == null) { - try { - myProjectStamps.close(); - } - finally { - try { - dataManager.close(); - } - catch (IOException e) { - e.printStackTrace(System.err); - } - } + try { + dataManager.close(); } - else { - try { - dataManager.close(); - } - catch (IOException e) { - e.printStackTrace(System.err); - } + catch (IOException e) { + e.printStackTrace(System.err); } } } @@ -142,7 +149,16 @@ public final class ProjectDescriptor { return myProject; } + /** + * @deprecated Use {@link BuildDataManager#getFileStampStorage(BuildTarget)}. + */ + @Deprecated(forRemoval = true) public ProjectStamps getProjectStamps() { - return myProjectStamps; + //noinspection TestOnlyProblems + if (deprecatedStamps != null) { + return deprecatedStamps; + } + //noinspection removal + return dataManager.getFileStampService(); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java index 4a8b0fd09dff..e3abe9fd6a5e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java @@ -33,20 +33,20 @@ import java.util.*; public final class BuildOperations { private BuildOperations() { } - public static void ensureFSStateInitialized(CompileContext context, BuildTarget target, boolean readOnly) throws IOException { - final ProjectDescriptor pd = context.getProjectDescriptor(); - final StampsStorage stampsStorage = pd.getProjectStamps().getStampStorage(); - final BuildTargetConfiguration configuration = pd.getTargetsState().getTargetConfiguration(target); + public static void ensureFSStateInitialized(@NotNull CompileContext context, @NotNull BuildTarget target, boolean readOnly) throws IOException { + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + BuildTargetConfiguration configuration = projectDescriptor.getTargetsState().getTargetConfiguration(target); if (JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)) { - FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampsStorage, true, null, null); - pd.fsState.markInitialScanPerformed(target); + StampsStorage stampStorage = projectDescriptor.dataManager.getFileStampStorage(target); + FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampStorage, true, null, null); + projectDescriptor.fsState.markInitialScanPerformed(target); if (!readOnly) { configuration.save(context); } } else { boolean isTargetDirty = false; - if (context.getScope().isBuildForced(target) || (isTargetDirty = configuration.isTargetDirty(context.getProjectDescriptor())) || (!pd.getBuildRootIndex().getTargetRoots(target, context).isEmpty() && configuration.outputRootWasDeleted(context))) { + if (context.getScope().isBuildForced(target) || (isTargetDirty = configuration.isTargetDirty(context.getProjectDescriptor())) || (!projectDescriptor.getBuildRootIndex().getTargetRoots(target, context).isEmpty() && configuration.outputRootWasDeleted(context))) { if (isTargetDirty) { configuration.logDiagnostics(context); } @@ -56,34 +56,34 @@ public final class BuildOperations { // case when target build is forced, is handled separately IncProjectBuilder.clearOutputFiles(context, target); } - pd.dataManager.cleanTargetStorages(target); + projectDescriptor.dataManager.cleanTargetStorages(target); configuration.save(context); } } - else if (!pd.fsState.isInitialScanPerformed(target)) { + else if (!projectDescriptor.fsState.isInitialScanPerformed(target)) { initTargetFSState(context, target, false); } } } private static void initTargetFSState(CompileContext context, BuildTarget target, final boolean forceMarkDirty) throws IOException { - final ProjectDescriptor pd = context.getProjectDescriptor(); - final StampsStorage stampsStorage = pd.getProjectStamps().getStampStorage(); + final ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + StampsStorage stampStorage = projectDescriptor.dataManager.getFileStampStorage(target); Set currentFiles = FileCollectionFactory.createCanonicalFileSet(); - FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampsStorage, forceMarkDirty, currentFiles, null); + FSOperations.markDirtyFiles(context, target, CompilationRound.CURRENT, stampStorage, forceMarkDirty, currentFiles, null); // handle deleted paths - final BuildFSState fsState = pd.fsState; - final SourceToOutputMapping sourceToOutputMap = pd.dataManager.getSourceToOutputMap(target); + final BuildFSState fsState = projectDescriptor.fsState; + final SourceToOutputMapping sourceToOutputMap = projectDescriptor.dataManager.getSourceToOutputMap(target); for (final Iterator it = sourceToOutputMap.getSourcesIterator(); it.hasNext(); ) { final String path = it.next(); // can check if the file exists final File file = new File(path); if (!currentFiles.contains(file)) { - fsState.registerDeleted(context, target, file, stampsStorage); + fsState.registerDeleted(context, target, file, stampStorage); } } - pd.fsState.markInitialScanPerformed(target); + projectDescriptor.fsState.markInitialScanPerformed(target); } public static void markTargetsUpToDate(CompileContext context, BuildTargetChunk chunk) throws IOException { @@ -102,7 +102,8 @@ public final class BuildOperations { if (target instanceof ModuleBuildTarget) { context.clearNonIncrementalMark((ModuleBuildTarget)target); } - StampsStorage stampStorage = projectDescriptor.getProjectStamps().getStampStorage(); + + StampsStorage stampStorage = projectDescriptor.dataManager.getFileStampStorage(target); long targetBuildStartStamp = context.getCompilationStartStamp(target); for (BuildRootDescriptor buildRootDescriptor : projectDescriptor.getBuildRootIndex().getTargetRoots(target, context)) { marked |= fsState.markAllUpToDate(context, buildRootDescriptor, stampStorage, targetBuildStartStamp); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java index 946394856c99..99cf4cc36bfd 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/ChainedTargetsBuildListener.java @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.incremental; import com.intellij.openapi.util.Pair; @@ -6,6 +6,7 @@ import com.intellij.openapi.util.io.FileUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.BuildRootDescriptor; import org.jetbrains.jps.builders.BuildRootIndex; +import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.cmdline.ProjectDescriptor; import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.messages.FileDeletedEvent; @@ -26,17 +27,18 @@ final class ChainedTargetsBuildListener implements BuildListener { @Override public void filesGenerated(@NotNull FileGeneratedEvent event) { - final ProjectDescriptor pd = myContext.getProjectDescriptor(); - final BuildFSState fsState = pd.fsState; + final ProjectDescriptor projectDescriptor = myContext.getProjectDescriptor(); + final BuildFSState fsState = projectDescriptor.fsState; for (Pair pair : event.getPaths()) { final String relativePath = pair.getSecond(); final File file = relativePath.equals(".") ? new File(pair.getFirst()) : new File(pair.getFirst(), relativePath); - for (BuildRootDescriptor desc : pd.getBuildRootIndex().findAllParentDescriptors(file, myContext)) { - if (!event.getSourceTarget().equals(desc.getTarget())) { - // do not mark files belonging to the target that originated the event - // It is assumed that those files will be explicitly marked dirty by particular builder, if needed. + for (BuildRootDescriptor buildRootDescriptor : projectDescriptor.getBuildRootIndex().findAllParentDescriptors(file, myContext)) { + BuildTarget target = buildRootDescriptor.getTarget(); + if (!event.getSourceTarget().equals(target)) { + // Do not mark files belonging to the target that originated the event; + // It is assumed that those files will be explicitly marked dirty by a particular builder, if needed. try { - fsState.markDirty(myContext, file, desc, pd.getProjectStamps().getStampStorage(), false); + fsState.markDirty(myContext, file, buildRootDescriptor, projectDescriptor.dataManager.getFileStampStorage(target), false); } catch (IOException ignored) { } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java index f6ef2d66cdfd..cce8341923c0 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/FSOperations.java @@ -15,6 +15,7 @@ import org.jetbrains.jps.builders.java.JavaBuilderUtil; import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor; import org.jetbrains.jps.cmdline.ProjectDescriptor; import org.jetbrains.jps.incremental.fs.CompilationRound; +import org.jetbrains.jps.incremental.storage.BuildDataManager; import org.jetbrains.jps.incremental.storage.StampsStorage; import org.jetbrains.jps.model.java.JpsJavaClasspathKind; import org.jetbrains.jps.model.java.JpsJavaExtensionService; @@ -59,18 +60,27 @@ public final class FSOperations { } public static void markDirty(CompileContext context, final CompilationRound round, final File file) throws IOException { - final JavaSourceRootDescriptor rd = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file); - if (rd != null) { - final ProjectDescriptor pd = context.getProjectDescriptor(); - pd.fsState.markDirty(context, round, file, rd, pd.getProjectStamps().getStampStorage(), false); + JavaSourceRootDescriptor rootDescriptor = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file); + if (rootDescriptor != null) { + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + projectDescriptor.fsState.markDirty(context, + round, + file, + rootDescriptor, + projectDescriptor.dataManager.getFileStampStorage(rootDescriptor.target), + false); } } public static void markDirtyIfNotDeleted(CompileContext context, final CompilationRound round, final File file) throws IOException { - final JavaSourceRootDescriptor rd = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file); - if (rd != null) { - final ProjectDescriptor pd = context.getProjectDescriptor(); - pd.fsState.markDirtyIfNotDeleted(context, round, file, rd, pd.getProjectStamps().getStampStorage()); + JavaSourceRootDescriptor rootDescriptor = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file); + if (rootDescriptor != null) { + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + projectDescriptor.fsState.markDirtyIfNotDeleted(context, + round, + file, + rootDescriptor, + projectDescriptor.dataManager.getFileStampStorage(rootDescriptor.target)); } } @@ -101,23 +111,25 @@ public final class FSOperations { @Override public DirtyFilesHolderBuilder markDirtyFile(T target, File file) throws IOException { - final ProjectDescriptor pd = context.getProjectDescriptor(); - final R rd = pd.getBuildRootIndex().findParentDescriptor(file, Collections.singleton(target.getTargetType()), context); - if (rd != null) { - if (pd.fsState.markDirtyIfNotDeleted(context, round, file, rd, pd.getProjectStamps().getStampStorage()) || - pd.fsState.isMarkedForRecompilation(context, round, rd, file)) { - Map> targetFiles = dirtyFiles.get(target); - if (targetFiles == null) { - targetFiles = new HashMap<>(); - dirtyFiles.put(target, targetFiles); - } - Set rootFiles = targetFiles.get(rd); - if (rootFiles == null) { - rootFiles = FileCollectionFactory.createCanonicalFileSet(); - targetFiles.put(rd, rootFiles); - } - rootFiles.add(file); + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + R rootDescriptor = projectDescriptor.getBuildRootIndex().findParentDescriptor(file, List.of(target.getTargetType()), context); + if (rootDescriptor == null) { + return this; + } + + if (projectDescriptor.fsState.markDirtyIfNotDeleted(context, round, file, rootDescriptor, projectDescriptor.dataManager.getFileStampStorage(target)) || + projectDescriptor.fsState.isMarkedForRecompilation(context, round, rootDescriptor, file)) { + Map> targetFiles = dirtyFiles.get(target); + if (targetFiles == null) { + targetFiles = new HashMap<>(); + dirtyFiles.put(target, targetFiles); } + Set rootFiles = targetFiles.get(rootDescriptor); + if (rootFiles == null) { + rootFiles = FileCollectionFactory.createCanonicalFileSet(); + targetFiles.put(rootDescriptor, rootFiles); + } + rootFiles.add(file); } return this; } @@ -158,10 +170,13 @@ public final class FSOperations { } public static void markDeleted(CompileContext context, File file) throws IOException { - final JavaSourceRootDescriptor rd = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file); - if (rd != null) { - final ProjectDescriptor pd = context.getProjectDescriptor(); - pd.fsState.registerDeleted(context, rd.target, file, pd.getProjectStamps().getStampStorage()); + JavaSourceRootDescriptor rootDescriptor = context.getProjectDescriptor().getBuildRootIndex().findJavaRootDescriptor(context, file); + if (rootDescriptor != null) { + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + projectDescriptor.fsState.registerDeleted(context, + rootDescriptor.target, + file, + projectDescriptor.dataManager.getFileStampStorage(rootDescriptor.target)); } } @@ -171,9 +186,15 @@ public final class FSOperations { } } - public static void markDirty(CompileContext context, final CompilationRound round, final ModuleBuildTarget target, @Nullable FileFilter filter) throws IOException { - final ProjectDescriptor pd = context.getProjectDescriptor(); - markDirtyFiles(context, target, round, pd.getProjectStamps().getStampStorage(), true, null, filter); + public static void markDirty(CompileContext context, CompilationRound round, ModuleBuildTarget target, @Nullable FileFilter filter) throws IOException { + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + markDirtyFiles(context, + target, + round, + projectDescriptor.dataManager.getFileStampStorage(target), + true, + null, + filter); } public static void markDirtyRecursively(CompileContext context, final CompilationRound round, ModuleChunk chunk) throws IOException { @@ -224,11 +245,11 @@ public final class FSOperations { removeTargetsAlreadyMarkedDirty(context, dirtyTargets); - final StampsStorage stampsStorage = context.getProjectDescriptor().getProjectStamps().getStampStorage(); + BuildDataManager dataManager = context.getProjectDescriptor().dataManager; for (ModuleBuildTarget target : dirtyTargets) { - markDirtyFiles(context, target, round, stampsStorage, true, null, filter); + StampsStorage stampStorage = dataManager.getFileStampStorage(target); + markDirtyFiles(context, target, round, stampStorage, true, null, filter); } - } private static Set getDependentModulesRecursively(final JpsModule module, final JpsJavaClasspathKind kind) { @@ -248,7 +269,7 @@ public final class FSOperations { static void markDirtyFiles(CompileContext context, BuildTarget target, final CompilationRound round, - StampsStorage stampsStorage, + @Nullable StampsStorage stampStorage, boolean forceMarkDirty, @Nullable Set currentFiles, @Nullable FileFilter filter) throws IOException { @@ -262,7 +283,7 @@ public final class FSOperations { if (filter == null) { context.getProjectDescriptor().fsState.clearRecompile(rd); } - completelyMarkedDirty &= traverseRecursively(context, rd, round, rd.getRootFile(), stampsStorage, forceMarkDirty, currentFiles, filter); + completelyMarkedDirty &= traverseRecursively(context, rd, round, rd.getRootFile(), stampStorage, forceMarkDirty, currentFiles, filter); } if (completelyMarkedDirty) { @@ -279,7 +300,7 @@ public final class FSOperations { final BuildRootDescriptor rd, final CompilationRound round, final File file, - final @NotNull StampsStorage stampStorage, + @Nullable StampsStorage stampStorage, final boolean forceDirty, @Nullable Set currentFiles, @Nullable FileFilter filter) throws IOException { @@ -294,15 +315,12 @@ public final class FSOperations { boolean markDirty = forceDirty; if (!markDirty) { Path nioFile = file.toPath(); - StampsStorage.Stamp previousStamp = stampStorage.getPreviousStamp(nioFile, rd.getTarget()); - markDirty = previousStamp == null || (attrs == null - ? stampStorage.isDirtyStamp(previousStamp, nioFile) - : stampStorage.isDirtyStamp(previousStamp, nioFile, attrs)); + markDirty = stampStorage == null || stampStorage.getCurrentStampIfUpToDate(nioFile, rd.getTarget(), attrs) == null; } if (markDirty) { // if it is a full project rebuild, all storages are already completely cleared; // so passing null because there is no need to access the storage to clear non-existing data - final StampsStorage marker = context.isProjectRebuild()? null : stampStorage; + StampsStorage marker = context.isProjectRebuild()? null : stampStorage; context.getProjectDescriptor().fsState.markDirty(context, round, file, rd, marker, false); } if (currentFiles != null) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/FileHashUtil.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/FileHashUtil.java index 8a2b892f2376..402c734997e9 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/FileHashUtil.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/FileHashUtil.java @@ -6,6 +6,7 @@ import com.dynatrace.hash4j.hashing.Hashing; import com.intellij.openapi.util.SystemInfoRt; import com.intellij.openapi.util.text.StringUtilRt; import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.incremental.storage.ProjectStamps; @@ -17,12 +18,11 @@ import java.nio.file.StandardOpenOption; @ApiStatus.Internal public final class FileHashUtil { - /** * @param path a normalized system-independent path, possibly relative, but without "." and ".." relative references * @param hash hash sink to be updated */ - public static void computePathHashCode(@Nullable String path, HashSink hash) { + public static void computePathHashCode(@Nullable String path, @NotNull HashSink hash) { int length = path == null? 0 : path.length(); if (length == 0) { hash.putInt(0); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index f3bfc5359598..60a347c4858d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -1,6 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.incremental; +import com.intellij.concurrency.ContextAwareRunnable; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.*; import com.intellij.openapi.util.io.FileUtil; @@ -210,16 +211,11 @@ public final class IncProjectBuilder { checkRebuildRequired(scope); rebuildRequiredSpan.complete(); + BuildDataManager dataManager = myProjectDescriptor.dataManager; final LowMemoryWatcher memWatcher = LowMemoryWatcher.register(() -> { JavacMain.clearCompilerZipFileCache(); - StorageManager storageManager = myProjectDescriptor.dataManager.getStorageManager(); - if (storageManager == null) { - ((StorageOwner)myProjectDescriptor.getProjectStamps().getStampStorage()).flush(false); - } - myProjectDescriptor.dataManager.flush(false); - if (storageManager != null) { - storageManager.clearCache(); - } + dataManager.flush(false); + dataManager.clearCache(); }); final CleanupTempDirectoryExtension cleaner = CleanupTempDirectoryExtension.getInstance(); @@ -240,8 +236,8 @@ public final class IncProjectBuilder { Tracer.Span buildSpan = Tracer.start("IncProjectBuilder.runBuild"); runBuild(context, forceCleanCaches); buildSpan.complete(); - myProjectDescriptor.dataManager.saveVersion(); - myProjectDescriptor.dataManager.reportUnhandledRelativizerPaths(); + dataManager.saveVersion(); + dataManager.reportUnhandledRelativizerPaths(); sourcesState.reportSourcesState(); reportRebuiltModules(context); reportUnprocessedChanges(context); @@ -305,10 +301,11 @@ public final class IncProjectBuilder { } private void checkRebuildRequired(final CompileScope scope) throws RebuildRequestedException { + boolean isDebugEnabled = LOG.isDebugEnabled(); if (myIsTestMode || isAutoBuild()) { // do not use the heuristic in tests in order to properly test all cases // automatic builds should not cause start full project rebuilds to avoid situations when rebuild is not expected by user - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Rebuild heuristic: skipping the check; isTestMode = " + myIsTestMode + "; isAutoBuild = " + isAutoBuild()); } return; @@ -316,7 +313,7 @@ public final class IncProjectBuilder { final BuildTargetsState targetsState = myProjectDescriptor.getTargetsState(); final long timeThreshold = targetsState.getLastSuccessfulRebuildDuration() * 95 / 100; // 95% of last registered clean rebuild time if (timeThreshold <= 0) { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Rebuild heuristic: no stats available"); } return; @@ -325,13 +322,13 @@ public final class IncProjectBuilder { // checking only JavaModuleBuildTargetType because these target types directly correspond to project modules for (BuildTargetType type : JavaModuleBuildTargetType.ALL_TYPES) { if (!scope.isBuildIncrementally(type)) { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Rebuild heuristic: skipping the check because rebuild is forced for targets of type " + type.getTypeId()); } return; } if (!scope.isAllTargetsOfTypeAffected(type)) { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Rebuild heuristic: skipping the check because some targets are excluded from compilation scope, e.g. targets of type " + type.getTypeId()); } return; @@ -346,7 +343,7 @@ public final class IncProjectBuilder { return allTargetsAffected.contains(target.getTargetType()) || scope.isAffected(target); } }); - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Rebuild heuristic: estimated build time / timeThreshold : " + estimatedWorkTime + " / " + timeThreshold); } @@ -436,11 +433,7 @@ public final class IncProjectBuilder { private static void flushContext(CompileContext context) { if (context != null) { - ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); - if (projectDescriptor.dataManager.getStorageManager() == null) { - ((StorageOwner)projectDescriptor.getProjectStamps().getStampStorage()).flush(false); - } - projectDescriptor.dataManager.flush(false); + context.getProjectDescriptor().dataManager.flush(false); } ExternalJavacManager server = ExternalJavacManager.KEY.get(context); @@ -558,7 +551,6 @@ public final class IncProjectBuilder { } context.processMessage(new ProgressMessage(JpsBuildBundle.message("progress.message.finished.saving.caches"))); } - } private void sendElapsedTimeMessages(CompileContext context) { @@ -653,35 +645,20 @@ public final class IncProjectBuilder { LOG.info("Cleaned output directories in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - cleanStart) + " ms"); if (cleanCaches) { try { - if (projectDescriptor.dataManager.getStorageManager() == null) { - ((StorageOwner)projectDescriptor.getProjectStamps().getStampStorage()).clean(); - } + projectDescriptor.dataManager.clean(myAsyncTasks::add); } catch (IOException e) { if (ex == null) { - ex = new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.timestamps.storage"), e); + ex = new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.compiler.storages"), e); } else { - LOG.info("Error cleaning timestamps storage", e); + LOG.info("Error cleaning compiler storages", e); } } finally { - try { - projectDescriptor.dataManager.clean(myAsyncTasks::add); - } - catch (IOException e) { - if (ex == null) { - ex = new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.compiler.storages"), e); - } - else { - LOG.info("Error cleaning compiler storages", e); - } - } - finally { - projectDescriptor.fsState.clearAll(); - if (ex != null) { - throw ex; - } + projectDescriptor.fsState.clearAll(); + if (ex != null) { + throw ex; } } } @@ -955,15 +932,14 @@ public final class IncProjectBuilder { } } - private static void runTasks(CompileContext context, final List tasks) throws ProjectBuildException { + private static void runTasks(@NotNull CompileContext context, @NotNull List tasks) throws ProjectBuildException { for (BuildTask task : tasks) { task.build(context); } } - private void buildChunks(final CompileContextImpl context, BuildProgress buildProgress) throws ProjectBuildException { + private void buildChunks(@NotNull CompileContextImpl context, @NotNull BuildProgress buildProgress) throws ProjectBuildException { try { - boolean compileInParallel = isParallelBuild(); if (compileInParallel && MAX_BUILDER_THREADS <= 1) { LOG.info("Switched off parallel compilation because maximum number of builder threads is less than 2. Set '" @@ -977,14 +953,15 @@ public final class IncProjectBuilder { } else { // non-parallel build - final ProjectDescriptor pd = context.getProjectDescriptor(); - final Runnable flushCommand = Utils.asCountedRunnable(FLUSH_INVOCATIONS_TO_SKIP, () -> pd.dataManager.flush(true)); - for (BuildTargetChunk chunk : pd.getBuildTargetIndex().getSortedTargetChunks(context)) { + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + BuildDataManager dataManager = projectDescriptor.dataManager; + final Runnable flushCommand = Utils.asCountedRunnable(FLUSH_INVOCATIONS_TO_SKIP, () -> dataManager.flush(true)); + for (BuildTargetChunk chunk : projectDescriptor.getBuildTargetIndex().getSortedTargetChunks(context)) { try { buildChunkIfAffected(context, context.getScope(), chunk, buildProgress); } finally { - pd.dataManager.closeSourceToOutputStorages(Collections.singleton(chunk)); + dataManager.closeSourceToOutputStorages(chunk); flushCommand.run(); } } @@ -1028,12 +1005,14 @@ public final class IncProjectBuilder { } } - public List markAsFinishedAndGetNextReadyTasks() { - List nextTasks = new SmartList<>(); + @Nullable List getNextReadyTasks() { + List nextTasks = null; for (BuildChunkTask task : myTasksDependsOnThis) { int dependenciesCount = task.myNotBuildDependenciesCount.decrementAndGet(); - if (dependenciesCount == 0) { + if (nextTasks == null) { + nextTasks = new SmartList<>(); + } nextTasks.add(task); } } @@ -1042,16 +1021,10 @@ public final class IncProjectBuilder { } private final class BuildParallelizer { - private final ExecutorService myParallelBuildExecutor = AppExecutorUtil.createCustomPriorityQueueBoundedApplicationPoolExecutor( - "IncProjectBuilder Executor Pool", SharedThreadPool.getInstance(), MAX_BUILDER_THREADS, (o1, o2) -> { - int p1 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o1).priority : 1; - int p2 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o2).priority : 1; - return Integer.compare(p2, p1); - }); private final CompileContext myContext; private final BuildProgress myBuildProgress; private final AtomicReference myException = new AtomicReference<>(); - private final CountDownLatch myTasksCountDown; + private final CountDownLatch taskCountDown; private final List myTasks; private final Runnable myFlushCommand; @@ -1091,7 +1064,6 @@ public final class IncProjectBuilder { } collectTaskDependantsSpan.complete(); - Tracer.Span prioritisationSpan = Tracer.start("IncProjectBuilder.prioritisation"); // bitset stores indexes of transitively dependant tasks HashMap chunkToTransitive = new HashMap<>(); @@ -1112,7 +1084,7 @@ public final class IncProjectBuilder { } prioritisationSpan.complete(); - myTasksCountDown = new CountDownLatch(myTasks.size()); + taskCountDown = new CountDownLatch(myTasks.size()); span.complete(); } @@ -1123,10 +1095,21 @@ public final class IncProjectBuilder { initialTasks.add(task); } } - queueTasks(initialTasks); + @NotNull ExecutorService parallelBuildExecutor = + AppExecutorUtil.createCustomPriorityQueueBoundedApplicationPoolExecutor( + "IncProjectBuilder Executor Pool", + SharedThreadPool.getInstance(), + MAX_BUILDER_THREADS, + (o1, o2) -> { + int p1 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o1).priority : 1; + int p2 = o1 instanceof RunnableWithPriority ? ((RunnableWithPriority)o2).priority : 1; + return Integer.compare(p2, p1); + }); + + queueTasks(initialTasks, LOG.isDebugEnabled(), parallelBuildExecutor); try { - myTasksCountDown.await(); + taskCountDown.await(); } catch (InterruptedException e) { LOG.info(e); @@ -1141,29 +1124,29 @@ public final class IncProjectBuilder { } } - private void queueTasks(List tasks) { - if (tasks.isEmpty()) return; - ArrayList sorted = new ArrayList<>(tasks); - sorted.sort(Comparator.comparingLong(BuildChunkTask::getScore).reversed()); + private void queueTasks(List tasks, boolean isDebugLogEnabled, @NotNull ExecutorService parallelBuildExecutor) { + BuildChunkTask[] sorted = tasks.toArray(new BuildChunkTask[0]); + Arrays.sort(sorted, Comparator.comparingLong(BuildChunkTask::getScore).reversed()); - if (LOG.isDebugEnabled()) { - final List chunksToLog = new ArrayList<>(); + if (isDebugLogEnabled) { + List chunksToLog = new ArrayList<>(sorted.length); for (BuildChunkTask task : sorted) { chunksToLog.add(task.getChunk()); } - final StringBuilder logBuilder = new StringBuilder("Queuing " + chunksToLog.size() + " chunks in parallel: "); + StringBuilder logBuilder = new StringBuilder().append("Queuing ").append(chunksToLog.size()).append(" chunks in parallel: "); chunksToLog.sort(Comparator.comparing(BuildTargetChunk::toString)); for (BuildTargetChunk chunk : chunksToLog) { logBuilder.append(chunk.toString()).append("; "); } LOG.debug(logBuilder.toString()); } + for (BuildChunkTask task : sorted) { - queueTask(task); + queueTask(task, isDebugLogEnabled, parallelBuildExecutor); } } - private abstract class RunnableWithPriority implements Runnable { + private abstract class RunnableWithPriority implements Runnable, ContextAwareRunnable { public final int priority; RunnableWithPriority(int priority) { @@ -1171,9 +1154,9 @@ public final class IncProjectBuilder { } } - private void queueTask(final BuildChunkTask task) { - final CompileContext chunkLocalContext = createContextWrapper(myContext); - myParallelBuildExecutor.execute(new RunnableWithPriority(task.getScore()) { + private void queueTask(@NotNull BuildChunkTask task, boolean isDebugLogEnabled, @NotNull ExecutorService parallelBuildExecutor) { + CompileContext chunkLocalContext = createContextWrapper(myContext); + parallelBuildExecutor.execute(new RunnableWithPriority(task.getScore()) { @Override public void run() { try { @@ -1183,10 +1166,8 @@ public final class IncProjectBuilder { } } finally { - Tracer.Span flush = Tracer.start("flushing"); - myProjectDescriptor.dataManager.closeSourceToOutputStorages(Collections.singletonList(task.getChunk())); + myProjectDescriptor.dataManager.closeSourceToOutputStorages(task.getChunk()); myFlushCommand.run(); - flush.complete(); } } catch (Throwable e) { @@ -1194,12 +1175,18 @@ public final class IncProjectBuilder { LOG.info(e); } finally { - LOG.debug("Finished compilation of " + task.getChunk().toString()); - myTasksCountDown.countDown(); - List nextTasks; - nextTasks = task.markAsFinishedAndGetNextReadyTasks(); - if (!nextTasks.isEmpty()) { - queueTasks(nextTasks); + try { + if (isDebugLogEnabled) { + LOG.debug("Finished compilation of " + task.getChunk().toString()); + } + + List nextTasks = task.getNextReadyTasks(); + if (nextTasks != null && !nextTasks.isEmpty()) { + queueTasks(nextTasks, isDebugLogEnabled, parallelBuildExecutor); + } + } + finally { + taskCountDown.countDown(); } } } @@ -1489,7 +1476,8 @@ public final class IncProjectBuilder { } final int buildTargetId = context.getProjectDescriptor().getTargetsState().getBuildTargetId(target); final boolean shouldPruneEmptyDirs = target instanceof ModuleBasedTarget; - final SourceToOutputMapping sourceToOutputStorage = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target); + BuildDataManager dataManager = context.getProjectDescriptor().dataManager; + final SourceToOutputMapping sourceToOutputStorage = dataManager.getSourceToOutputMap(target); final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); // actually delete outputs associated with removed paths final Collection pathsForIteration; @@ -1506,7 +1494,7 @@ public final class IncProjectBuilder { final Collection outputs = sourceToOutputStorage.getOutputs(deletedSource); if (outputs != null && !outputs.isEmpty()) { List deletedOutputPaths = new ArrayList<>(); - final OutputToTargetRegistry outputToSourceRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetRegistry(); + final OutputToTargetRegistry outputToSourceRegistry = dataManager.getOutputToTargetRegistry(); for (String output : outputToSourceRegistry.getSafeToDeleteOutputs(outputs, buildTargetId)) { final boolean deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, shouldPruneEmptyDirs ? dirsToDelete : null); if (deleted) { @@ -1526,8 +1514,8 @@ public final class IncProjectBuilder { if (target instanceof ModuleBuildTarget) { // check if the deleted source was associated with a form - final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); - final Collection boundForms = sourceToFormMap.getOutputs(deletedSource); + OneToManyPathMapping sourceToFormMap = dataManager.getSourceToFormMap(target); + Collection boundForms = sourceToFormMap.getOutputs(deletedSource); if (boundForms != null) { for (String formPath : boundForms) { final File formFile = new File(formPath); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java index 2115af051034..f3161dbde8d0 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/fs/BuildFSState.java @@ -167,7 +167,7 @@ public final class BuildFSState { public void registerDeleted(@Nullable CompileContext context, BuildTarget target, File file, - @Nullable StampsStorage stampStorage) throws IOException { + @Nullable StampsStorage stampStorage) throws IOException { registerDeleted(context, target, file); if (stampStorage != null) { stampStorage.removeStamp(file.toPath(), target); @@ -239,7 +239,7 @@ public final class BuildFSState { public boolean markDirty(@Nullable CompileContext context, File file, final BuildRootDescriptor rd, - @Nullable StampsStorage stampStorage, + @Nullable StampsStorage stampStorage, boolean saveEventStamp) throws IOException { return markDirty(context, CompilationRound.NEXT, file, rd, stampStorage, saveEventStamp); } @@ -247,21 +247,21 @@ public final class BuildFSState { public boolean markDirty(@Nullable CompileContext context, CompilationRound round, File file, - final BuildRootDescriptor rd, - @Nullable StampsStorage stampStorage, + @NotNull BuildRootDescriptor buildRootDescriptor, + @Nullable StampsStorage stampStorage, boolean saveEventStamp) throws IOException { final FilesDelta roundDelta = getRoundDelta(round == CompilationRound.NEXT? NEXT_ROUND_DELTA_KEY : CURRENT_ROUND_DELTA_KEY, context); - if (roundDelta != null && isInCurrentContextTargets(context, rd)) { - roundDelta.markRecompile(rd, file); + if (roundDelta != null && isInCurrentContextTargets(context, buildRootDescriptor)) { + roundDelta.markRecompile(buildRootDescriptor, file); } - final FilesDelta filesDelta = getDelta(rd.getTarget()); + final FilesDelta filesDelta = getDelta(buildRootDescriptor.getTarget()); filesDelta.lockData(); try { - final boolean marked = filesDelta.markRecompile(rd, file); + final boolean marked = filesDelta.markRecompile(buildRootDescriptor, file); if (marked) { if (LOG.isDebugEnabled()) { - LOG.debug(rd.getTarget() + ": MARKED DIRTY: " + file.getPath()); + LOG.debug(buildRootDescriptor.getTarget() + ": MARKED DIRTY: " + file.getPath()); } if (saveEventStamp) { final long eventStamp = System.currentTimeMillis(); @@ -270,12 +270,12 @@ public final class BuildFSState { } } if (stampStorage != null) { - stampStorage.removeStamp(file.toPath(), rd.getTarget()); + stampStorage.removeStamp(file.toPath(), buildRootDescriptor.getTarget()); } } else { if (LOG.isDebugEnabled()) { - LOG.debug(rd.getTarget() + ": NOT MARKED DIRTY: " + file.getPath()); + LOG.debug(buildRootDescriptor.getTarget() + ": NOT MARKED DIRTY: " + file.getPath()); } } return marked; @@ -297,7 +297,7 @@ public final class BuildFSState { CompilationRound round, File file, final BuildRootDescriptor rd, - @Nullable StampsStorage stampStorage) throws IOException { + @Nullable StampsStorage stampStorage) throws IOException { final boolean marked = getDelta(rd.getTarget()).markRecompileIfNotDeleted(rd, file); if (marked && stampStorage != null) { stampStorage.removeStamp(file.toPath(), rd.getTarget()); @@ -384,7 +384,7 @@ public final class BuildFSState { */ public boolean markAllUpToDate(@NotNull CompileContext context, @NotNull BuildRootDescriptor buildRootDescriptor, - @NotNull StampsStorage stampsStorage, + @NotNull StampsStorage stampStorage, long targetBuildStartStamp) throws IOException { boolean marked = false; final BuildTarget target = buildRootDescriptor.getTarget(); @@ -402,7 +402,6 @@ public final class BuildFSState { if (scope.isAffected(target, file)) { Path nioFile = file.toPath(); long currentFileTimestamp = FSOperations.lastModified(nioFile); - StampsStorage.Stamp stamp = stampsStorage.getCurrentStamp(nioFile, currentFileTimestamp); if (!buildRootDescriptor.isGenerated() && (currentFileTimestamp > targetBuildStartStamp || getEventRegistrationStamp(file) > targetBuildStartStamp)) { // if the file was modified after the compilation had started, // do not save the stamp considering a file dirty @@ -416,7 +415,7 @@ public final class BuildFSState { } else { marked = true; - stampsStorage.saveStamp(nioFile, target, stamp); // todo: ask jeka + stampStorage.updateStamp(nioFile, target, currentFileTimestamp); } } else { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index 6454a6c74a51..712b6af2b091 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -5,12 +5,15 @@ import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.io.FileUtilRt; import com.intellij.openapi.util.io.NioFiles; +import com.intellij.tracing.Tracer; import com.intellij.util.io.PersistentHashMapValueStorage; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; import org.jetbrains.jps.builders.BuildTarget; import org.jetbrains.jps.builders.BuildTargetType; +import org.jetbrains.jps.builders.JpsBuildBundle; import org.jetbrains.jps.builders.impl.BuildTargetChunk; import org.jetbrains.jps.builders.impl.storage.BuildTargetStorages; import org.jetbrains.jps.builders.java.JavaBuilderUtil; @@ -24,6 +27,7 @@ import org.jetbrains.jps.dependency.impl.Containers; import org.jetbrains.jps.dependency.impl.DependencyGraphImpl; import org.jetbrains.jps.dependency.impl.LoggingDependencyGraph; import org.jetbrains.jps.dependency.impl.PathSourceMapper; +import org.jetbrains.jps.incremental.ProjectBuildException; import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; import java.io.*; @@ -31,6 +35,7 @@ import java.nio.file.Path; import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Future; @@ -53,8 +58,15 @@ public final class BuildDataManager { private static final String OUT_TARGET_STORAGE = "out-target"; private static final String MAPPINGS_STORAGE = "mappings"; private static final String SRC_TO_OUTPUT_FILE_NAME = "data"; - private final ConcurrentMap, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>(); - private final OneToManyPathMapping mySrcToFormMap; + + private final @NotNull ConcurrentMap, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>(); + private final @NotNull ConcurrentMap, SourceToOutputMappingWrapper> buildTargetToSourceToOutputMapping = new ConcurrentHashMap<>(); + // only for new experimental storage + private final @NotNull ConcurrentMap, PerTargetMapManager> targetToMapManager = new ConcurrentHashMap<>(); + + private final @Nullable ProjectStamps fileStampService; + + private final @Nullable OneToManyPathsMapping sourceToFormMap; private final Mappings myMappings; private final Object myGraphManagementLock = new Object(); private DependencyGraph myDepGraph; @@ -67,29 +79,33 @@ public final class BuildDataManager { private final PathRelativizerService myRelativizer; private boolean myProcessConstantsIncrementally = !Boolean.parseBoolean(System.getProperty(PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY, "false")); - private final @NotNull ConcurrentMap, SourceToOutputMappingWrapper> buildTargetToSourceToOutputMapping = new ConcurrentHashMap<>(); + @ApiStatus.Internal + @TestOnly + public BuildDataManager(BuildDataPaths dataPaths, + BuildTargetsState targetsState, + @NotNull PathRelativizerService relativizer) throws IOException { + this(dataPaths, targetsState, relativizer, null, null); + } @ApiStatus.Internal public BuildDataManager(BuildDataPaths dataPaths, BuildTargetsState targetsState, - PathRelativizerService relativizer, + @NotNull PathRelativizerService relativizer, + @Nullable ProjectStamps fileStampService, @Nullable StorageManager storageManager) throws IOException { + this.fileStampService = fileStampService; + myDataPaths = dataPaths; myTargetsState = targetsState; this.storageManager = storageManager; try { - if (storageManager == null) { - mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer); - } - else { - mySrcToFormMap = new ExperimentalOneToManyPathMapping("source-to-form", storageManager, relativizer); - } + sourceToFormMap = storageManager == null ? new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer) : null; myOutputToTargetRegistry = new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer); File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot()); if (JavaBuilderUtil.isDepGraphEnabled()) { myMappings = null; createDependencyGraph(mappingsRoot, false); - FileUtil.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), false)); // delete older mappings data if available + FileUtilRt.delete(getMappingsRoot(myDataPaths.getDataStorageRoot(), false)); // delete older mappings data if available LOG.info("Using DependencyGraph-based build incremental analysis"); } else { @@ -107,12 +123,15 @@ public final class BuildDataManager { throw e; } myVersionFile = new File(myDataPaths.getDataStorageRoot(), "version.dat"); - myDepGraphPathMapper = relativizer != null? new PathSourceMapper(relativizer::toFull, relativizer::toRelative) : new PathSourceMapper(); + myDepGraphPathMapper = new PathSourceMapper(relativizer::toFull, relativizer::toRelative); myRelativizer = relativizer; } - public @Nullable StorageManager getStorageManager() { - return storageManager; + @ApiStatus.Internal + public void clearCache() { + if (storageManager != null) { + storageManager.clearCache(); + } } public void setProcessConstantsIncrementally(boolean processInc) { @@ -135,7 +154,7 @@ public final class BuildDataManager { try { FileUtilRt.deleteRecursively(getDataPaths().getTargetDataRoot(targetType, targetId)); if (storageManager != null) { - storageManager.removeStaleMaps(targetId, targetType.getTypeId()); + storageManager.removeMaps(targetId, targetType.getTypeId()); } } finally { @@ -148,30 +167,53 @@ public final class BuildDataManager { } public @NotNull SourceToOutputMapping getSourceToOutputMap(@NotNull BuildTarget target) throws IOException { - int targetId = myTargetsState.getBuildTargetId(target); + if (storageManager == null) { + try { + return buildTargetToSourceToOutputMapping.computeIfAbsent(target, this::createSourceToOutputMap); + } + catch (BuildDataCorruptedException e) { + LOG.info(e); + throw e.getCause(); + } + } + else { + return getPerTargetMapManager(target).getSourceToOutputMapping(); + } + } + + private @NotNull SourceToOutputMappingWrapper createSourceToOutputMap(@NotNull BuildTarget target) { + SourceToOutputMapping map; try { - return buildTargetToSourceToOutputMapping.computeIfAbsent(target, t -> { - SourceToOutputMapping map; - if (storageManager == null) { - try { - Path file = myDataPaths.getTargetDataRootDir(t).resolve(SRC_TO_OUTPUT_STORAGE).resolve(SRC_TO_OUTPUT_FILE_NAME); - map = new SourceToOutputMappingImpl(file, myRelativizer); - } - catch (IOException e) { - LOG.info(e); - throw new BuildDataCorruptedException(e); - } - } - else { - map = ExperimentalSourceToOutputMapping.createSourceToOutputMap(storageManager, myRelativizer, target); - } - return new SourceToOutputMappingWrapper(map, targetId); - }); + Path file = myDataPaths.getTargetDataRootDir(target).resolve(SRC_TO_OUTPUT_STORAGE).resolve(SRC_TO_OUTPUT_FILE_NAME); + map = new SourceToOutputMappingImpl(file, myRelativizer); } - catch (BuildDataCorruptedException e) { + catch (IOException e) { LOG.info(e); - throw e.getCause(); + throw new BuildDataCorruptedException(e); } + return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target)); + } + + public @Nullable StampsStorage getFileStampStorage(@NotNull BuildTarget target) { + if (storageManager == null) { + return fileStampService == null ? null : fileStampService.getStampStorage(); + } + return getPerTargetMapManager(target).stamp; + } + + private @NotNull PerTargetMapManager getPerTargetMapManager(@NotNull BuildTarget target) { + return targetToMapManager.computeIfAbsent(target, it -> { + assert storageManager != null; + return new PerTargetMapManager(storageManager, myRelativizer, it, mapping -> new SourceToOutputMappingWrapper(mapping, myTargetsState.getBuildTargetId(it))); + }); + } + + /** + * @deprecated Use {@link BuildDataManager#getFileStampStorage(BuildTarget)}. + */ + @Deprecated(forRemoval = true) + public @NotNull ProjectStamps getFileStampService() { + return Objects.requireNonNull(fileStampService); } @ApiStatus.Internal @@ -185,8 +227,8 @@ public final class BuildDataManager { } @ApiStatus.Internal - public OneToManyPathMapping getSourceToFormMap() { - return mySrcToFormMap; + public @NotNull OneToManyPathMapping getSourceToFormMap(@NotNull BuildTarget target) { + return sourceToFormMap == null ? getPerTargetMapManager(target).getSourceToForm() : sourceToFormMap; } @ApiStatus.Internal @@ -224,6 +266,8 @@ public final class BuildDataManager { if (sourceToOutput != null && sourceToOutput.myDelegate instanceof StorageOwner) { ((StorageOwner)sourceToOutput.myDelegate).close(); } + + targetToMapManager.remove(target); } } finally { @@ -240,10 +284,20 @@ public final class BuildDataManager { } } - public void clean(Consumer> asyncTaskCollector) throws IOException { + public void clean(@NotNull Consumer> asyncTaskCollector) throws IOException { + if (fileStampService != null) { + try { + ((StorageOwner)fileStampService.getStampStorage()).clean(); + } + catch (Throwable e) { + LOG.error(new ProjectBuildException(JpsBuildBundle.message("build.message.error.cleaning.timestamps.storage"), e)); + } + } + try { allTargetStorages(asyncTaskCollector).clean(); buildTargetToSourceToOutputMapping.clear(); + targetToMapManager.clear(); myTargetStorages.clear(); if (storageManager != null) { storageManager.clean(); @@ -251,8 +305,8 @@ public final class BuildDataManager { } finally { try { - if (mySrcToFormMap instanceof StorageOwner) { - wipeStorage(getSourceToFormsRoot(), (StorageOwner)mySrcToFormMap); + if (sourceToFormMap != null) { + wipeStorage(getSourceToFormsRoot(), sourceToFormMap); } } finally { @@ -314,14 +368,19 @@ public final class BuildDataManager { } public void flush(boolean memoryCachesOnly) { - if (storageManager != null) { + if (storageManager == null) { + if (fileStampService != null) { + ((StorageOwner)fileStampService.getStampStorage()).flush(false); + } + } + else if (!memoryCachesOnly) { storageManager.commit(); } allTargetStorages().flush(memoryCachesOnly); myOutputToTargetRegistry.flush(memoryCachesOnly); - if (mySrcToFormMap instanceof StorageOwner) { - ((StorageOwner)mySrcToFormMap).flush(memoryCachesOnly); + if (sourceToFormMap != null) { + sourceToFormMap.flush(memoryCachesOnly); } Mappings mappings = myMappings; @@ -334,15 +393,6 @@ public final class BuildDataManager { public void close() throws IOException { try { - if (storageManager != null) { - try { - storageManager.close(); - } - catch (Throwable e) { - LOG.error(e); - } - } - myTargetsState.save(); try { allTargetStorages().close(); @@ -351,6 +401,24 @@ public final class BuildDataManager { myTargetStorages.clear(); buildTargetToSourceToOutputMapping.clear(); } + + if (storageManager != null) { + targetToMapManager.clear(); + try { + storageManager.close(); + } + catch (Throwable e) { + LOG.error(e); + } + } + else if (fileStampService != null) { + try { + fileStampService.close(); + } + catch (Throwable e) { + LOG.error(e); + } + } } finally { try { @@ -358,9 +426,9 @@ public final class BuildDataManager { } finally { try { - if (mySrcToFormMap instanceof StorageOwner) { - synchronized (mySrcToFormMap) { - ((StorageOwner)mySrcToFormMap).close(); + if (sourceToFormMap != null) { + synchronized (sourceToFormMap) { + sourceToFormMap.close(); } } } @@ -392,27 +460,38 @@ public final class BuildDataManager { } } - public void closeSourceToOutputStorages(@NotNull Collection chunks) throws IOException { - IOException ex = null; - for (BuildTargetChunk chunk : chunks) { + public void closeSourceToOutputStorages(@NotNull BuildTargetChunk chunk) throws IOException { + if (storageManager != null) { for (BuildTarget target : chunk.getTargets()) { - try { - SourceToOutputMappingWrapper sourceToOutputMapping = buildTargetToSourceToOutputMapping.remove(target); - if (sourceToOutputMapping != null) { - (((StorageOwner)sourceToOutputMapping.myDelegate)).close(); - } - } - catch (IOException e) { - LOG.info(e); - if (ex == null) { - ex = e; - } + buildTargetToSourceToOutputMapping.remove(target); + } + return; + } + + Tracer.Span flush = Tracer.start("flushing"); + IOException error = null; + for (BuildTarget target : chunk.getTargets()) { + SourceToOutputMappingWrapper sourceToOutputMapping = buildTargetToSourceToOutputMapping.remove(target); + if (sourceToOutputMapping == null) { + continue; + } + + StorageOwner delegate = (StorageOwner)sourceToOutputMapping.myDelegate; + try { + delegate.close(); + } + catch (IOException e) { + LOG.info(e); + if (error == null) { + error = e; } } } - if (ex != null) { - throw ex; + if (error != null) { + throw error; } + + flush.complete(); } private @NotNull Path getSourceToOutputMapRoot(BuildTarget target) { @@ -449,6 +528,7 @@ public final class BuildDataManager { private static void wipeStorage(@NotNull Path root, @Nullable StorageOwner storage) { if (storage != null) { + //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (storage) { try { storage.clean(); @@ -507,7 +587,7 @@ public final class BuildDataManager { myRelativizer.reportUnhandledPaths(); } - private final class SourceToOutputMappingWrapper implements SourceToOutputMapping, Supplier<@Nullable StorageOwner> { + final class SourceToOutputMappingWrapper implements SourceToOutputMapping, Supplier<@Nullable StorageOwner> { private final SourceToOutputMapping myDelegate; private final int myBuildTargetId; @@ -701,5 +781,4 @@ public final class BuildDataManager { } }; } - } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetConfiguration.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetConfiguration.java index 3a1b4ff37cfd..4be272d24ee4 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetConfiguration.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetConfiguration.java @@ -54,7 +54,7 @@ public final class BuildTargetConfiguration { catch (NoSuchFileException ignore) { } catch (IOException e) { - LOG.info("Cannot load configuration of " + target); + LOG.warn("Cannot load configuration of " + target, e); } return ""; } @@ -77,12 +77,11 @@ public final class BuildTargetConfiguration { if (LOG.isDebugEnabled()) { LOG.debug(target + " configuration was changed:"); - LOG.debug("Old:"); - LOG.debug(configuration); - LOG.debug("New:"); - LOG.debug(currentState); + LOG.debug("Old: " + configuration); + LOG.debug("New: " + currentState); LOG.debug(target + " will be recompiled"); } + if (target instanceof ModuleBuildTarget) { final JpsModule module = ((ModuleBuildTarget)target).getModule(); synchronized (MODULES_WITH_TARGET_CONFIG_CHANGED_KEY) { @@ -124,7 +123,7 @@ public final class BuildTargetConfiguration { return myTargetsState.getDataPaths().getTargetDataRootDir(target).resolve("nonexistent-outputs.dat"); } - private @NotNull String getCurrentState(@NotNull ProjectDescriptor pd) { + private @NotNull String getCurrentState(@NotNull ProjectDescriptor projectDescriptor) { String state = currentState; if (state != null) { return state; @@ -132,12 +131,12 @@ public final class BuildTargetConfiguration { if (target instanceof BuildTargetHashSupplier) { HashStream64 hash = Hashing.komihash5_0().hashStream(); - ((BuildTargetHashSupplier)target).computeConfigurationDigest(pd, hash); + ((BuildTargetHashSupplier)target).computeConfigurationDigest(projectDescriptor, hash); state = Long.toUnsignedString(hash.getAsLong(), Character.MAX_RADIX); } else { StringWriter out = new StringWriter(); - target.writeConfiguration(pd, new PrintWriter(out)); + target.writeConfiguration(projectDescriptor, new PrintWriter(out)); state = out.toString(); } currentState = state; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java index 414eec714653..8aa6684fad31 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java @@ -68,10 +68,9 @@ public final class BuildTargetSourcesState implements BuildListener { // Some modules can have same out folder for different BuildTarget's to avoid an extra hash calculation collection will be used // There are no pre-calculated hashes for entries from this collection in FileStampStorage private final Map calculatedHashes = new ConcurrentHashMap<>(); - private final PathRelativizerService relativizer; private final BuildTargetIndex buildTargetIndex; private final BuildRootIndex buildRootIndex; - private final ProjectStamps projectStamps; + private final BuildDataManager dataManager; private final CompileContext context; private final String outputFolderPath; private final Path targetStateStorage; @@ -79,14 +78,13 @@ public final class BuildTargetSourcesState implements BuildListener { public BuildTargetSourcesState(@NotNull CompileContext context) { this.context = context; - ProjectDescriptor pd = context.getProjectDescriptor(); - projectStamps = pd.getProjectStamps(); - buildRootIndex = pd.getBuildRootIndex(); - buildTargetIndex = pd.getBuildTargetIndex(); - relativizer = pd.dataManager.getRelativizer(); - outputFolderPath = getOutputFolderPath(pd.getProject()); + ProjectDescriptor projectDescriptor = context.getProjectDescriptor(); + dataManager = projectDescriptor.dataManager; + buildRootIndex = projectDescriptor.getBuildRootIndex(); + buildTargetIndex = projectDescriptor.getBuildTargetIndex(); + outputFolderPath = getOutputFolderPath(projectDescriptor.getProject()); - BuildDataPaths dataPaths = pd.getTargetsState().getDataPaths(); + BuildDataPaths dataPaths = projectDescriptor.getTargetsState().getDataPaths(); targetStateStorage = dataPaths.getDataStorageRoot().toPath().resolve(TARGET_SOURCES_STATE_FILE_NAME); // subscribe to events for reporting only changed build targets @@ -116,6 +114,7 @@ public final class BuildTargetSourcesState implements BuildListener { result = Collections.emptyList(); } else { + PathRelativizerService relativizer = dataManager.getRelativizer(); List> list = new ArrayList<>(buildTargets.size()); for (BuildTarget t : buildTargets) { list.add(parallelBuildExecutor.submit(() -> { @@ -265,6 +264,8 @@ public final class BuildTargetSourcesState implements BuildListener { return; } + HashStampStorage stampStorage = (HashStampStorage)dataManager.getFileStampStorage(target); + Files.walkFileTree(rootFile, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { @@ -276,7 +277,7 @@ public final class BuildTargetSourcesState implements BuildListener { if (!buildRootIndex.isFileAccepted(path.toFile(), rootDescriptor)) { return FileVisitResult.CONTINUE; } - getFileHash(target, path, rootFile, hash, hashToReuse); + getFileHash(path, rootFile, hash, hashToReuse, stampStorage); return FileVisitResult.CONTINUE; } }); @@ -305,16 +306,13 @@ public final class BuildTargetSourcesState implements BuildListener { .getAsLong(); } - private void getFileHash(@NotNull BuildTarget target, - @NotNull Path path, - @NotNull Path rootFile, - @NotNull LongArrayList hash, - @NotNull HashStream64 hashToReuse) { - StampsStorage storage = projectStamps.getStampStorage(); - assert storage instanceof HashStampStorage; - HashStampStorage fileStampStorage = (HashStampStorage)storage; - Long fileHash = fileStampStorage.getStoredFileHash(path, target); - if (fileHash == null) { + private static void getFileHash(@NotNull Path path, + @NotNull Path rootFile, + @NotNull LongArrayList hash, + @NotNull HashStream64 hashToReuse, + @NotNull HashStampStorage stampStorage) { + HashStamp stamp = stampStorage.getStoredFileStamp(path); + if (stamp == null) { return; } @@ -325,7 +323,7 @@ public final class BuildTargetSourcesState implements BuildListener { hash.add(hashToReuse .reset() - .putLong(fileHash) + .putLong(stamp.hash) .putString(relativePath) .getAsLong()); } @@ -353,8 +351,8 @@ public final class BuildTargetSourcesState implements BuildListener { return new HashMap<>(); } - private boolean reportStateUnavailable() { - return !PORTABLE_CACHES || projectStamps == null; + private static boolean reportStateUnavailable() { + return !PORTABLE_CACHES; } private static @NotNull String toRelative(@NotNull Path target, @NotNull Path rootPath) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt index 684748e8ee0a..a56638d60792 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt @@ -3,24 +3,16 @@ package org.jetbrains.jps.incremental.storage -import com.dynatrace.hash4j.hashing.Hashing import org.jetbrains.annotations.ApiStatus import org.jetbrains.jps.incremental.relativizer.PathRelativizerService -import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType -import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType +import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash @ApiStatus.Internal -open class ExperimentalOneToManyPathMapping protected constructor( +open class ExperimentalOneToManyPathMapping( @JvmField protected val mapHandle: MapHandle>, @JvmField protected val relativizer: PathRelativizerService, private val valueOffset: Int = 0, -) : OneToManyPathMapping{ - constructor( - mapName: String, - storageManager: StorageManager, - relativizer: PathRelativizerService, - ) : this(storageManager.openMap(mapName, LongPairKeyDataType, StringListDataType), relativizer) - +) : OneToManyPathMapping { protected fun getKey(path: String): LongArray = stringTo128BitHash(relativizer.toRelative(path)) @Suppress("ReplaceGetOrSet") @@ -53,9 +45,4 @@ open class ExperimentalOneToManyPathMapping protected constructor( final override fun remove(path: String) { mapHandle.map.remove(getKey(path)) } -} - -internal fun stringTo128BitHash(string: String): LongArray { - val bytes = string.toByteArray() - return longArrayOf(Hashing.xxh3_64().hashBytesToLong(bytes), Hashing.komihash5_0().hashBytesToLong(bytes)) } \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt index 314e2acb841b..c1a7e17b4312 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt @@ -12,6 +12,7 @@ import org.jetbrains.jps.builders.storage.SourceToOutputMapping import org.jetbrains.jps.incremental.relativizer.PathRelativizerService import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType +import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash @Internal class ExperimentalSourceToOutputMapping private constructor( @@ -25,7 +26,6 @@ class ExperimentalSourceToOutputMapping private constructor( it.setValueType(StringListDataType) } - @JvmStatic fun createSourceToOutputMap( storageManager: StorageManager, relativizer: PathRelativizerService, @@ -50,7 +50,7 @@ class ExperimentalSourceToOutputMapping private constructor( // we can use composite key and sort by target id, but as we compile targets in parallel: // * avoid blocking - in-memory lock per map root, // * avoid a huge B-tree and reduce rebalancing time due to contention. - val mapName = storageManager.getMapName(targetId = targetId, typeId = targetTypeId, suffix = "src-to-out-v1") + val mapName = storageManager.getMapName(targetId = targetId, targetTypeId = targetTypeId, suffix = "src-to-out-v1") return ExperimentalSourceToOutputMapping( mapHandle = storageManager.openMap(mapName, mapBuilder), relativizer = relativizer, diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalTimeStampStorage.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalTimeStampStorage.kt new file mode 100644 index 000000000000..42f9dc3d5f72 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalTimeStampStorage.kt @@ -0,0 +1,89 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.jps.incremental.storage + +import org.h2.mvstore.DataUtils.readVarLong +import org.h2.mvstore.WriteBuffer +import org.h2.mvstore.type.DataType +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.VisibleForTesting +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.incremental.FSOperations +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType +import org.jetbrains.jps.incremental.storage.dataTypes.stringTo128BitHash +import java.nio.ByteBuffer +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributes + +@ApiStatus.Internal +class ExperimentalTimeStampStorage private constructor( + private val mapHandle: MapHandle, + private val relativizer: PathRelativizerService, +) : StampsStorage { + companion object { + @VisibleForTesting + fun createSourceToStampMap( + storageManager: StorageManager, + relativizer: PathRelativizerService, + targetId: String, + targetTypeId: String, + ): ExperimentalTimeStampStorage { + val mapName = storageManager.getMapName(targetId, targetTypeId, "file-hash-and-mtime-v1") + return ExperimentalTimeStampStorage( + mapHandle = storageManager.openMap(mapName, LongPairKeyDataType, LongStorageValueType), + relativizer = relativizer, + ) + } + } + + override fun getStorageRoot(): Path? = null + + override fun updateStamp(file: Path, buildTarget: BuildTarget<*>?, currentFileTimestamp: Long) { + mapHandle.map.put(createKey(file), currentFileTimestamp) + } + + private fun createKey(file: Path): LongArray = stringTo128BitHash(relativizer.toRelative(file)) + + override fun removeStamp(file: Path, target: BuildTarget<*>?) { + mapHandle.map.remove(createKey(file)) + } + + override fun getCurrentStampIfUpToDate(file: Path, target: BuildTarget<*>?, attrs: BasicFileAttributes?): Long? { + return mapHandle.map.get(createKey(file))?.takeIf { + it == (if (attrs == null || !attrs.isRegularFile) FSOperations.lastModified(file) else attrs.lastModifiedTime().toMillis()) + } + } +} + +private object LongStorageValueType : DataType { + override fun isMemoryEstimationAllowed() = true + + override fun getMemory(obj: Long): Int = Long.SIZE_BYTES + + override fun createStorage(size: Int): Array = arrayOfNulls(size) + + override fun write(buff: WriteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + for (value in (storage as Array)) { + buff.putVarLong(value) + } + } + + override fun write(buff: WriteBuffer, obj: Long) = throw IllegalStateException("Must not be called") + + override fun read(buff: ByteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + storage as Array + for (i in 0 until len) { + storage[i] = readVarLong(buff) + } + } + + override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called") + + override fun compare(a: Long, b: Long) = throw IllegalStateException("Must not be called") + + override fun binarySearch(key: Long?, storage: Any?, size: Int, initialGuess: Int) = throw IllegalStateException("Must not be called") +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java index 478ddb3cd275..4efebdac040e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java @@ -39,42 +39,28 @@ final class FileTimestampStorage extends AbstractStateStorage target) throws IOException { + public FileTimestamp getCurrentStampIfUpToDate(@NotNull Path file, @NotNull BuildTarget target, @Nullable BasicFileAttributes attrs) throws IOException { TimestampPerTarget[] state = getState(file.toFile()); - if (state != null) { - int targetId = myTargetsState.getBuildTargetId(target); - for (TimestampPerTarget timestampPerTarget : state) { - if (timestampPerTarget.targetId == targetId) { - return FileTimestamp.fromLong(timestampPerTarget.timestamp); - } + if (state == null) { + return null; + } + + int targetId = myTargetsState.getBuildTargetId(target); + for (TimestampPerTarget timestampPerTarget : state) { + if (timestampPerTarget.targetId == targetId) { + long current = timestampPerTarget.timestamp; + long timestamp = (attrs == null || !attrs.isRegularFile()) ? FSOperations.lastModified(file) : attrs.lastModifiedTime().toMillis(); + return current == timestamp ? FileTimestamp.fromLong(current) : null; } } return null; } @Override - public @NotNull FileTimestamp getCurrentStamp(@NotNull Path file, long currentFileTimestamp) { - return FileTimestamp.fromLong(currentFileTimestamp); - } - - @Override - public boolean isDirtyStamp(@NotNull Stamp stamp, @NotNull Path file) { - return !(stamp instanceof FileTimestamp) || ((FileTimestamp)stamp).myTimestamp != FSOperations.lastModified(file); - } - - @Override - public boolean isDirtyStamp(@Nullable Stamp stamp, @NotNull Path file, @NotNull BasicFileAttributes attrs) { - if (!(stamp instanceof FileTimestamp)) return true; - FileTimestamp timestamp = (FileTimestamp) stamp; - // for symlinks, the attr structure reflects the symlink's timestamp and not symlink's target timestamp - return attrs.isRegularFile() ? attrs.lastModifiedTime().toMillis() != timestamp.myTimestamp : isDirtyStamp(timestamp, file); - } - - @Override - public void saveStamp(@NotNull Path file, BuildTarget buildTarget, @NotNull FileTimestamp stamp) throws IOException { + public void updateStamp(@NotNull Path file, BuildTarget buildTarget, long currentFileTimestamp) throws IOException { int targetId = myTargetsState.getBuildTargetId(buildTarget); File ioFile = file.toFile(); - update(ioFile, updateTimestamp(getState(ioFile), targetId, stamp.asLong())); + update(ioFile, updateTimestamp(getState(ioFile), targetId, currentFileTimestamp)); } private static TimestampPerTarget @NotNull [] updateTimestamp(TimestampPerTarget[] oldState, final int targetId, long timestamp) { @@ -146,26 +132,20 @@ final class FileTimestampStorage extends AbstractStateStorage, stamp: HashStamp) { - mapHandle.map.put(createKey(buildTarget, file), stamp) - } - - private fun createKey(target: BuildTarget<*>, file: Path): HashStampStorageKey { - return HashStampStorageKey( - targetId = targetState.getBuildTargetId(target), - // getBytes is faster (70k op/s vs. 50 op/s) - // use xxh3_64 as it is more proven hash algo than komihash - pathHash = Hashing.xxh3_64().hashBytesToLong(relativizer.toRelative(file).toByteArray()), - ) - } - - override fun removeStamp(file: Path, target: BuildTarget<*>) { - mapHandle.map.remove(createKey(target, file)) - } - - override fun getPreviousStamp(file: Path, target: BuildTarget<*>): HashStamp? { - return mapHandle.map.get(createKey(target, file)) - } - - fun getStoredFileHash(file: Path, target: BuildTarget<*>): Long? { - return mapHandle.map.get(createKey(target, file))?.hash - } - - override fun getCurrentStamp(file: Path, currentFileTimestamp: Long): HashStamp { - return HashStamp(hash = FileHashUtil.getFileHash(file), timestamp = currentFileTimestamp) - } - - override fun isDirtyStamp(stamp: StampsStorage.Stamp, file: Path): Boolean { - if (stamp !is HashStamp) { - return true - } - if (stamp.timestamp == FSOperations.lastModified(file)) { - return false - } - return stamp.hash != FileHashUtil.getFileHash(file) - } - - override fun isDirtyStamp(stamp: StampsStorage.Stamp?, file: Path, attrs: BasicFileAttributes): Boolean { - if (stamp !is HashStamp) { - return true - } - - // If equal, then non-dirty. - // If not equal, then we check the hash to avoid marking the file as `dirty` only because of a different timestamp. - // We cannot rely solely on the hash, as getting the last-modified timestamp is much cheaper than computing the file hash. - if ((if (attrs.isRegularFile) attrs.lastModifiedTime().toMillis() else FSOperations.lastModified(file)) == stamp.timestamp) { - return false - } - - return stamp.hash != FileHashUtil.getFileHash(file) - } -} - -internal class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp - -private class HashStampStorageKey(@JvmField val targetId: Int, @JvmField val pathHash: Long) - -private object HashStampStorageKeyType : DataType { - override fun isMemoryEstimationAllowed() = true - - override fun getMemory(obj: HashStampStorageKey): Int = Int.SIZE_BYTES + Long.SIZE_BYTES - - override fun createStorage(size: Int): Array = arrayOfNulls(size) - - override fun write(buff: WriteBuffer, storage: Any, len: Int) { - @Suppress("UNCHECKED_CAST") - for (key in (storage as Array)) { - buff.putVarInt(key.targetId) - // not var long - maybe negative number - buff.putLong(key.pathHash) + companion object { + @VisibleForTesting + fun createSourceToStampMap( + storageManager: StorageManager, + relativizer: PathRelativizerService, + targetId: String, + targetTypeId: String, + ): HashStampStorage { + val mapName = storageManager.getMapName(targetId = targetId, targetTypeId = targetTypeId, suffix = "file-hash-and-mtime-v1") + return HashStampStorage( + mapHandle = storageManager.openMap(mapName, LongPairKeyDataType, HashStampStorageValueType), + relativizer = relativizer, + ) } } - override fun write(buff: WriteBuffer, obj: HashStampStorageKey) = throw IllegalStateException("Must not be called") + override fun getStorageRoot(): Path? = null - override fun read(buff: ByteBuffer, storage: Any, len: Int) { - @Suppress("UNCHECKED_CAST") - storage as Array - for (i in 0 until len) { - storage[i] = HashStampStorageKey(targetId = readVarInt(buff), pathHash = buff.getLong()) - } + override fun updateStamp(file: Path, buildTarget: BuildTarget<*>?, currentFileTimestamp: Long) { + mapHandle.map.put(createKey(file), HashStamp(hash = FileHashUtil.getFileHash(file), timestamp = currentFileTimestamp)) } - override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called") + private fun createKey(file: Path): LongArray = stringTo128BitHash(relativizer.toRelative(file)) - override fun binarySearch(key: HashStampStorageKey, storage: Any, size: Int, initialGuess: Int): Int { - @Suppress("UNCHECKED_CAST") - storage as Array - - var low = 0 - var high = size - 1 - // the cached index minus one, so that for the first time (when cachedCompare is 0), the default value is used - var x = initialGuess - 1 - if (x < 0 || x > high) { - x = high ushr 1 - } - while (low <= high) { - val b = storage[x] - val compare = when { - key.targetId > b.targetId -> 1 - key.targetId < b.targetId -> -1 - key.pathHash > b.pathHash -> 1 - key.pathHash < b.pathHash -> -1 - else -> 0 - } - - when { - compare > 0 -> low = x + 1 - compare < 0 -> high = x - 1 - else -> return x - } - x = (low + high) ushr 1 - } - return low.inv() + override fun removeStamp(file: Path, target: BuildTarget<*>?) { + mapHandle.map.remove(createKey(file)) } - @Suppress("DuplicatedCode") - override fun compare(a: HashStampStorageKey, b: HashStampStorageKey): Int { - return when { - a.targetId > b.targetId -> 1 - a.targetId < b.targetId -> -1 - a.pathHash > b.pathHash -> 1 - a.pathHash < b.pathHash -> -1 - else -> 0 + fun getStoredFileStamp(file: Path): HashStamp? { + return mapHandle.map.get(createKey(file)) + } + + override fun getCurrentStampIfUpToDate(file: Path, target: BuildTarget<*>?, attrs: BasicFileAttributes?): HashStamp? { + return mapHandle.map.get(createKey(file))?.takeIf { + val timestamp = if (attrs == null || !attrs.isRegularFile) FSOperations.lastModified(file) else attrs.lastModifiedTime().toMillis() + timestamp == it.timestamp || it.hash == FileHashUtil.getFileHash(file) } } } +@ApiStatus.Internal +@VisibleForTesting +class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) + private object HashStampStorageValueType : DataType { override fun isMemoryEstimationAllowed() = true diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java index 543c03fa9191..cd9c4f153c54 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java @@ -1,14 +1,15 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.incremental.storage; -import com.intellij.openapi.util.io.FileUtil; -import com.intellij.openapi.util.text.StringUtil; +import com.dynatrace.hash4j.hashing.Hashing; +import com.intellij.openapi.util.io.FileUtilRt; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.EnumeratorIntegerDescriptor; import com.intellij.util.io.PersistentMapBuilder; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; +import it.unimi.dsi.fastutil.ints.IntSets; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; @@ -16,17 +17,17 @@ import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; public final class OutputToTargetRegistry extends AbstractStateStorage { - private final PathRelativizerService myRelativizer; - - private static final DataExternalizer DATA_EXTERNALIZER = new DataExternalizer() { + private static final DataExternalizer DATA_EXTERNALIZER = new DataExternalizer<>() { @Override - public void save(final @NotNull DataOutput out, IntSet value) throws IOException { + public void save(@NotNull DataOutput out, IntSet value) throws IOException { IntIterator iterator = value.iterator(); while (iterator.hasNext()) { out.writeInt(iterator.nextInt()); @@ -35,27 +36,28 @@ public final class OutputToTargetRegistry extends AbstractStateStorage 0) { result.add(in.readInt()); } return result; } }; + private final PathRelativizerService relativizer; OutputToTargetRegistry(@NotNull Path storePath, PathRelativizerService relativizer) throws IOException { super(PersistentMapBuilder.newBuilder(storePath, EnumeratorIntegerDescriptor.INSTANCE, DATA_EXTERNALIZER)); - myRelativizer = relativizer; + + this.relativizer = relativizer; } - void addMapping(String outputPath, int buildTargetId) throws IOException { - addMapping(Collections.singleton(outputPath), buildTargetId); + void addMapping(@NotNull String outputPath, int buildTargetId) throws IOException { + appendData(pathHashCode(outputPath), IntSets.singleton(buildTargetId)); } - void addMapping(Collection outputPaths, int buildTargetId) throws IOException { - final IntSet set = new IntOpenHashSet(); - set.add(buildTargetId); + void addMapping(@NotNull Collection outputPaths, int buildTargetId) throws IOException { + IntSet set = IntSets.singleton(buildTargetId); for (String outputPath : outputPaths) { appendData(pathHashCode(outputPath), set); } @@ -78,26 +80,25 @@ public final class OutputToTargetRegistry extends AbstractStateStorage getSafeToDeleteOutputs(Collection outputPaths, int currentTargetId) throws IOException { - final int size = outputPaths.size(); + public @NotNull Collection getSafeToDeleteOutputs(Collection outputPaths, int currentTargetId) throws IOException { + int size = outputPaths.size(); if (size == 0) { return outputPaths; } - final Collection result = new ArrayList<>(size); + + List result = new ArrayList<>(size); for (String outputPath : outputPaths) { - final int key = pathHashCode(outputPath); + int key = pathHashCode(outputPath); synchronized (dataLock) { - final IntSet associatedTargets = getState(key); + IntSet associatedTargets = getState(key); if (associatedTargets == null || associatedTargets.size() != 1) { continue; } @@ -110,11 +111,12 @@ public final class OutputToTargetRegistry extends AbstractStateStorage, + sourceToOutputMappingWrapper: UnaryOperator, +) { + @JvmField + val stamp: StampsStorage<*> = if (ProjectStamps.PORTABLE_CACHES) { + HashStampStorage.createSourceToStampMap( + storageManager = storageManager, + relativizer = relativizer, + targetId = target.id, + targetTypeId = target.targetType.typeId, + ) + } + else { + ExperimentalTimeStampStorage.createSourceToStampMap( + storageManager = storageManager, + relativizer = relativizer, + targetId = target.id, + targetTypeId = target.targetType.typeId, + ) + } + + val sourceToOutputMapping: SourceToOutputMapping by lazy { + sourceToOutputMappingWrapper.apply(ExperimentalSourceToOutputMapping.createSourceToOutputMap( + storageManager = storageManager, + relativizer = relativizer, + target = target, + )) + } + + val sourceToForm: ExperimentalOneToManyPathMapping by lazy { + ExperimentalOneToManyPathMapping( + mapHandle = storageManager.openMap( + name = storageManager.getMapName(targetId = target.id, targetTypeId = target.targetType.typeId, suffix = "source-to-form-v1"), + keyType = LongPairKeyDataType, + valueType = StringListDataType, + ), + relativizer = relativizer, + ) + } +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java index d4cd8268069c..27eee5d661d5 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java @@ -3,7 +3,6 @@ package org.jetbrains.jps.incremental.storage; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.io.NioFiles; -import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; @@ -23,15 +22,10 @@ public final class ProjectStamps { private static final Logger LOG = Logger.getInstance(ProjectStamps.class); - private final StampsStorage stampStorage; - - @ApiStatus.Internal - public ProjectStamps(@NotNull StampsStorage stampStorage) throws IOException { - this.stampStorage = stampStorage; - } + private final StampsStorage stampStorage; public ProjectStamps(@NotNull Path dataStorageRoot, @NotNull BuildTargetsState targetsState) throws IOException { - this(new FileTimestampStorage(dataStorageRoot, targetsState)); + this.stampStorage = new FileTimestampStorage(dataStorageRoot, targetsState); } /** @@ -43,7 +37,7 @@ public final class ProjectStamps { this(dataStorageRoot.toPath(), targetsState); } - public @NotNull StampsStorage getStampStorage() { + public @NotNull StampsStorage getStampStorage() { return stampStorage; } @@ -56,7 +50,10 @@ public final class ProjectStamps { catch (IOException e) { LOG.error(e); try { - NioFiles.deleteRecursively(stampStorage.getStorageRoot()); + Path root = stampStorage.getStorageRoot(); + if (root != null) { + NioFiles.deleteRecursively(root); + } } catch (IOException ignore) { } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java index 553c1a21c2a2..bdbe6758e9fd 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java @@ -9,27 +9,16 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import static org.jetbrains.jps.incremental.storage.StampsStorage.Stamp; - /** * @author Eugene Zhuravlev */ -public interface StampsStorage { - Path getStorageRoot(); +public interface StampsStorage { + @Nullable Path getStorageRoot(); - void saveStamp(@NotNull Path file, BuildTarget buildTarget, @NotNull T stamp) throws IOException; + void updateStamp(@NotNull Path file, BuildTarget buildTarget, long currentFileTimestamp) throws IOException; void removeStamp(@NotNull Path file, BuildTarget buildTarget) throws IOException; @Nullable - T getPreviousStamp(@NotNull Path file, BuildTarget target) throws IOException; - - @NotNull - T getCurrentStamp(@NotNull Path file, long currentFileTimestamp) throws IOException; - - boolean isDirtyStamp(@NotNull Stamp stamp, @NotNull Path file) throws IOException; - - boolean isDirtyStamp(@Nullable Stamp stamp, @NotNull Path file, @NotNull BasicFileAttributes attrs) throws IOException; - - interface Stamp { } + T getCurrentStampIfUpToDate(@NotNull Path file, BuildTarget target, @Nullable BasicFileAttributes attrs) throws IOException; } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt index 399233e26f8b..d5937da1ebcf 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt @@ -2,7 +2,8 @@ package org.jetbrains.jps.incremental.storage import com.intellij.openapi.diagnostic.Logger -import com.intellij.openapi.diagnostic.thisLogger +import com.intellij.openapi.diagnostic.debug +import com.intellij.openapi.diagnostic.logger import com.intellij.util.concurrency.SynchronizedClearableLazy import org.h2.mvstore.MVMap import org.h2.mvstore.MVStore @@ -15,15 +16,22 @@ import kotlin.time.measureTime private val MV_STORE_CACHE_SIZE_IN_MB = System.getProperty("jps.new.storage.cache.size.mb", "64").toInt() +private val LOG = logger() + @ApiStatus.Internal class StorageManager(@JvmField val file: Path, private val allowedCompactionTimeOnClose: Int) { - private val store = SynchronizedClearableLazy { createOrResetMvStore(file = file, readOnly = false, ::thisLogger) } + private val storeValue = SynchronizedClearableLazy { + LOG.debug { "Opening storage $file" } + createOrResetMvStore(file = file, readOnly = false, logSupplier = { LOG }) + } fun open() { - store.value + storeValue.value } fun openMap(name: String, keyType: DataType, valueType: DataType): MapHandle { + LOG.debug { "Open map $name" } + val mapBuilder = MVMap.Builder() mapBuilder.setKeyType(keyType) mapBuilder.setValueType(valueType) @@ -31,55 +39,71 @@ class StorageManager(@JvmField val file: Path, private val allowedCompactionTime } fun openMap(name: String, mapBuilder: MVMap.Builder): MapHandle { - return MapHandle(openOrResetMap(store = store.value, name = name, mapBuilder = mapBuilder, logSupplier = ::thisLogger)) + return MapHandle(openOrResetMap(store = storeValue.value, name = name, mapBuilder = mapBuilder, logSupplier = { LOG })) } /** Only if error occurred */ fun forceClose() { - store.valueIfInitialized?.closeImmediately() + if (LOG.isDebugEnabled) { + LOG.debug("Force closing storage $file", Throwable()) + } + + storeValue.valueIfInitialized?.let { + storeValue.drop() + it.closeImmediately() + } } fun close() { - store.valueIfInitialized?.let { - store.drop() + if (LOG.isDebugEnabled) { + LOG.debug("Closing storage $file", Throwable()) + } + + storeValue.valueIfInitialized?.let { + storeValue.drop() val isCompactOnClose = System.getProperty("jps.new.storage.compact.on.close", "true").toBoolean() it.close(if (isCompactOnClose) 0 else allowedCompactionTimeOnClose) if (isCompactOnClose && Files.exists(file)) { val time = measureTime { MVStoreTool.compact(file.toString(), false) } - thisLogger().info("Compacted storage in $time") + LOG.info("Compacted storage in $time") } } } fun commit() { - store.valueIfInitialized?.tryCommit() + storeValue.valueIfInitialized?.tryCommit() } fun clearCache() { // set again to force to clear the cache (in kb) - store.valueIfInitialized?.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024 + storeValue.valueIfInitialized?.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024 } fun clean() { - store.valueIfInitialized?.let { - store.drop() - it.closeImmediately() + val store = storeValue.valueIfInitialized + if (store == null) { Files.deleteIfExists(file) } - } - - fun removeStaleMaps(targetId: String, typeId: String) { - val store = store.value - for (mapName in store.mapNames) { - if (mapName.startsWith(getMapName(targetId = targetId, typeId = typeId, suffix = ""))) { + else { + // we cannot recreate the store if reference to map is already acquired - so, remove all maps + for (mapName in store.mapNames) { store.removeMap(mapName) } } } - fun getMapName(targetId: String, typeId: String, suffix: String): String = "$targetId|$typeId|$suffix" + fun removeMaps(targetId: String, typeId: String) { + val store = storeValue.value + for (mapName in store.mapNames) { + if (mapName.startsWith(getMapName(targetId = targetId, targetTypeId = typeId, suffix = ""))) { + store.removeMap(mapName) + } + } + } + + fun getMapName(targetId: String, targetTypeId: String, suffix: String): String = "$targetId|$targetTypeId|$suffix" } @ApiStatus.Internal @@ -142,9 +166,12 @@ private fun tryOpenMvStore(file: Path?, readOnly: Boolean, logSupplier: () -> Lo val store = MVStore.Builder() .fileName(file?.toAbsolutePath()?.toString()) .backgroundExceptionHandler(storeErrorHandler) - // avoid extra thread - db maintainer should use coroutines - .autoCommitDisabled() - .autoCommitBufferSize(4096) + // We do not disable auto-commit as JPS doesn't use Kotlin coroutines, so it's okay to use a separate daemon thread. + // Additionally, we ensure that the write operation will not slow down any tasks, + // as the actual save will be done in a background thread. + // Use an 8MB threshold for auto-commit instead of the default 1MB - + // if writes are performed too often, do not save intermediate B-Tree pages to disk. + .autoCommitBufferSize(8192) .cacheSize(MV_STORE_CACHE_SIZE_IN_MB) .let { if (readOnly) it.readOnly() else it diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt index a30e6f518bbe..7e93e7bee70d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt @@ -1,10 +1,18 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.incremental.storage.dataTypes +import com.dynatrace.hash4j.hashing.Hashing import org.h2.mvstore.WriteBuffer import org.h2.mvstore.type.DataType import java.nio.ByteBuffer +// getBytes is faster (70k op/s vs. 50 op/s) +// use xxh3_64 as first as it is more proven hash algo than komihash +internal fun stringTo128BitHash(string: String): LongArray { + val bytes = string.toByteArray() + return longArrayOf(Hashing.xxh3_64().hashBytesToLong(bytes), Hashing.komihash5_0().hashBytesToLong(bytes)) +} + internal object LongPairKeyDataType : DataType { override fun isMemoryEstimationAllowed() = true diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java index 337c971aa29d..bce188f0be61 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java @@ -201,8 +201,8 @@ public abstract class JpsBuildTestCase extends UsefulTestCase { BuildTargetsState targetsState = new BuildTargetsState(dataPaths, myModel, buildRootIndex); PathRelativizerService relativizer = new PathRelativizerService(myModel.getProject()); ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState); - BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, null); - return new ProjectDescriptor(myModel, new BuildFSState(true), projectStamps, dataManager, buildLoggingManager, index, + BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, projectStamps, null); + return new ProjectDescriptor(myModel, new BuildFSState(true), dataManager, buildLoggingManager, index, targetIndex, buildRootIndex, ignoredFileIndex); } catch (IOException e) { diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt new file mode 100644 index 000000000000..b3a1a9d300fd --- /dev/null +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/HashStampStorageFuzzTest.kt @@ -0,0 +1,85 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage + +import com.github.marschall.memoryfilesystem.MemoryFileSystemBuilder +import com.intellij.util.io.write +import net.jqwik.api.* +import net.jqwik.api.lifecycle.AfterProperty +import net.jqwik.api.lifecycle.BeforeProperty +import org.assertj.core.api.Assertions.assertThat +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import java.nio.file.FileSystem +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.deleteIfExists +import kotlin.io.path.deleteRecursively + +class HashStampStorageFuzzTest { + private lateinit var hashStampStorage: HashStampStorage + private lateinit var storageManager: StorageManager + private var file: Path? = null + + private lateinit var fs: FileSystem + + @BeforeProperty + fun setUp() { + file = Files.createTempFile("mvstore", ".db") + storageManager = StorageManager(file!!, 0) + hashStampStorage = HashStampStorage.createSourceToStampMap( + storageManager = storageManager, + relativizer = PathRelativizerService(), + targetId = "test-module", + targetTypeId = "java" + ) + + fs = MemoryFileSystemBuilder.newLinux().setCurrentWorkingDirectory("/").build() + } + + @AfterProperty + fun tearDown() { + storageManager.close() + file?.deleteIfExists() + + fs.close() + } + + @Provide + fun paths(): Arbitrary { + return Arbitraries.strings().alpha().numeric().withChars('/').ofMinLength(2).ofMaxLength(255) + } + + @Provide + fun hashStamps(): Arbitrary { + return Arbitraries.longs().map { HashStamp(hash = it, timestamp = System.currentTimeMillis()) } + } + + @Property + fun saveAndRetrieveStamp(@ForAll("paths") pathStr: String, @ForAll("hashStamps") stamp: HashStamp) { + val file = prepareAndGetFile(pathStr) + + hashStampStorage.updateStamp(file, null, stamp.timestamp) + val retrievedStamp = hashStampStorage.getStoredFileStamp(file) + assertThat(retrievedStamp).isNotNull() + assertThat(retrievedStamp!!.timestamp).isEqualTo(stamp.timestamp) + } + + @OptIn(ExperimentalPathApi::class) + private fun prepareAndGetFile(pathStr: String): Path { + val file = fs.getPath("/test", pathStr.replace("//", "/").let { if (it.length == 1) "/test" else it }) + file.parent.deleteRecursively() + file.write("") + return file + } + + @Property + fun removeAndCheckStamp(@ForAll("paths") pathStr: String) { + val file = prepareAndGetFile(pathStr) + + val stamp = HashStamp(hash = 12345L, timestamp = System.currentTimeMillis()) + hashStampStorage.updateStamp(file, null, stamp.timestamp) + hashStampStorage.removeStamp(file, null) + val retrievedStamp = hashStampStorage.getStoredFileStamp(file) + assertThat(retrievedStamp).isNull() + } +} \ No newline at end of file diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt index 67dd137d16e6..4d4053b50d6c 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/StorageManagerTest.kt @@ -45,7 +45,7 @@ class StorageManagerTest { mapping.appendOutput("foo/bar/Baz.java", "out/bar/Baz.class") assertThat(mapping.getOutputs("foo/bar/Baz.java")).containsExactly("out/bar/Baz.class") - storageManager.removeStaleMaps(targetId = "test-module", typeId = "java") + storageManager.removeMaps(targetId = "test-module", typeId = "java") assertThat(mapping.getOutputs("foo/bar/Baz.java")).isNull() } finally { diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/logging/jps/JpsFileLoggerFactory.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/logging/jps/JpsFileLoggerFactory.kt index d15a9caf7d29..fb6b23bf7253 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/logging/jps/JpsFileLoggerFactory.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/logging/jps/JpsFileLoggerFactory.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl.logging.jps; import com.intellij.openapi.diagnostic.IdeaLogRecordFormatter @@ -13,12 +13,10 @@ import java.util.logging.Level @ApiStatus.Internal class JpsFileLoggerFactory(logFile: Path, categoriesWithDebugLevel: String) : Factory { - private val appender: RollingFileHandler - private val categoriesWithDebugLevel: List + private val appender = RollingFileHandler(logFile, 20_000_000L, 10, true) + private val categoriesWithDebugLevel = if (categoriesWithDebugLevel.isEmpty()) emptyList() else categoriesWithDebugLevel.split(',') init { - this.categoriesWithDebugLevel = if (categoriesWithDebugLevel.isEmpty()) emptyList() else categoriesWithDebugLevel.split(",") - this.appender = RollingFileHandler(logFile, 20_000_000L, 10, true) this.appender.setFormatter(IdeaLogRecordFormatter()) } diff --git a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java index 977aa0aa93a8..d18a70113197 100644 --- a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java +++ b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java @@ -22,7 +22,7 @@ import org.jetbrains.jps.incremental.java.CopyResourcesUtil; import org.jetbrains.jps.incremental.java.FormsParsing; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; -import org.jetbrains.jps.incremental.storage.OneToManyPathMapping; +import org.jetbrains.jps.incremental.storage.BuildDataManager; import org.jetbrains.jps.model.JpsProject; import org.jetbrains.jps.model.java.JpsJavaExtensionService; import org.jetbrains.jps.model.java.compiler.JpsCompilerExcludes; @@ -158,22 +158,24 @@ public final class FormsBindingManager extends FormsBuilder { formsToCompile.keySet().removeAll(alienForms); // form should be considered dirty if the class it is bound to is dirty - final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); + BuildDataManager dataManager = context.getProjectDescriptor().dataManager; for (Map.Entry entry : filesToCompile.entrySet()) { - final File srcFile = entry.getKey(); - final ModuleBuildTarget target = entry.getValue(); - final Collection boundForms = sourceToFormMap.getOutputs(srcFile.getPath()); - if (boundForms != null) { - for (String formPath : boundForms) { - final File formFile = new File(formPath); - if (!excludes.isExcluded(formFile) && formFile.exists()) { - addBinding(srcFile, formFile, srcToForms); - holderBuilder.markDirtyFile(target, formFile); + File srcFile = entry.getKey(); + ModuleBuildTarget target = entry.getValue(); + Collection boundForms = dataManager.getSourceToFormMap(target).getOutputs(srcFile.getPath()); + if (boundForms == null) { + continue; + } - context.getScope().markIndirectlyAffected(target, formFile); - formsToCompile.put(formFile, target); - exitCode = ExitCode.OK; - } + for (String formPath : boundForms) { + File formFile = new File(formPath); + if (!excludes.isExcluded(formFile) && formFile.exists()) { + addBinding(srcFile, formFile, srcToForms); + holderBuilder.markDirtyFile(target, formFile); + + context.getScope().markIndirectlyAffected(target, formFile); + formsToCompile.put(formFile, target); + exitCode = ExitCode.OK; } } } diff --git a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java index 92c6e0a6d52f..274e390ad98f 100644 --- a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java +++ b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java @@ -86,10 +86,8 @@ public final class FormsInstrumenter extends FormsBuilder { final InstrumentationClassFinder finder = ClassProcessingBuilder.createInstrumentationClassFinder(sdk, platformCp, classpath, outputConsumer); try { - final Map> processed = instrumentForms(context, chunk, chunkSourcePath, finder, formsToCompile, outputConsumer, config.isUseDynamicBundles()); - - final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); - + Map> processed = instrumentForms(context, chunk, chunkSourcePath, finder, formsToCompile, outputConsumer, config.isUseDynamicBundles()); + OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(chunk.representativeTarget()); for (Map.Entry> entry : processed.entrySet()) { final File src = entry.getKey(); final Collection forms = entry.getValue();