diff --git a/build/src/org/jetbrains/intellij/build/CommunityStandaloneJpsBuilder.kt b/build/src/org/jetbrains/intellij/build/CommunityStandaloneJpsBuilder.kt index e442803b9e67..58702006a704 100644 --- a/build/src/org/jetbrains/intellij/build/CommunityStandaloneJpsBuilder.kt +++ b/build/src/org/jetbrains/intellij/build/CommunityStandaloneJpsBuilder.kt @@ -102,6 +102,7 @@ suspend fun buildCommunityStandaloneJpsBuilder(targetDir: Path, "netty-buffer", "aalto-xml", "caffeine", + "mvstore", "jetbrains.kotlinx.metadata.jvm", "hash4j" )) { diff --git a/jps/jps-builders/intellij.platform.jps.build.iml b/jps/jps-builders/intellij.platform.jps.build.iml index e4ce4fe06256..3d4f52f21c56 100644 --- a/jps/jps-builders/intellij.platform.jps.build.iml +++ b/jps/jps-builders/intellij.platform.jps.build.iml @@ -73,5 +73,6 @@ + \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java index 4725c4f84a29..fc847a248797 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/BuildRunner.java @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.cmdline; import com.intellij.openapi.diagnostic.Logger; @@ -24,10 +24,7 @@ import org.jetbrains.jps.incremental.fs.BuildFSState; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; -import org.jetbrains.jps.incremental.storage.BuildDataManager; -import org.jetbrains.jps.incremental.storage.BuildTargetsState; -import org.jetbrains.jps.incremental.storage.ProjectStamps; -import org.jetbrains.jps.incremental.storage.StampsStorage; +import org.jetbrains.jps.incremental.storage.*; import org.jetbrains.jps.indices.ModuleExcludeIndex; import org.jetbrains.jps.indices.impl.IgnoredFileIndexImpl; import org.jetbrains.jps.indices.impl.ModuleExcludeIndexImpl; @@ -43,6 +40,8 @@ import static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage import static org.jetbrains.jps.backwardRefs.JavaBackwardReferenceIndexWriter.isCompilerReferenceFSCaseSensitive; public final class BuildRunner { + private static final boolean USE_EXPERIMENTAL_STORAGE = Boolean.getBoolean("jps.use.experimental.storage"); + private static final Logger LOG = Logger.getInstance(BuildRunner.class); private final JpsModelLoader myModelLoader; private List myFilePaths = Collections.emptyList(); @@ -73,6 +72,22 @@ public final class BuildRunner { return load(msgHandler, dataStorageRoot.toPath(), fsState); } + private static @NotNull ProjectStamps initProjectStampStorage(@NotNull Path dataStorageRoot, + @NotNull PathRelativizerService relativizer, + @NotNull BuildTargetsState targetsState, + @Nullable StorageManager storageManager) + throws IOException { + if (ProjectStamps.PORTABLE_CACHES) { + // allow compaction on close (not more than 10 seconds) to ensure minimal storage size + assert storageManager != null; + HashStampStorage stampStorage = new HashStampStorage(storageManager, relativizer, targetsState); + return new ProjectStamps(stampStorage); + } + else { + return new ProjectStamps(dataStorageRoot, targetsState); + } + } + public ProjectDescriptor load(@NotNull MessageHandler msgHandler, @NotNull Path dataStorageRoot, @NotNull BuildFSState fsState) throws IOException { final JpsModel jpsModel = myModelLoader.loadModel(); BuildDataPaths dataPaths = new BuildDataPathsImpl(dataStorageRoot.toFile()); @@ -87,9 +102,13 @@ public final class BuildRunner { ProjectStamps projectStamps = null; BuildDataManager dataManager = null; + StorageManager storageManager = USE_EXPERIMENTAL_STORAGE || ProjectStamps.PORTABLE_CACHES + ? new StorageManager(dataStorageRoot.resolve("jps-portable-cache.db"), 10_000) + : null; try { - projectStamps = new ProjectStamps(dataStorageRoot, targetsState, relativizer); - dataManager = new BuildDataManager(dataPaths, targetsState, relativizer); + projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager); + + dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager); if (dataManager.versionDiffers()) { myForceCleanCaches = true; msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO, @@ -99,6 +118,11 @@ public final class BuildRunner { catch (Exception e) { // second try LOG.info(e); + + if (storageManager != null) { + storageManager.forceClose(); + } + if (projectStamps != null) { projectStamps.close(); } @@ -108,9 +132,9 @@ public final class BuildRunner { myForceCleanCaches = true; NioFiles.deleteRecursively(dataStorageRoot); targetsState = new BuildTargetsState(dataPaths, jpsModel, buildRootIndex); - projectStamps = new ProjectStamps(dataStorageRoot, targetsState, relativizer); - dataManager = new BuildDataManager(dataPaths, targetsState, relativizer); - // second attempt succeeded + projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager); + dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager); + // the second attempt succeeded msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO, JpsBuildBundle.message("build.message.project.rebuild.forced.0", e.getMessage()))); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/cmdline/ClasspathBootstrap.java b/jps/jps-builders/src/org/jetbrains/jps/cmdline/ClasspathBootstrap.java index 5c41d29ff29a..7509265b2b12 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/cmdline/ClasspathBootstrap.java +++ b/jps/jps-builders/src/org/jetbrains/jps/cmdline/ClasspathBootstrap.java @@ -26,6 +26,7 @@ import io.netty.resolver.AddressResolverGroup; import io.netty.util.NetUtil; import kotlinx.metadata.jvm.JvmMetadataUtil; import net.n3.nanoxml.IXMLBuilder; +import org.h2.mvstore.MVStore; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -134,6 +135,7 @@ public final class ClasspathBootstrap { addToClassPath(Caffeine.class, cp); // Hashing addToClassPath(Hashing.class, cp); + addToClassPath(MVStore.class, cp); addToClassPath(cp, ArtifactRepositoryManager.getClassesFromDependencies()); addToClassPath(Tracer.class, cp); // tracing infrastructure diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 30f5212b6d5c..8d57b3fa9b5b 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -17,7 +17,6 @@ import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.jps.ModuleChunk; -import org.jetbrains.jps.model.serialization.impl.TimingLog; import org.jetbrains.jps.api.BuildParametersKeys; import org.jetbrains.jps.api.CanceledStatus; import org.jetbrains.jps.api.GlobalOptions; @@ -44,6 +43,7 @@ import org.jetbrains.jps.javac.JavacMain; import org.jetbrains.jps.model.java.JpsJavaExtensionService; import org.jetbrains.jps.model.java.compiler.JpsJavaCompilerConfiguration; import org.jetbrains.jps.model.module.JpsModule; +import org.jetbrains.jps.model.serialization.impl.TimingLog; import org.jetbrains.jps.service.SharedThreadPool; import org.jetbrains.jps.util.JpsPathUtil; @@ -1506,8 +1506,8 @@ public final class IncProjectBuilder { } if (target instanceof ModuleBuildTarget) { - // check if deleted source was associated with a form - final OneToManyPathsMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); + // check if the deleted source was associated with a form + final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); final Collection boundForms = sourceToFormMap.getState(deletedSource); if (boundForms != null) { for (String formPath : boundForms) { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index a737da391bdc..521848932072 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -52,7 +52,7 @@ public final class BuildDataManager { private static final String MAPPINGS_STORAGE = "mappings"; private static final String SRC_TO_OUTPUT_FILE_NAME = "data"; private final ConcurrentMap, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>(16, 0.75f, getConcurrencyLevel()); - private final OneToManyPathsMapping mySrcToFormMap; + private final OneToManyPathMapping mySrcToFormMap; private final Mappings myMappings; private final Object myGraphManagementLock = new Object(); private DependencyGraph myDepGraph; @@ -76,11 +76,19 @@ public final class BuildDataManager { } }; - public BuildDataManager(BuildDataPaths dataPaths, BuildTargetsState targetsState, PathRelativizerService relativizer) throws IOException { + public BuildDataManager(BuildDataPaths dataPaths, + BuildTargetsState targetsState, + PathRelativizerService relativizer, + @Nullable StorageManager storageManager) throws IOException { myDataPaths = dataPaths; myTargetsState = targetsState; try { - mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer); + if (storageManager == null) { + mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer); + } + else { + mySrcToFormMap = new ExperimentalOneToManyPathMapping("source-to-form", storageManager, relativizer); + } myOutputToTargetRegistry = new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer); File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot()); if (JavaBuilderUtil.isDepGraphEnabled()) { @@ -128,8 +136,8 @@ public final class BuildDataManager { return myOutputToTargetRegistry; } - public SourceToOutputMapping getSourceToOutputMap(final BuildTarget target) throws IOException { - final SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER); + public SourceToOutputMapping getSourceToOutputMap(BuildTarget target) throws IOException { + SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER); return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target)); } @@ -143,7 +151,7 @@ public final class BuildDataManager { return targetStorages.getOrCreateStorage(provider, myRelativizer); } - public OneToManyPathsMapping getSourceToFormMap() { + public OneToManyPathMapping getSourceToFormMap() { return mySrcToFormMap; } @@ -370,10 +378,14 @@ public final class BuildDataManager { return new File(dataStorageRoot, forDepGraph? MAPPINGS_STORAGE + "-graph" : MAPPINGS_STORAGE); } - private static void wipeStorage(@NotNull Path root, @Nullable AbstractStateStorage storage) { + private static void wipeStorage(@NotNull Path root, @Nullable StorageOwner storage) { if (storage != null) { synchronized (storage) { - storage.wipe(); + try { + storage.clean(); + } + catch (IOException ignore) { + } } } else { @@ -388,7 +400,7 @@ public final class BuildDataManager { } } - private static void closeStorage(@Nullable AbstractStateStorage storage) throws IOException { + private static void closeStorage(@Nullable StorageOwner storage) throws IOException { if (storage != null) { synchronized (storage) { storage.close(); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt new file mode 100644 index 000000000000..ca0a47de2b09 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt @@ -0,0 +1,93 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.jps.incremental.storage + +import com.dynatrace.hash4j.hashing.Hashing +import org.h2.mvstore.type.DataType +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType +import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType +import java.io.IOException +import kotlin.Throws + +internal interface OneToManyPathMapping : StorageOwner { + @Throws(IOException::class) + fun getState(path: String): Collection? + + @Throws(IOException::class) + fun update(path: String, outPaths: List) + + @Throws(IOException::class) + fun remove(path: String) +} + +internal class ExperimentalOneToManyPathMapping( + mapName: String, + storageManager: StorageManager, + private val relativizer: PathRelativizerService, +) : OneToManyPathMapping, + StorageOwnerByMap>( + mapName = mapName, + storageManager = storageManager, + keyType = LongPairKeyDataType, + valueType = StringListDataType, + ) { + + private fun getKey(path: String): LongArray { + val stringKey = relativizer.toRelative(path).toByteArray() + return longArrayOf(Hashing.xxh3_64().hashBytesToLong(stringKey), Hashing.komihash5_0().hashBytesToLong(stringKey)) + } + + @Suppress("ReplaceGetOrSet") + override fun getState(path: String): Collection? { + val key = getKey(path) + val list = mapHandle.map.get(key) ?: return null + return Array(list.size) { relativizer.toFull(list.get(it)) }.asList() + } + + override fun update(path: String, outPaths: List) { + val key = getKey(path) + if (outPaths.isEmpty()) { + mapHandle.map.remove(key) + } + else { + val listWithRelativePaths = Array(outPaths.size) { + relativizer.toRelative(outPaths.get(it)) + } + mapHandle.map.put(key, listWithRelativePaths) + } + } + + override fun remove(path: String) { + mapHandle.map.remove(getKey(path)) + } +} + +internal sealed class StorageOwnerByMap( + mapName: String, + storageManager: StorageManager, + keyType: DataType, + valueType: DataType, +) : StorageOwner { + @JvmField + protected val mapHandle = storageManager.openMap(mapName, keyType, valueType) + + final override fun flush(memoryCachesOnly: Boolean) { + if (memoryCachesOnly) { + // set again to force to clear the cache (in kb) + mapHandle.map.store.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024 + } + else { + mapHandle.tryCommit() + } + } + + final override fun clean() { + mapHandle.map.clear() + } + + final override fun close() { + mapHandle.release() + } +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java index eda68bf24f33..d0ac92100e76 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/FileTimestampStorage.java @@ -22,7 +22,7 @@ final class FileTimestampStorage extends AbstractStateStorage>( - PersistentMapBuilder.newBuilder(getStorageRoot(dataStorageRoot).resolve("data"), JpsCachePathStringDescriptor, StateExternalizer) - .withVersion(2), - false, -), StampsStorage { - private val fileStampRoot = getStorageRoot(dataStorageRoot) +) : StampsStorage { + private val mapHandle = storageManager.openMap("file-hash-and-mtime-v1", HashStampStorageKeyType, HashStampStorageValueType) - override fun getStorageRoot(): Path = fileStampRoot - - override fun saveStamp(file: Path, buildTarget: BuildTarget<*>, stamp: HashStamp) { - val targetId = targetState.getBuildTargetId(buildTarget) - val path = relativizer.toRelative(file) - update(path, updateFilesStamp(oldState = getState(path), targetId = targetId, stamp = stamp)) + override fun force() { + mapHandle.tryCommit() } - override fun removeStamp(file: Path, buildTarget: BuildTarget<*>) { - val path = relativizer.toRelative(file) - val state = getState(path) ?: return - val targetId = targetState.getBuildTargetId(buildTarget) - for (i in state.indices) { - if (state[i].targetId == targetId) { - if (state.size == 1) { - remove(path) - } - else { - val newState = ArrayUtil.remove(state, i) - update(path, newState) - break - } - } - } + override fun clean() { + mapHandle.map.clear() + } + + override fun close() { + mapHandle.release() + } + + override fun getStorageRoot(): Path = storageManager.file + + override fun saveStamp(file: Path, buildTarget: BuildTarget<*>, stamp: HashStamp) { + mapHandle.map.put(createKey(buildTarget, file), stamp) + } + + private fun createKey(target: BuildTarget<*>, file: Path): HashStampStorageKey { + return HashStampStorageKey( + targetId = targetState.getBuildTargetId(target), + // getBytes is faster (70k op/s vs. 50 op/s) + // use xxh3_64 as it is more proven hash algo than komihash + pathHash = Hashing.xxh3_64().hashBytesToLong(relativizer.toRelative(file).toByteArray()), + ) + } + + override fun removeStamp(file: Path, target: BuildTarget<*>) { + mapHandle.map.remove(createKey(target, file)) } override fun getPreviousStamp(file: Path, target: BuildTarget<*>): HashStamp? { - val state = getState(relativizer.toRelative(file)) ?: return null - val targetId = targetState.getBuildTargetId(target) - return state - .firstOrNull { it.targetId == targetId } - ?.let { HashStamp(hash = it.hash, timestamp = it.timestamp) } + return mapHandle.map.get(createKey(target, file)) } fun getStoredFileHash(file: Path, target: BuildTarget<*>): Long? { - val state = getState(relativizer.toRelative(file)) ?: return null - val targetId = targetState.getBuildTargetId(target) - return state.firstOrNull { it.targetId == targetId }?.hash + return mapHandle.map.get(createKey(target, file))?.hash } override fun getCurrentStamp(file: Path): HashStamp { @@ -94,47 +90,109 @@ internal class HashStampStorage( } } -internal class HashStampPerTarget(@JvmField val targetId: Int, @JvmField val hash: Long, @JvmField val timestamp: Long) +internal class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp -internal data class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp +private class HashStampStorageKey(@JvmField val targetId: Int, @JvmField val pathHash: Long) -private fun getStorageRoot(dataStorageRoot: Path): Path = dataStorageRoot.resolve("hashes") +private object HashStampStorageKeyType : DataType { + override fun isMemoryEstimationAllowed() = true -private fun updateFilesStamp(oldState: Array?, targetId: Int, stamp: HashStamp): Array { - val newItem = HashStampPerTarget(targetId = targetId, hash = stamp.hash, timestamp = stamp.timestamp) - if (oldState == null) { - return arrayOf(newItem) - } + override fun getMemory(obj: HashStampStorageKey): Int = Int.SIZE_BYTES + Long.SIZE_BYTES - var i = 0 - val length = oldState.size - while (i < length) { - if (oldState[i].targetId == targetId) { - oldState[i] = newItem - return oldState + override fun createStorage(size: Int): Array = arrayOfNulls(size) + + override fun write(buff: WriteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + for (key in (storage as Array)) { + buff.putVarInt(key.targetId) + // not var long - maybe negative number + buff.putLong(key.pathHash) + } + } + + override fun write(buff: WriteBuffer, obj: HashStampStorageKey) = throw IllegalStateException("Must not be called") + + override fun read(buff: ByteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + storage as Array + for (i in 0 until len) { + storage[i] = HashStampStorageKey(targetId = readVarInt(buff), pathHash = buff.getLong()) + } + } + + override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called") + + override fun binarySearch(key: HashStampStorageKey, storage: Any, size: Int, initialGuess: Int): Int { + @Suppress("UNCHECKED_CAST") + storage as Array + + var low = 0 + var high = size - 1 + // the cached index minus one, so that for the first time (when cachedCompare is 0), the default value is used + var x = initialGuess - 1 + if (x < 0 || x > high) { + x = high ushr 1 + } + while (low <= high) { + val b = storage[x] + val compare = when { + key.targetId > b.targetId -> 1 + key.targetId < b.targetId -> -1 + key.pathHash > b.pathHash -> 1 + key.pathHash < b.pathHash -> -1 + else -> 0 + } + + when { + compare > 0 -> low = x + 1 + compare < 0 -> high = x - 1 + else -> return x + } + x = (low + high) ushr 1 + } + return low.inv() + } + + @Suppress("DuplicatedCode") + override fun compare(a: HashStampStorageKey, b: HashStampStorageKey): Int { + return when { + a.targetId > b.targetId -> 1 + a.targetId < b.targetId -> -1 + a.pathHash > b.pathHash -> 1 + a.pathHash < b.pathHash -> -1 + else -> 0 } - i++ } - return oldState + newItem } -private object StateExternalizer : DataExternalizer> { - override fun save(out: DataOutput, value: Array) { - out.writeInt(value.size) - for (target in value) { - out.writeInt(target.targetId) - out.writeLong(target.hash) - out.writeLong(target.timestamp) +private object HashStampStorageValueType : DataType { + override fun isMemoryEstimationAllowed() = true + + override fun getMemory(obj: HashStamp): Int = Long.SIZE_BYTES + Long.SIZE_BYTES + + override fun createStorage(size: Int): Array = arrayOfNulls(size) + + override fun write(buff: WriteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + for (value in (storage as Array)) { + buff.putLong(value.hash) + buff.putVarLong(value.timestamp) } } - override fun read(`in`: DataInput): Array { - val size = `in`.readInt() - return Array(size) { - val id = `in`.readInt() - val hash = `in`.readLong() - val timestamp = `in`.readLong() - HashStampPerTarget(targetId = id, hash = hash, timestamp = timestamp) + override fun write(buff: WriteBuffer, obj: HashStamp) = throw IllegalStateException("Must not be called") + + override fun read(buff: ByteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + storage as Array + for (i in 0 until len) { + storage[i] = HashStamp(hash = buff.getLong(), timestamp = readVarLong(buff)) } } + + override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called") + + override fun compare(a: HashStamp, b: HashStamp) = throw IllegalStateException("Must not be called") + + override fun binarySearch(key: HashStamp?, storage: Any?, size: Int, initialGuess: Int) = throw IllegalStateException("Must not be called") } \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java index 2efdae2021c0..43f422c99327 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java @@ -20,7 +20,7 @@ import java.util.*; /** * @author Eugene Zhuravlev */ -public final class OneToManyPathsMapping extends AbstractStateStorage> { +public final class OneToManyPathsMapping extends AbstractStateStorage> implements OneToManyPathMapping { private final PathRelativizerService relativizer; public OneToManyPathsMapping(@NotNull Path storePath, PathRelativizerService relativizer) throws IOException { @@ -29,8 +29,8 @@ public final class OneToManyPathsMapping extends AbstractStateStorage boundPaths) throws IOException { - super.update(normalizePath(keyPath), normalizePaths((List)boundPaths)); + public void update(@NotNull String keyPath, @SuppressWarnings("NullableProblems") @NotNull List boundPaths) throws IOException { + super.update(normalizePath(keyPath), normalizePaths(boundPaths)); } public void update(@NotNull String keyPath, @NotNull String boundPath) throws IOException { diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java index 4d7e73712ef4..a2a454db0c62 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ProjectStamps.java @@ -3,6 +3,8 @@ package org.jetbrains.jps.incremental.storage; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.io.NioFiles; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; import java.io.File; @@ -23,31 +25,28 @@ public final class ProjectStamps { private final StampsStorage stampStorage; - public ProjectStamps(Path dataStorageRoot, BuildTargetsState targetsState, PathRelativizerService relativizer) throws IOException { - if (PORTABLE_CACHES) { - stampStorage = new HashStampStorage(dataStorageRoot, relativizer, targetsState); - } - else { - stampStorage = new FileTimestampStorage(dataStorageRoot, targetsState); - } + @ApiStatus.Internal + public ProjectStamps(@NotNull StampsStorage stampStorage) throws IOException { + this.stampStorage = stampStorage; + } + + public ProjectStamps(@NotNull Path dataStorageRoot, @NotNull BuildTargetsState targetsState) throws IOException { + this(new FileTimestampStorage(dataStorageRoot, targetsState)); } /** - * @deprecated Please use {@link #ProjectStamps(Path, BuildTargetsState, PathRelativizerService)} + * @deprecated Please use {@link #ProjectStamps(Path, BuildTargetsState)} */ + @SuppressWarnings("unused") @Deprecated public ProjectStamps(File dataStorageRoot, BuildTargetsState targetsState, PathRelativizerService relativizer) throws IOException { - this(dataStorageRoot.toPath(), targetsState, relativizer); + this(dataStorageRoot.toPath(), targetsState); } - public StampsStorage getStampStorage() { + public @NotNull StampsStorage getStampStorage() { return stampStorage; } - public void clean() { - stampStorage.wipe(); - } - public void close() { try { stampStorage.close(); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java index ffd13bdeb008..7661ecb19e87 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java @@ -16,7 +16,7 @@ import java.util.Iterator; public final class SourceToOutputMappingImpl implements SourceToOutputMapping, StorageOwner { private final OneToManyPathsMapping myMapping; - public SourceToOutputMappingImpl(@NotNull Path storePath, PathRelativizerService relativizer) throws IOException { + public SourceToOutputMappingImpl(@NotNull Path storePath, @NotNull PathRelativizerService relativizer) throws IOException { myMapping = new OneToManyPathsMapping(storePath, relativizer); } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java index 9c73c8cdaf5f..c85625d8306d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StampsStorage.java @@ -31,8 +31,6 @@ public interface StampsStorage { @NotNull T getCurrentStamp(@NotNull Path file) throws IOException; - boolean wipe(); - void close() throws IOException; boolean isDirtyStamp(@NotNull Stamp stamp, @NotNull Path file) throws IOException; diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt new file mode 100644 index 000000000000..c328057f61ce --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/StorageManager.kt @@ -0,0 +1,182 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage + +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.diagnostic.thisLogger +import org.h2.mvstore.MVMap +import org.h2.mvstore.MVStore +import org.h2.mvstore.type.DataType +import org.jetbrains.annotations.ApiStatus +import java.nio.file.Files +import java.nio.file.Path + +internal const val MV_STORE_CACHE_SIZE_IN_MB = 32 + +@ApiStatus.Internal +class StorageManager(@JvmField val file: Path, allowedCompactionTimeOnClose: Int) { + private val storeValue = StoreValue(file = file, allowedCompactionTimeOnClose = allowedCompactionTimeOnClose) + + fun openMap(name: String, keyType: DataType, valueType: DataType): MapHandle { + val mapBuilder = MVMap.Builder() + mapBuilder.setKeyType(keyType) + mapBuilder.setValueType(valueType) + return openMap(name, mapBuilder) + } + + fun openMap(name: String, mapBuilder: MVMap.Builder): MapHandle { + val store = storeValue.openStore() + return MapHandle(storeValue, openOrResetMap(store = store, name = name, mapBuilder = mapBuilder, logSupplier = ::thisLogger)) + } + + /** Only if error occurred and you release all [MapHandle]s */ + fun forceClose() { + storeValue.forceClose() + } +} + +internal class StoreValue(private val file: Path, private val allowedCompactionTimeOnClose: Int) { + private var refCount = 0 + private var store: MVStore? = null + + @Synchronized + fun forceClose() { + refCount = 0 + store?.let { + store = null + it.closeImmediately() + } + } + + @Synchronized + fun openStore(): MVStore { + if (refCount == 0) { + require(store == null) + val store = createOrResetMvStore(file = file, readOnly = false, ::thisLogger) + this.store = store + refCount++ + return store + } + + refCount++ + return store!! + } + + @Synchronized + fun release() { + when (refCount) { + 1 -> { + store!!.close(allowedCompactionTimeOnClose) + store = null + refCount = 0 + } + 0 -> throw IllegalStateException("Store is already closed") + else -> refCount-- + } + } +} + +@ApiStatus.Internal +class MapHandle internal constructor( + private val storeValue: StoreValue, + @JvmField val map: MVMap, +) { + @Volatile + private var isReleased = false + + fun release() { + if (!isReleased) { + storeValue.release() + isReleased = true + } + } + + fun tryCommit() { + require(!isReleased) + map.store.tryCommit() + } +} + +private fun openOrResetMap( + store: MVStore, + name: String, + mapBuilder: MVMap.Builder, + logSupplier: () -> Logger, +): MVMap { + try { + return store.openMap(name, mapBuilder) + } + catch (e: Throwable) { + logSupplier().error("Cannot open map $name, map will be removed", e) + try { + store.removeMap(name) + } + catch (e2: Throwable) { + e.addSuppressed(e2) + } + } + return store.openMap(name, mapBuilder) +} + +private fun createOrResetMvStore( + file: Path?, + @Suppress("SameParameterValue") readOnly: Boolean = false, + logSupplier: () -> Logger, +): MVStore { + // If read-only and DB does not yet exist, create an in-memory DB + if (file == null || (readOnly && Files.notExists(file))) { + // in-memory + return tryOpenMvStore(file = null, readOnly = readOnly, logSupplier = logSupplier) + } + + val markerFile = getInvalidateMarkerFile(file) + if (Files.exists(markerFile)) { + Files.deleteIfExists(file) + Files.deleteIfExists(markerFile) + } + + file.parent?.let { Files.createDirectories(it) } + try { + return tryOpenMvStore(file, readOnly, logSupplier) + } + catch (e: Throwable) { + logSupplier().warn("Cannot open cache state storage, will be recreated", e) + } + + Files.deleteIfExists(file) + return tryOpenMvStore(file, readOnly, logSupplier) +} + +private fun getInvalidateMarkerFile(file: Path): Path = file.resolveSibling("${file.fileName}.invalidated") + +private fun tryOpenMvStore(file: Path?, readOnly: Boolean, logSupplier: () -> Logger): MVStore { + val storeErrorHandler = StoreErrorHandler(file, logSupplier) + val store = MVStore.Builder() + .fileName(file?.toAbsolutePath()?.toString()) + .backgroundExceptionHandler(storeErrorHandler) + // avoid extra thread - db maintainer should use coroutines + .autoCommitDisabled() + .cacheSize(MV_STORE_CACHE_SIZE_IN_MB) + .let { + if (readOnly) it.readOnly() else it + } + .open() + storeErrorHandler.isStoreOpened = true + // versioning isn't required, otherwise the file size will be larger than needed + store.setVersionsToKeep(0) + return store +} + +private class StoreErrorHandler(private val dbFile: Path?, private val logSupplier: () -> Logger) : Thread.UncaughtExceptionHandler { + @JvmField + var isStoreOpened: Boolean = false + + override fun uncaughtException(t: Thread, e: Throwable) { + val log = logSupplier() + if (isStoreOpened) { + log.error("Store error (db=$dbFile)", e) + } + else { + log.warn("Store will be recreated (db=$dbFile)", e) + } + } +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt new file mode 100644 index 000000000000..a30e6f518bbe --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/LongPairKeyDataType.kt @@ -0,0 +1,77 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage.dataTypes + +import org.h2.mvstore.WriteBuffer +import org.h2.mvstore.type.DataType +import java.nio.ByteBuffer + +internal object LongPairKeyDataType : DataType { + override fun isMemoryEstimationAllowed() = true + + // don't care about non-ASCII strings for memory estimation + override fun getMemory(obj: LongArray): Int = 16 + + override fun createStorage(size: Int): Array = arrayOfNulls(size) + + override fun write(buff: WriteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + for (key in (storage as Array)) { + buff.putLong(key[0]) + buff.putLong(key[1]) + } + } + + override fun write(buff: WriteBuffer, obj: LongArray) = throw IllegalStateException("Must not be called") + + override fun read(buff: ByteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + storage as Array + for (i in 0 until len) { + storage[i] = longArrayOf(buff.getLong(), buff.getLong()) + } + } + + override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called") + + override fun binarySearch(key: LongArray, storage: Any, size: Int, initialGuess: Int): Int { + @Suppress("UNCHECKED_CAST") + storage as Array + + var low = 0 + var high = size - 1 + // the cached index minus one, so that for the first time (when cachedCompare is 0), the default value is used + var x = initialGuess - 1 + if (x < 0 || x > high) { + x = high ushr 1 + } + while (low <= high) { + val b = storage[x] + val compare = when { + key[0] > b[0] -> 1 + key[0] < b[0] -> -1 + key[1] > b[1] -> 1 + key[1] < b[1] -> -1 + else -> 0 + } + + when { + compare > 0 -> low = x + 1 + compare < 0 -> high = x - 1 + else -> return x + } + x = (low + high) ushr 1 + } + return low.inv() + } + + @Suppress("DuplicatedCode") + override fun compare(a: LongArray, b: LongArray): Int { + return when { + a[0] > b[0] -> 1 + a[0] < b[0] -> -1 + a[1] > b[1] -> 1 + a[1] < b[1] -> -1 + else -> 0 + } + } +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/StringListDataType.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/StringListDataType.kt new file mode 100644 index 000000000000..f11ca949273e --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/dataTypes/StringListDataType.kt @@ -0,0 +1,49 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage.dataTypes + +import org.h2.mvstore.DataUtils.readVarInt +import org.h2.mvstore.WriteBuffer +import org.h2.mvstore.type.DataType + +import java.nio.ByteBuffer + +internal object StringListDataType : DataType> { + override fun isMemoryEstimationAllowed() = true + + // don't care about non-ASCII for size computation - non-ASCII strings should be quite rare + override fun getMemory(obj: Array): Int = Int.SIZE_BYTES + obj.sumOf { it.length } + + override fun createStorage(size: Int): Array?> = arrayOfNulls(size) + + override fun write(buff: WriteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + storage as Array> + for (l in storage) { + buff.putVarInt(l.size) + for (s in l) { + val bytes = s.toByteArray() + buff.putVarInt(bytes.size).put(bytes) + } + } + } + + override fun write(buff: WriteBuffer, obj: Array) = throw IllegalStateException("Must not be called") + + override fun read(buff: ByteBuffer, storage: Any, len: Int) { + @Suppress("UNCHECKED_CAST") + storage as Array> + for (i in 0 until len) { + storage[i] = Array(readVarInt(buff)) { + val bytes = ByteArray(readVarInt(buff)) + buff.get(bytes) + String(bytes) + } + } + } + + override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called") + + override fun compare(a: Array, b: Array) = throw IllegalStateException("Must not be called") + + override fun binarySearch(key: Array?, storage: Any?, size: Int, initialGuess: Int) = throw IllegalStateException("Must not be called") +} \ No newline at end of file diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java index 9afaf046c1ca..337c971aa29d 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/JpsBuildTestCase.java @@ -1,4 +1,4 @@ -// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.builders; import com.intellij.openapi.application.PathManager; @@ -200,8 +200,8 @@ public abstract class JpsBuildTestCase extends UsefulTestCase { BuildTargetIndexImpl targetIndex = new BuildTargetIndexImpl(targetRegistry, buildRootIndex); BuildTargetsState targetsState = new BuildTargetsState(dataPaths, myModel, buildRootIndex); PathRelativizerService relativizer = new PathRelativizerService(myModel.getProject()); - ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState, relativizer); - BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer); + ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState); + BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, null); return new ProjectDescriptor(myModel, new BuildFSState(true), projectStamps, dataManager, buildLoggingManager, index, targetIndex, buildRootIndex, ignoredFileIndex); } diff --git a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java index c50d3ed8c941..6269fd4d47f1 100644 --- a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java +++ b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java @@ -1,4 +1,4 @@ -// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.uiDesigner.compiler; import com.intellij.openapi.util.Key; @@ -22,7 +22,7 @@ import org.jetbrains.jps.incremental.java.CopyResourcesUtil; import org.jetbrains.jps.incremental.java.FormsParsing; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; -import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping; +import org.jetbrains.jps.incremental.storage.OneToManyPathMapping; import org.jetbrains.jps.model.JpsProject; import org.jetbrains.jps.model.java.JpsJavaExtensionService; import org.jetbrains.jps.model.java.compiler.JpsCompilerExcludes; @@ -158,7 +158,7 @@ public final class FormsBindingManager extends FormsBuilder { formsToCompile.keySet().removeAll(alienForms); // form should be considered dirty if the class it is bound to is dirty - final OneToManyPathsMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); + final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); for (Map.Entry entry : filesToCompile.entrySet()) { final File srcFile = entry.getKey(); final ModuleBuildTarget target = entry.getValue(); diff --git a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java index 7de24dec4b0d..0c3e7bbbf032 100644 --- a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java +++ b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java @@ -1,4 +1,4 @@ -// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.uiDesigner.compiler; import com.intellij.compiler.instrumentation.FailSafeClassReader; @@ -26,7 +26,7 @@ import org.jetbrains.jps.incremental.instrumentation.ClassProcessingBuilder; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; import org.jetbrains.jps.incremental.messages.ProgressMessage; -import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping; +import org.jetbrains.jps.incremental.storage.OneToManyPathMapping; import org.jetbrains.jps.model.JpsDummyElement; import org.jetbrains.jps.model.JpsProject; import org.jetbrains.jps.model.java.JpsJavaSdkType; @@ -88,13 +88,13 @@ public final class FormsInstrumenter extends FormsBuilder { try { final Map> processed = instrumentForms(context, chunk, chunkSourcePath, finder, formsToCompile, outputConsumer, config.isUseDynamicBundles()); - final OneToManyPathsMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); + final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); for (Map.Entry> entry : processed.entrySet()) { final File src = entry.getKey(); final Collection forms = entry.getValue(); - final Collection formPaths = new ArrayList<>(forms.size()); + List formPaths = new ArrayList<>(forms.size()); for (File form : forms) { formPaths.add(form.getPath()); }