experimental compact storage for JPS Cache (part 1 - HashStampStorage and ExperimentalOneToManyPathMapping)

JPS Cache - don't use PHM to store hash-and-mtime stamp (80MB -> 10MB), use composite key to avoid value as an array, use 64-bit hash of a path instead of using string as is for a key

GitOrigin-RevId: 20936753d1ba58fcf5f07c4d859e02491e9a9cc5
This commit is contained in:
Vladimir Krivosheev
2024-09-12 14:33:18 +02:00
committed by intellij-monorepo-bot
parent 58ba64e6e5
commit ff1b118f42
19 changed files with 621 additions and 125 deletions

View File

@@ -102,6 +102,7 @@ suspend fun buildCommunityStandaloneJpsBuilder(targetDir: Path,
"netty-buffer",
"aalto-xml",
"caffeine",
"mvstore",
"jetbrains.kotlinx.metadata.jvm",
"hash4j"
)) {

View File

@@ -73,5 +73,6 @@
<orderEntry type="library" scope="PROVIDED" name="kotlin-stdlib" level="project" />
<orderEntry type="library" name="hash4j" level="project" />
<orderEntry type="library" name="netty-codec-protobuf" level="project" />
<orderEntry type="library" name="mvstore" level="project" />
</component>
</module>

View File

@@ -1,4 +1,4 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.cmdline;
import com.intellij.openapi.diagnostic.Logger;
@@ -24,10 +24,7 @@ import org.jetbrains.jps.incremental.fs.BuildFSState;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
import org.jetbrains.jps.incremental.storage.BuildDataManager;
import org.jetbrains.jps.incremental.storage.BuildTargetsState;
import org.jetbrains.jps.incremental.storage.ProjectStamps;
import org.jetbrains.jps.incremental.storage.StampsStorage;
import org.jetbrains.jps.incremental.storage.*;
import org.jetbrains.jps.indices.ModuleExcludeIndex;
import org.jetbrains.jps.indices.impl.IgnoredFileIndexImpl;
import org.jetbrains.jps.indices.impl.ModuleExcludeIndexImpl;
@@ -43,6 +40,8 @@ import static org.jetbrains.jps.api.CmdlineRemoteProto.Message.ControllerMessage
import static org.jetbrains.jps.backwardRefs.JavaBackwardReferenceIndexWriter.isCompilerReferenceFSCaseSensitive;
public final class BuildRunner {
private static final boolean USE_EXPERIMENTAL_STORAGE = Boolean.getBoolean("jps.use.experimental.storage");
private static final Logger LOG = Logger.getInstance(BuildRunner.class);
private final JpsModelLoader myModelLoader;
private List<String> myFilePaths = Collections.emptyList();
@@ -73,6 +72,22 @@ public final class BuildRunner {
return load(msgHandler, dataStorageRoot.toPath(), fsState);
}
private static @NotNull ProjectStamps initProjectStampStorage(@NotNull Path dataStorageRoot,
@NotNull PathRelativizerService relativizer,
@NotNull BuildTargetsState targetsState,
@Nullable StorageManager storageManager)
throws IOException {
if (ProjectStamps.PORTABLE_CACHES) {
// allow compaction on close (not more than 10 seconds) to ensure minimal storage size
assert storageManager != null;
HashStampStorage stampStorage = new HashStampStorage(storageManager, relativizer, targetsState);
return new ProjectStamps(stampStorage);
}
else {
return new ProjectStamps(dataStorageRoot, targetsState);
}
}
public ProjectDescriptor load(@NotNull MessageHandler msgHandler, @NotNull Path dataStorageRoot, @NotNull BuildFSState fsState) throws IOException {
final JpsModel jpsModel = myModelLoader.loadModel();
BuildDataPaths dataPaths = new BuildDataPathsImpl(dataStorageRoot.toFile());
@@ -87,9 +102,13 @@ public final class BuildRunner {
ProjectStamps projectStamps = null;
BuildDataManager dataManager = null;
StorageManager storageManager = USE_EXPERIMENTAL_STORAGE || ProjectStamps.PORTABLE_CACHES
? new StorageManager(dataStorageRoot.resolve("jps-portable-cache.db"), 10_000)
: null;
try {
projectStamps = new ProjectStamps(dataStorageRoot, targetsState, relativizer);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer);
projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager);
if (dataManager.versionDiffers()) {
myForceCleanCaches = true;
msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO,
@@ -99,6 +118,11 @@ public final class BuildRunner {
catch (Exception e) {
// second try
LOG.info(e);
if (storageManager != null) {
storageManager.forceClose();
}
if (projectStamps != null) {
projectStamps.close();
}
@@ -108,9 +132,9 @@ public final class BuildRunner {
myForceCleanCaches = true;
NioFiles.deleteRecursively(dataStorageRoot);
targetsState = new BuildTargetsState(dataPaths, jpsModel, buildRootIndex);
projectStamps = new ProjectStamps(dataStorageRoot, targetsState, relativizer);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer);
// second attempt succeeded
projectStamps = initProjectStampStorage(dataStorageRoot, relativizer, targetsState, storageManager);
dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, storageManager);
// the second attempt succeeded
msgHandler.processMessage(new CompilerMessage(getRootCompilerName(), BuildMessage.Kind.INFO,
JpsBuildBundle.message("build.message.project.rebuild.forced.0", e.getMessage())));
}

View File

@@ -26,6 +26,7 @@ import io.netty.resolver.AddressResolverGroup;
import io.netty.util.NetUtil;
import kotlinx.metadata.jvm.JvmMetadataUtil;
import net.n3.nanoxml.IXMLBuilder;
import org.h2.mvstore.MVStore;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -134,6 +135,7 @@ public final class ClasspathBootstrap {
addToClassPath(Caffeine.class, cp);
// Hashing
addToClassPath(Hashing.class, cp);
addToClassPath(MVStore.class, cp);
addToClassPath(cp, ArtifactRepositoryManager.getClassesFromDependencies());
addToClassPath(Tracer.class, cp); // tracing infrastructure

View File

@@ -17,7 +17,6 @@ import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.ModuleChunk;
import org.jetbrains.jps.model.serialization.impl.TimingLog;
import org.jetbrains.jps.api.BuildParametersKeys;
import org.jetbrains.jps.api.CanceledStatus;
import org.jetbrains.jps.api.GlobalOptions;
@@ -44,6 +43,7 @@ import org.jetbrains.jps.javac.JavacMain;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.java.compiler.JpsJavaCompilerConfiguration;
import org.jetbrains.jps.model.module.JpsModule;
import org.jetbrains.jps.model.serialization.impl.TimingLog;
import org.jetbrains.jps.service.SharedThreadPool;
import org.jetbrains.jps.util.JpsPathUtil;
@@ -1506,8 +1506,8 @@ public final class IncProjectBuilder {
}
if (target instanceof ModuleBuildTarget) {
// check if deleted source was associated with a form
final OneToManyPathsMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
// check if the deleted source was associated with a form
final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
final Collection<String> boundForms = sourceToFormMap.getState(deletedSource);
if (boundForms != null) {
for (String formPath : boundForms) {

View File

@@ -52,7 +52,7 @@ public final class BuildDataManager {
private static final String MAPPINGS_STORAGE = "mappings";
private static final String SRC_TO_OUTPUT_FILE_NAME = "data";
private final ConcurrentMap<BuildTarget<?>, BuildTargetStorages> myTargetStorages = new ConcurrentHashMap<>(16, 0.75f, getConcurrencyLevel());
private final OneToManyPathsMapping mySrcToFormMap;
private final OneToManyPathMapping mySrcToFormMap;
private final Mappings myMappings;
private final Object myGraphManagementLock = new Object();
private DependencyGraph myDepGraph;
@@ -76,11 +76,19 @@ public final class BuildDataManager {
}
};
public BuildDataManager(BuildDataPaths dataPaths, BuildTargetsState targetsState, PathRelativizerService relativizer) throws IOException {
public BuildDataManager(BuildDataPaths dataPaths,
BuildTargetsState targetsState,
PathRelativizerService relativizer,
@Nullable StorageManager storageManager) throws IOException {
myDataPaths = dataPaths;
myTargetsState = targetsState;
try {
mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer);
if (storageManager == null) {
mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer);
}
else {
mySrcToFormMap = new ExperimentalOneToManyPathMapping("source-to-form", storageManager, relativizer);
}
myOutputToTargetRegistry = new OutputToTargetRegistry(getOutputToSourceRegistryRoot().resolve("data"), relativizer);
File mappingsRoot = getMappingsRoot(myDataPaths.getDataStorageRoot());
if (JavaBuilderUtil.isDepGraphEnabled()) {
@@ -128,8 +136,8 @@ public final class BuildDataManager {
return myOutputToTargetRegistry;
}
public SourceToOutputMapping getSourceToOutputMap(final BuildTarget<?> target) throws IOException {
final SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER);
public SourceToOutputMapping getSourceToOutputMap(BuildTarget<?> target) throws IOException {
SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER);
return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target));
}
@@ -143,7 +151,7 @@ public final class BuildDataManager {
return targetStorages.getOrCreateStorage(provider, myRelativizer);
}
public OneToManyPathsMapping getSourceToFormMap() {
public OneToManyPathMapping getSourceToFormMap() {
return mySrcToFormMap;
}
@@ -370,10 +378,14 @@ public final class BuildDataManager {
return new File(dataStorageRoot, forDepGraph? MAPPINGS_STORAGE + "-graph" : MAPPINGS_STORAGE);
}
private static void wipeStorage(@NotNull Path root, @Nullable AbstractStateStorage<?, ?> storage) {
private static void wipeStorage(@NotNull Path root, @Nullable StorageOwner storage) {
if (storage != null) {
synchronized (storage) {
storage.wipe();
try {
storage.clean();
}
catch (IOException ignore) {
}
}
}
else {
@@ -388,7 +400,7 @@ public final class BuildDataManager {
}
}
private static void closeStorage(@Nullable AbstractStateStorage<?, ?> storage) throws IOException {
private static void closeStorage(@Nullable StorageOwner storage) throws IOException {
if (storage != null) {
synchronized (storage) {
storage.close();

View File

@@ -0,0 +1,93 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("ReplaceGetOrSet")
package org.jetbrains.jps.incremental.storage
import com.dynatrace.hash4j.hashing.Hashing
import org.h2.mvstore.type.DataType
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType
import java.io.IOException
import kotlin.Throws
internal interface OneToManyPathMapping : StorageOwner {
@Throws(IOException::class)
fun getState(path: String): Collection<String>?
@Throws(IOException::class)
fun update(path: String, outPaths: List<String>)
@Throws(IOException::class)
fun remove(path: String)
}
internal class ExperimentalOneToManyPathMapping(
mapName: String,
storageManager: StorageManager,
private val relativizer: PathRelativizerService,
) : OneToManyPathMapping,
StorageOwnerByMap<LongArray, Array<String>>(
mapName = mapName,
storageManager = storageManager,
keyType = LongPairKeyDataType,
valueType = StringListDataType,
) {
private fun getKey(path: String): LongArray {
val stringKey = relativizer.toRelative(path).toByteArray()
return longArrayOf(Hashing.xxh3_64().hashBytesToLong(stringKey), Hashing.komihash5_0().hashBytesToLong(stringKey))
}
@Suppress("ReplaceGetOrSet")
override fun getState(path: String): Collection<String>? {
val key = getKey(path)
val list = mapHandle.map.get(key) ?: return null
return Array<String>(list.size) { relativizer.toFull(list.get(it)) }.asList()
}
override fun update(path: String, outPaths: List<String>) {
val key = getKey(path)
if (outPaths.isEmpty()) {
mapHandle.map.remove(key)
}
else {
val listWithRelativePaths = Array(outPaths.size) {
relativizer.toRelative(outPaths.get(it))
}
mapHandle.map.put(key, listWithRelativePaths)
}
}
override fun remove(path: String) {
mapHandle.map.remove(getKey(path))
}
}
internal sealed class StorageOwnerByMap<K : Any, V : Any>(
mapName: String,
storageManager: StorageManager,
keyType: DataType<K>,
valueType: DataType<V>,
) : StorageOwner {
@JvmField
protected val mapHandle = storageManager.openMap(mapName, keyType, valueType)
final override fun flush(memoryCachesOnly: Boolean) {
if (memoryCachesOnly) {
// set again to force to clear the cache (in kb)
mapHandle.map.store.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024
}
else {
mapHandle.tryCommit()
}
}
final override fun clean() {
mapHandle.map.clear()
}
final override fun close() {
mapHandle.release()
}
}

View File

@@ -22,7 +22,7 @@ final class FileTimestampStorage extends AbstractStateStorage<File, TimestampPer
private final BuildTargetsState myTargetsState;
private final Path timestampRoot;
FileTimestampStorage(Path dataStorageRoot, BuildTargetsState targetsState) throws IOException {
FileTimestampStorage(@NotNull Path dataStorageRoot, @NotNull BuildTargetsState targetsState) throws IOException {
super(calcStorageRoot(dataStorageRoot).resolve("data").toFile(), new FileKeyDescriptor(), new StateExternalizer());
timestampRoot = calcStorageRoot(dataStorageRoot);
myTargetsState = targetsState;

View File

@@ -1,67 +1,63 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage
import com.intellij.util.ArrayUtil
import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.PersistentMapBuilder
import com.dynatrace.hash4j.hashing.Hashing
import org.h2.mvstore.DataUtils.readVarInt
import org.h2.mvstore.DataUtils.readVarLong
import org.h2.mvstore.WriteBuffer
import org.h2.mvstore.type.DataType
import org.jetbrains.jps.builders.BuildTarget
import org.jetbrains.jps.incremental.FSOperations
import org.jetbrains.jps.incremental.FileHashUtil
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
import java.io.DataInput
import java.io.DataOutput
import java.nio.ByteBuffer
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
internal class HashStampStorage(
dataStorageRoot: Path,
private val storageManager: StorageManager,
private val relativizer: PathRelativizerService,
private val targetState: BuildTargetsState,
) : AbstractStateStorage<String, Array<HashStampPerTarget>>(
PersistentMapBuilder.newBuilder(getStorageRoot(dataStorageRoot).resolve("data"), JpsCachePathStringDescriptor, StateExternalizer)
.withVersion(2),
false,
), StampsStorage<HashStamp> {
private val fileStampRoot = getStorageRoot(dataStorageRoot)
) : StampsStorage<HashStamp> {
private val mapHandle = storageManager.openMap("file-hash-and-mtime-v1", HashStampStorageKeyType, HashStampStorageValueType)
override fun getStorageRoot(): Path = fileStampRoot
override fun saveStamp(file: Path, buildTarget: BuildTarget<*>, stamp: HashStamp) {
val targetId = targetState.getBuildTargetId(buildTarget)
val path = relativizer.toRelative(file)
update(path, updateFilesStamp(oldState = getState(path), targetId = targetId, stamp = stamp))
override fun force() {
mapHandle.tryCommit()
}
override fun removeStamp(file: Path, buildTarget: BuildTarget<*>) {
val path = relativizer.toRelative(file)
val state = getState(path) ?: return
val targetId = targetState.getBuildTargetId(buildTarget)
for (i in state.indices) {
if (state[i].targetId == targetId) {
if (state.size == 1) {
remove(path)
}
else {
val newState = ArrayUtil.remove(state, i)
update(path, newState)
break
}
}
}
override fun clean() {
mapHandle.map.clear()
}
override fun close() {
mapHandle.release()
}
override fun getStorageRoot(): Path = storageManager.file
override fun saveStamp(file: Path, buildTarget: BuildTarget<*>, stamp: HashStamp) {
mapHandle.map.put(createKey(buildTarget, file), stamp)
}
private fun createKey(target: BuildTarget<*>, file: Path): HashStampStorageKey {
return HashStampStorageKey(
targetId = targetState.getBuildTargetId(target),
// getBytes is faster (70k op/s vs. 50 op/s)
// use xxh3_64 as it is more proven hash algo than komihash
pathHash = Hashing.xxh3_64().hashBytesToLong(relativizer.toRelative(file).toByteArray()),
)
}
override fun removeStamp(file: Path, target: BuildTarget<*>) {
mapHandle.map.remove(createKey(target, file))
}
override fun getPreviousStamp(file: Path, target: BuildTarget<*>): HashStamp? {
val state = getState(relativizer.toRelative(file)) ?: return null
val targetId = targetState.getBuildTargetId(target)
return state
.firstOrNull { it.targetId == targetId }
?.let { HashStamp(hash = it.hash, timestamp = it.timestamp) }
return mapHandle.map.get(createKey(target, file))
}
fun getStoredFileHash(file: Path, target: BuildTarget<*>): Long? {
val state = getState(relativizer.toRelative(file)) ?: return null
val targetId = targetState.getBuildTargetId(target)
return state.firstOrNull { it.targetId == targetId }?.hash
return mapHandle.map.get(createKey(target, file))?.hash
}
override fun getCurrentStamp(file: Path): HashStamp {
@@ -94,47 +90,109 @@ internal class HashStampStorage(
}
}
internal class HashStampPerTarget(@JvmField val targetId: Int, @JvmField val hash: Long, @JvmField val timestamp: Long)
internal class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp
internal data class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp
private class HashStampStorageKey(@JvmField val targetId: Int, @JvmField val pathHash: Long)
private fun getStorageRoot(dataStorageRoot: Path): Path = dataStorageRoot.resolve("hashes")
private object HashStampStorageKeyType : DataType<HashStampStorageKey> {
override fun isMemoryEstimationAllowed() = true
private fun updateFilesStamp(oldState: Array<HashStampPerTarget>?, targetId: Int, stamp: HashStamp): Array<HashStampPerTarget> {
val newItem = HashStampPerTarget(targetId = targetId, hash = stamp.hash, timestamp = stamp.timestamp)
if (oldState == null) {
return arrayOf(newItem)
}
override fun getMemory(obj: HashStampStorageKey): Int = Int.SIZE_BYTES + Long.SIZE_BYTES
var i = 0
val length = oldState.size
while (i < length) {
if (oldState[i].targetId == targetId) {
oldState[i] = newItem
return oldState
override fun createStorage(size: Int): Array<HashStampStorageKey?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
for (key in (storage as Array<HashStampStorageKey>)) {
buff.putVarInt(key.targetId)
// not var long - maybe negative number
buff.putLong(key.pathHash)
}
}
override fun write(buff: WriteBuffer, obj: HashStampStorageKey) = throw IllegalStateException("Must not be called")
override fun read(buff: ByteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<HashStampStorageKey>
for (i in 0 until len) {
storage[i] = HashStampStorageKey(targetId = readVarInt(buff), pathHash = buff.getLong())
}
}
override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called")
override fun binarySearch(key: HashStampStorageKey, storage: Any, size: Int, initialGuess: Int): Int {
@Suppress("UNCHECKED_CAST")
storage as Array<HashStampStorageKey>
var low = 0
var high = size - 1
// the cached index minus one, so that for the first time (when cachedCompare is 0), the default value is used
var x = initialGuess - 1
if (x < 0 || x > high) {
x = high ushr 1
}
while (low <= high) {
val b = storage[x]
val compare = when {
key.targetId > b.targetId -> 1
key.targetId < b.targetId -> -1
key.pathHash > b.pathHash -> 1
key.pathHash < b.pathHash -> -1
else -> 0
}
when {
compare > 0 -> low = x + 1
compare < 0 -> high = x - 1
else -> return x
}
x = (low + high) ushr 1
}
return low.inv()
}
@Suppress("DuplicatedCode")
override fun compare(a: HashStampStorageKey, b: HashStampStorageKey): Int {
return when {
a.targetId > b.targetId -> 1
a.targetId < b.targetId -> -1
a.pathHash > b.pathHash -> 1
a.pathHash < b.pathHash -> -1
else -> 0
}
i++
}
return oldState + newItem
}
private object StateExternalizer : DataExternalizer<Array<HashStampPerTarget>> {
override fun save(out: DataOutput, value: Array<HashStampPerTarget>) {
out.writeInt(value.size)
for (target in value) {
out.writeInt(target.targetId)
out.writeLong(target.hash)
out.writeLong(target.timestamp)
private object HashStampStorageValueType : DataType<HashStamp> {
override fun isMemoryEstimationAllowed() = true
override fun getMemory(obj: HashStamp): Int = Long.SIZE_BYTES + Long.SIZE_BYTES
override fun createStorage(size: Int): Array<HashStamp?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
for (value in (storage as Array<HashStamp>)) {
buff.putLong(value.hash)
buff.putVarLong(value.timestamp)
}
}
override fun read(`in`: DataInput): Array<HashStampPerTarget> {
val size = `in`.readInt()
return Array(size) {
val id = `in`.readInt()
val hash = `in`.readLong()
val timestamp = `in`.readLong()
HashStampPerTarget(targetId = id, hash = hash, timestamp = timestamp)
override fun write(buff: WriteBuffer, obj: HashStamp) = throw IllegalStateException("Must not be called")
override fun read(buff: ByteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<HashStamp>
for (i in 0 until len) {
storage[i] = HashStamp(hash = buff.getLong(), timestamp = readVarLong(buff))
}
}
override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called")
override fun compare(a: HashStamp, b: HashStamp) = throw IllegalStateException("Must not be called")
override fun binarySearch(key: HashStamp?, storage: Any?, size: Int, initialGuess: Int) = throw IllegalStateException("Must not be called")
}

View File

@@ -20,7 +20,7 @@ import java.util.*;
/**
* @author Eugene Zhuravlev
*/
public final class OneToManyPathsMapping extends AbstractStateStorage<String, Collection<String>> {
public final class OneToManyPathsMapping extends AbstractStateStorage<String, Collection<String>> implements OneToManyPathMapping {
private final PathRelativizerService relativizer;
public OneToManyPathsMapping(@NotNull Path storePath, PathRelativizerService relativizer) throws IOException {
@@ -29,8 +29,8 @@ public final class OneToManyPathsMapping extends AbstractStateStorage<String, Co
}
@Override
public void update(@NotNull String keyPath, @SuppressWarnings("NullableProblems") @NotNull Collection<String> boundPaths) throws IOException {
super.update(normalizePath(keyPath), normalizePaths((List<String>)boundPaths));
public void update(@NotNull String keyPath, @SuppressWarnings("NullableProblems") @NotNull List<String> boundPaths) throws IOException {
super.update(normalizePath(keyPath), normalizePaths(boundPaths));
}
public void update(@NotNull String keyPath, @NotNull String boundPath) throws IOException {

View File

@@ -3,6 +3,8 @@ package org.jetbrains.jps.incremental.storage;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.NioFiles;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
import java.io.File;
@@ -23,31 +25,28 @@ public final class ProjectStamps {
private final StampsStorage<? extends StampsStorage.Stamp> stampStorage;
public ProjectStamps(Path dataStorageRoot, BuildTargetsState targetsState, PathRelativizerService relativizer) throws IOException {
if (PORTABLE_CACHES) {
stampStorage = new HashStampStorage(dataStorageRoot, relativizer, targetsState);
}
else {
stampStorage = new FileTimestampStorage(dataStorageRoot, targetsState);
}
@ApiStatus.Internal
public ProjectStamps(@NotNull StampsStorage<? extends StampsStorage.Stamp> stampStorage) throws IOException {
this.stampStorage = stampStorage;
}
public ProjectStamps(@NotNull Path dataStorageRoot, @NotNull BuildTargetsState targetsState) throws IOException {
this(new FileTimestampStorage(dataStorageRoot, targetsState));
}
/**
* @deprecated Please use {@link #ProjectStamps(Path, BuildTargetsState, PathRelativizerService)}
* @deprecated Please use {@link #ProjectStamps(Path, BuildTargetsState)}
*/
@SuppressWarnings("unused")
@Deprecated
public ProjectStamps(File dataStorageRoot, BuildTargetsState targetsState, PathRelativizerService relativizer) throws IOException {
this(dataStorageRoot.toPath(), targetsState, relativizer);
this(dataStorageRoot.toPath(), targetsState);
}
public StampsStorage<? extends StampsStorage.Stamp> getStampStorage() {
public @NotNull StampsStorage<? extends StampsStorage.Stamp> getStampStorage() {
return stampStorage;
}
public void clean() {
stampStorage.wipe();
}
public void close() {
try {
stampStorage.close();

View File

@@ -16,7 +16,7 @@ import java.util.Iterator;
public final class SourceToOutputMappingImpl implements SourceToOutputMapping, StorageOwner {
private final OneToManyPathsMapping myMapping;
public SourceToOutputMappingImpl(@NotNull Path storePath, PathRelativizerService relativizer) throws IOException {
public SourceToOutputMappingImpl(@NotNull Path storePath, @NotNull PathRelativizerService relativizer) throws IOException {
myMapping = new OneToManyPathsMapping(storePath, relativizer);
}

View File

@@ -31,8 +31,6 @@ public interface StampsStorage<T extends Stamp> {
@NotNull
T getCurrentStamp(@NotNull Path file) throws IOException;
boolean wipe();
void close() throws IOException;
boolean isDirtyStamp(@NotNull Stamp stamp, @NotNull Path file) throws IOException;

View File

@@ -0,0 +1,182 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.diagnostic.thisLogger
import org.h2.mvstore.MVMap
import org.h2.mvstore.MVStore
import org.h2.mvstore.type.DataType
import org.jetbrains.annotations.ApiStatus
import java.nio.file.Files
import java.nio.file.Path
internal const val MV_STORE_CACHE_SIZE_IN_MB = 32
@ApiStatus.Internal
class StorageManager(@JvmField val file: Path, allowedCompactionTimeOnClose: Int) {
private val storeValue = StoreValue(file = file, allowedCompactionTimeOnClose = allowedCompactionTimeOnClose)
fun <K : Any, V : Any> openMap(name: String, keyType: DataType<K>, valueType: DataType<V>): MapHandle<K, V> {
val mapBuilder = MVMap.Builder<K, V>()
mapBuilder.setKeyType(keyType)
mapBuilder.setValueType(valueType)
return openMap(name, mapBuilder)
}
fun <K : Any, V : Any> openMap(name: String, mapBuilder: MVMap.Builder<K, V>): MapHandle<K, V> {
val store = storeValue.openStore()
return MapHandle(storeValue, openOrResetMap(store = store, name = name, mapBuilder = mapBuilder, logSupplier = ::thisLogger))
}
/** Only if error occurred and you release all [MapHandle]s */
fun forceClose() {
storeValue.forceClose()
}
}
internal class StoreValue(private val file: Path, private val allowedCompactionTimeOnClose: Int) {
private var refCount = 0
private var store: MVStore? = null
@Synchronized
fun forceClose() {
refCount = 0
store?.let {
store = null
it.closeImmediately()
}
}
@Synchronized
fun openStore(): MVStore {
if (refCount == 0) {
require(store == null)
val store = createOrResetMvStore(file = file, readOnly = false, ::thisLogger)
this.store = store
refCount++
return store
}
refCount++
return store!!
}
@Synchronized
fun release() {
when (refCount) {
1 -> {
store!!.close(allowedCompactionTimeOnClose)
store = null
refCount = 0
}
0 -> throw IllegalStateException("Store is already closed")
else -> refCount--
}
}
}
@ApiStatus.Internal
class MapHandle<K : Any, V: Any> internal constructor(
private val storeValue: StoreValue,
@JvmField val map: MVMap<K, V>,
) {
@Volatile
private var isReleased = false
fun release() {
if (!isReleased) {
storeValue.release()
isReleased = true
}
}
fun tryCommit() {
require(!isReleased)
map.store.tryCommit()
}
}
private fun <K : Any, V: Any> openOrResetMap(
store: MVStore,
name: String,
mapBuilder: MVMap.Builder<K, V>,
logSupplier: () -> Logger,
): MVMap<K, V> {
try {
return store.openMap(name, mapBuilder)
}
catch (e: Throwable) {
logSupplier().error("Cannot open map $name, map will be removed", e)
try {
store.removeMap(name)
}
catch (e2: Throwable) {
e.addSuppressed(e2)
}
}
return store.openMap(name, mapBuilder)
}
private fun createOrResetMvStore(
file: Path?,
@Suppress("SameParameterValue") readOnly: Boolean = false,
logSupplier: () -> Logger,
): MVStore {
// If read-only and DB does not yet exist, create an in-memory DB
if (file == null || (readOnly && Files.notExists(file))) {
// in-memory
return tryOpenMvStore(file = null, readOnly = readOnly, logSupplier = logSupplier)
}
val markerFile = getInvalidateMarkerFile(file)
if (Files.exists(markerFile)) {
Files.deleteIfExists(file)
Files.deleteIfExists(markerFile)
}
file.parent?.let { Files.createDirectories(it) }
try {
return tryOpenMvStore(file, readOnly, logSupplier)
}
catch (e: Throwable) {
logSupplier().warn("Cannot open cache state storage, will be recreated", e)
}
Files.deleteIfExists(file)
return tryOpenMvStore(file, readOnly, logSupplier)
}
private fun getInvalidateMarkerFile(file: Path): Path = file.resolveSibling("${file.fileName}.invalidated")
private fun tryOpenMvStore(file: Path?, readOnly: Boolean, logSupplier: () -> Logger): MVStore {
val storeErrorHandler = StoreErrorHandler(file, logSupplier)
val store = MVStore.Builder()
.fileName(file?.toAbsolutePath()?.toString())
.backgroundExceptionHandler(storeErrorHandler)
// avoid extra thread - db maintainer should use coroutines
.autoCommitDisabled()
.cacheSize(MV_STORE_CACHE_SIZE_IN_MB)
.let {
if (readOnly) it.readOnly() else it
}
.open()
storeErrorHandler.isStoreOpened = true
// versioning isn't required, otherwise the file size will be larger than needed
store.setVersionsToKeep(0)
return store
}
private class StoreErrorHandler(private val dbFile: Path?, private val logSupplier: () -> Logger) : Thread.UncaughtExceptionHandler {
@JvmField
var isStoreOpened: Boolean = false
override fun uncaughtException(t: Thread, e: Throwable) {
val log = logSupplier()
if (isStoreOpened) {
log.error("Store error (db=$dbFile)", e)
}
else {
log.warn("Store will be recreated (db=$dbFile)", e)
}
}
}

View File

@@ -0,0 +1,77 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage.dataTypes
import org.h2.mvstore.WriteBuffer
import org.h2.mvstore.type.DataType
import java.nio.ByteBuffer
internal object LongPairKeyDataType : DataType<LongArray> {
override fun isMemoryEstimationAllowed() = true
// don't care about non-ASCII strings for memory estimation
override fun getMemory(obj: LongArray): Int = 16
override fun createStorage(size: Int): Array<LongArray?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
for (key in (storage as Array<LongArray>)) {
buff.putLong(key[0])
buff.putLong(key[1])
}
}
override fun write(buff: WriteBuffer, obj: LongArray) = throw IllegalStateException("Must not be called")
override fun read(buff: ByteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<LongArray>
for (i in 0 until len) {
storage[i] = longArrayOf(buff.getLong(), buff.getLong())
}
}
override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called")
override fun binarySearch(key: LongArray, storage: Any, size: Int, initialGuess: Int): Int {
@Suppress("UNCHECKED_CAST")
storage as Array<LongArray>
var low = 0
var high = size - 1
// the cached index minus one, so that for the first time (when cachedCompare is 0), the default value is used
var x = initialGuess - 1
if (x < 0 || x > high) {
x = high ushr 1
}
while (low <= high) {
val b = storage[x]
val compare = when {
key[0] > b[0] -> 1
key[0] < b[0] -> -1
key[1] > b[1] -> 1
key[1] < b[1] -> -1
else -> 0
}
when {
compare > 0 -> low = x + 1
compare < 0 -> high = x - 1
else -> return x
}
x = (low + high) ushr 1
}
return low.inv()
}
@Suppress("DuplicatedCode")
override fun compare(a: LongArray, b: LongArray): Int {
return when {
a[0] > b[0] -> 1
a[0] < b[0] -> -1
a[1] > b[1] -> 1
a[1] < b[1] -> -1
else -> 0
}
}
}

View File

@@ -0,0 +1,49 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.incremental.storage.dataTypes
import org.h2.mvstore.DataUtils.readVarInt
import org.h2.mvstore.WriteBuffer
import org.h2.mvstore.type.DataType
import java.nio.ByteBuffer
internal object StringListDataType : DataType<Array<String>> {
override fun isMemoryEstimationAllowed() = true
// don't care about non-ASCII for size computation - non-ASCII strings should be quite rare
override fun getMemory(obj: Array<String>): Int = Int.SIZE_BYTES + obj.sumOf { it.length }
override fun createStorage(size: Int): Array<Array<String>?> = arrayOfNulls(size)
override fun write(buff: WriteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<Array<String>>
for (l in storage) {
buff.putVarInt(l.size)
for (s in l) {
val bytes = s.toByteArray()
buff.putVarInt(bytes.size).put(bytes)
}
}
}
override fun write(buff: WriteBuffer, obj: Array<String>) = throw IllegalStateException("Must not be called")
override fun read(buff: ByteBuffer, storage: Any, len: Int) {
@Suppress("UNCHECKED_CAST")
storage as Array<Array<String>>
for (i in 0 until len) {
storage[i] = Array(readVarInt(buff)) {
val bytes = ByteArray(readVarInt(buff))
buff.get(bytes)
String(bytes)
}
}
}
override fun read(buff: ByteBuffer) = throw IllegalStateException("Must not be called")
override fun compare(a: Array<String>, b: Array<String>) = throw IllegalStateException("Must not be called")
override fun binarySearch(key: Array<String>?, storage: Any?, size: Int, initialGuess: Int) = throw IllegalStateException("Must not be called")
}

View File

@@ -1,4 +1,4 @@
// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.builders;
import com.intellij.openapi.application.PathManager;
@@ -200,8 +200,8 @@ public abstract class JpsBuildTestCase extends UsefulTestCase {
BuildTargetIndexImpl targetIndex = new BuildTargetIndexImpl(targetRegistry, buildRootIndex);
BuildTargetsState targetsState = new BuildTargetsState(dataPaths, myModel, buildRootIndex);
PathRelativizerService relativizer = new PathRelativizerService(myModel.getProject());
ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState, relativizer);
BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer);
ProjectStamps projectStamps = new ProjectStamps(myDataStorageRoot.toPath(), targetsState);
BuildDataManager dataManager = new BuildDataManager(dataPaths, targetsState, relativizer, null);
return new ProjectDescriptor(myModel, new BuildFSState(true), projectStamps, dataManager, buildLoggingManager, index,
targetIndex, buildRootIndex, ignoredFileIndex);
}

View File

@@ -1,4 +1,4 @@
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.uiDesigner.compiler;
import com.intellij.openapi.util.Key;
@@ -22,7 +22,7 @@ import org.jetbrains.jps.incremental.java.CopyResourcesUtil;
import org.jetbrains.jps.incremental.java.FormsParsing;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping;
import org.jetbrains.jps.incremental.storage.OneToManyPathMapping;
import org.jetbrains.jps.model.JpsProject;
import org.jetbrains.jps.model.java.JpsJavaExtensionService;
import org.jetbrains.jps.model.java.compiler.JpsCompilerExcludes;
@@ -158,7 +158,7 @@ public final class FormsBindingManager extends FormsBuilder {
formsToCompile.keySet().removeAll(alienForms);
// form should be considered dirty if the class it is bound to is dirty
final OneToManyPathsMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
for (Map.Entry<File, ModuleBuildTarget> entry : filesToCompile.entrySet()) {
final File srcFile = entry.getKey();
final ModuleBuildTarget target = entry.getValue();

View File

@@ -1,4 +1,4 @@
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.uiDesigner.compiler;
import com.intellij.compiler.instrumentation.FailSafeClassReader;
@@ -26,7 +26,7 @@ import org.jetbrains.jps.incremental.instrumentation.ClassProcessingBuilder;
import org.jetbrains.jps.incremental.messages.BuildMessage;
import org.jetbrains.jps.incremental.messages.CompilerMessage;
import org.jetbrains.jps.incremental.messages.ProgressMessage;
import org.jetbrains.jps.incremental.storage.OneToManyPathsMapping;
import org.jetbrains.jps.incremental.storage.OneToManyPathMapping;
import org.jetbrains.jps.model.JpsDummyElement;
import org.jetbrains.jps.model.JpsProject;
import org.jetbrains.jps.model.java.JpsJavaSdkType;
@@ -88,13 +88,13 @@ public final class FormsInstrumenter extends FormsBuilder {
try {
final Map<File, Collection<File>> processed = instrumentForms(context, chunk, chunkSourcePath, finder, formsToCompile, outputConsumer, config.isUseDynamicBundles());
final OneToManyPathsMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
for (Map.Entry<File, Collection<File>> entry : processed.entrySet()) {
final File src = entry.getKey();
final Collection<File> forms = entry.getValue();
final Collection<String> formPaths = new ArrayList<>(forms.size());
List<String> formPaths = new ArrayList<>(forms.size());
for (File form : forms) {
formPaths.add(form.getPath());
}