diff --git a/jps/jps-builders/api-dump.txt b/jps/jps-builders/api-dump.txt index bc8ea524612f..26aa0c3ff679 100644 --- a/jps/jps-builders/api-dump.txt +++ b/jps/jps-builders/api-dump.txt @@ -2596,6 +2596,7 @@ f:org.jetbrains.jps.incremental.relativizer.PathRelativizerService - reportUnhandledPaths():V - toFull(java.lang.String):java.lang.String - toRelative(java.lang.String):java.lang.String +- toRelative(java.nio.file.Path):java.lang.String c:org.jetbrains.jps.incremental.resources.ResourcesBuilder - org.jetbrains.jps.incremental.TargetBuilder - ():V @@ -2609,16 +2610,16 @@ a:org.jetbrains.jps.incremental.storage.AbstractStateStorage - pf:dataLock:java.lang.Object - (java.io.File,com.intellij.util.io.KeyDescriptor,com.intellij.util.io.DataExternalizer):V - appendData(java.lang.Object,java.lang.Object):V -- clean():V -- close():V +- f:clean():V +- f:close():V - flush(Z):V -- force():V +- f:force():V - getKeys():java.util.Collection - getKeysIterator():java.util.Iterator - getState(java.lang.Object):java.lang.Object - remove(java.lang.Object):V - update(java.lang.Object,java.lang.Object):V -- wipe():Z +- f:wipe():Z f:org.jetbrains.jps.incremental.storage.BuildDataManager - sf:PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY:java.lang.String - (org.jetbrains.jps.builders.storage.BuildDataPaths,org.jetbrains.jps.incremental.storage.BuildTargetsState,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):V diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/relativizer/PathRelativizerService.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/relativizer/PathRelativizerService.java index 5df4debe737a..af9c3c2e88d0 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/relativizer/PathRelativizerService.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/relativizer/PathRelativizerService.java @@ -18,6 +18,7 @@ import org.jetbrains.jps.model.serialization.JpsModelSerializationDataService; import org.jetbrains.jps.util.JpsPathUtil; import java.io.File; +import java.nio.file.Path; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -70,6 +71,10 @@ public final class PathRelativizerService { myRelativizers.add(new GradlePathRelativizer()); } + public @NotNull String toRelative(@NotNull Path path) { + return toRelative(path.toString()); + } + /** * @param path absolute path which should be converted. * It may use forward or backward slashes as separators, so there is no need to convert it before passing to the method diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java index 1c66fbdf7edd..c6a651a36a0d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java @@ -1,17 +1,16 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.incremental.storage; -import com.intellij.openapi.util.io.FileUtilRt; import com.intellij.util.CommonProcessors; -import com.intellij.util.io.AppendablePersistentMap; -import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.KeyDescriptor; -import com.intellij.util.io.PersistentHashMap; +import com.intellij.util.io.*; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -19,45 +18,51 @@ import java.util.List; public abstract class AbstractStateStorage implements StorageOwner { protected final Object dataLock = new Object(); - private final File baseFile; - private final KeyDescriptor keyDescriptor; - private final DataExternalizer stateExternalizer; - private PersistentHashMap map; + private @NotNull final PersistentMapBuilder mapBuilder; + private @NotNull PersistentMapImpl map; + private final boolean isCompressed; public AbstractStateStorage(File storePath, KeyDescriptor keyDescriptor, DataExternalizer stateExternalizer) throws IOException { - baseFile = storePath; - this.keyDescriptor = keyDescriptor; - this.stateExternalizer = stateExternalizer; - map = createMap(storePath); + this(PersistentMapBuilder.newBuilder(storePath.toPath(), keyDescriptor, stateExternalizer), + Boolean.parseBoolean(System.getProperty("jps.storage.do.compression", "true"))); } - public void force() { + @ApiStatus.Internal + protected AbstractStateStorage(@NotNull PersistentMapBuilder mapBuilder, boolean isCompressed) throws IOException { + this.isCompressed = isCompressed; + this.mapBuilder = mapBuilder; + map = createMap(); + } + + public final void force() { synchronized (dataLock) { - map.force(); + try { + map.force(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } } @Override - public void close() throws IOException { + public final void close() throws IOException { synchronized (dataLock) { map.close(); } } @Override - public void clean() throws IOException { + public final void clean() throws IOException { wipe(); } - public boolean wipe() { + @SuppressWarnings("UnusedReturnValue") + public final boolean wipe() { synchronized (dataLock) { + map.closeAndDelete(); try { - map.closeAndClean(); - } - catch (IOException ignored) { - } - try { - map = createMap(baseFile); + map = createMap(); } catch (IOException ignored) { return false; @@ -79,7 +84,7 @@ public abstract class AbstractStateStorage implements StorageOwner { public void appendData(final Key key, final T data) throws IOException { synchronized (dataLock) { - map.appendData(key, (AppendablePersistentMap.ValueDataAppender)out -> stateExternalizer.save(out, data)); + map.appendData(key, out -> mapBuilder.getValueExternalizer().save(out, data)); } } @@ -98,7 +103,7 @@ public abstract class AbstractStateStorage implements StorageOwner { public Collection getKeys() throws IOException { synchronized (dataLock) { List result = new ArrayList<>(); - map.processKeysWithExistingMapping(new CommonProcessors.CollectProcessor<>(result)); + map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result)); return result; } } @@ -106,14 +111,14 @@ public abstract class AbstractStateStorage implements StorageOwner { public Iterator getKeysIterator() throws IOException { synchronized (dataLock) { List result = new ArrayList<>(); - map.processKeysWithExistingMapping(new CommonProcessors.CollectProcessor<>(result)); + map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result)); return result.iterator(); } } - private PersistentHashMap createMap(@NotNull File file) throws IOException { - FileUtilRt.createIfNotExists(file); //todo assert - return new PersistentHashMap<>(file, keyDescriptor, stateExternalizer); + private @NotNull PersistentMapImpl createMap() throws IOException { + Files.createDirectories(mapBuilder.getFile().getParent()); + return new PersistentMapImpl<>(mapBuilder, new PersistentHashMapValueStorage.CreationTimeOptions(false, false, false, isCompressed)); } @Override diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java index a5fffeefb237..414eec714653 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildTargetSourcesState.java @@ -272,12 +272,11 @@ public final class BuildTargetSourcesState implements BuildListener { } @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { - File file = path.toFile(); - if (!buildRootIndex.isFileAccepted(file, rootDescriptor)) { + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { + if (!buildRootIndex.isFileAccepted(path.toFile(), rootDescriptor)) { return FileVisitResult.CONTINUE; } - getFileHash(target, file, path, rootFile, hash, hashToReuse); + getFileHash(target, path, rootFile, hash, hashToReuse); return FileVisitResult.CONTINUE; } }); @@ -307,15 +306,14 @@ public final class BuildTargetSourcesState implements BuildListener { } private void getFileHash(@NotNull BuildTarget target, - @NotNull File file, @NotNull Path path, @NotNull Path rootFile, @NotNull LongArrayList hash, - @NotNull HashStream64 hashToReuse) throws IOException { + @NotNull HashStream64 hashToReuse) { StampsStorage storage = projectStamps.getStampStorage(); assert storage instanceof HashStampStorage; HashStampStorage fileStampStorage = (HashStampStorage)storage; - Long fileHash = fileStampStorage.getStoredFileHash(file, target); + Long fileHash = fileStampStorage.getStoredFileHash(path, target); if (fileHash == null) { return; } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/HashStampStorage.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/HashStampStorage.kt index e529cc58521e..a06182fb1ade 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/HashStampStorage.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/HashStampStorage.kt @@ -5,41 +5,37 @@ import com.dynatrace.hash4j.hashing.Hashing import com.intellij.util.ArrayUtil import com.intellij.util.io.DataExternalizer import com.intellij.util.io.EnumeratorStringDescriptor +import com.intellij.util.io.PersistentMapBuilder import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.incremental.FSOperations import org.jetbrains.jps.incremental.FileHashUtil import org.jetbrains.jps.incremental.relativizer.PathRelativizerService -import org.jetbrains.jps.incremental.storage.FileTimestampStorage.FileTimestamp import java.io.DataInput import java.io.DataOutput -import java.io.File import java.nio.file.Path import java.nio.file.attribute.BasicFileAttributes internal class HashStampStorage( dataStorageRoot: Path, private val relativizer: PathRelativizerService, - targetsState: BuildTargetsState, -) : AbstractStateStorage>( - calcStorageRoot(dataStorageRoot).resolve("data").toFile(), - JpsCachePathStringDescriptor, - StateExternalizer, -), StampsStorage { - private val timestampStorage = FileTimestampStorage(dataStorageRoot, targetsState) - private val targetState = targetsState - private val fileStampRoot = calcStorageRoot(dataStorageRoot) + private val targetState: BuildTargetsState, +) : AbstractStateStorage>( + PersistentMapBuilder.newBuilder(getStorageRoot(dataStorageRoot).resolve("data"), JpsCachePathStringDescriptor, StateExternalizer) + .withVersion(2), + false, +), StampsStorage { + private val fileStampRoot = getStorageRoot(dataStorageRoot) override fun getStorageRoot(): Path = fileStampRoot override fun saveStamp(file: Path, buildTarget: BuildTarget<*>, stamp: HashStamp) { - timestampStorage.saveStamp(file, buildTarget, FileTimestamp.fromLong(stamp.timestamp)) val targetId = targetState.getBuildTargetId(buildTarget) - val path = relativizer.toRelative(file.toString()) + val path = relativizer.toRelative(file) update(path, updateFilesStamp(oldState = getState(path), targetId = targetId, stamp = stamp)) } override fun removeStamp(file: Path, buildTarget: BuildTarget<*>) { - timestampStorage.removeStamp(file, buildTarget) - val path = relativizer.toRelative(file.toString()) + val path = relativizer.toRelative(file) val state = getState(path) ?: return val targetId = targetState.getBuildTargetId(buildTarget) for (i in state.indices) { @@ -57,34 +53,30 @@ internal class HashStampStorage( } override fun getPreviousStamp(file: Path, target: BuildTarget<*>): HashStamp? { - val previousTimestamp = timestampStorage.getPreviousStamp(file, target) ?: return null - val state = getState(relativizer.toRelative(file.toString())) ?: return null + val state = getState(relativizer.toRelative(file)) ?: return null val targetId = targetState.getBuildTargetId(target) return state .firstOrNull { it.targetId == targetId } - ?.let { HashStamp(it.hash, previousTimestamp.asLong()) } + ?.let { HashStamp(hash = it.hash, timestamp = it.timestamp) } } - fun getStoredFileHash(file: File, target: BuildTarget<*>): Long? { - val state = getState(relativizer.toRelative(file.absolutePath)) ?: return null + fun getStoredFileHash(file: Path, target: BuildTarget<*>): Long? { + val state = getState(relativizer.toRelative(file)) ?: return null val targetId = targetState.getBuildTargetId(target) return state.firstOrNull { it.targetId == targetId }?.hash } override fun getCurrentStamp(file: Path): HashStamp { - val currentTimestamp = timestampStorage.getCurrentStamp(file) - return HashStamp(FileHashUtil.getFileHash(file), currentTimestamp.asLong()) + return HashStamp(hash = FileHashUtil.getFileHash(file), timestamp = FSOperations.lastModified(file)) } override fun isDirtyStamp(stamp: StampsStorage.Stamp, file: Path): Boolean { if (stamp !is HashStamp) { return true } - - if (!timestampStorage.isDirtyStamp(FileTimestamp.fromLong(stamp.timestamp), file)) { + if (stamp.timestamp == FSOperations.lastModified(file)) { return false } - return stamp.hash != FileHashUtil.getFileHash(file) } @@ -93,41 +85,25 @@ internal class HashStampStorage( return true } - if (!timestampStorage.isDirtyStamp(FileTimestamp.fromLong(stamp.timestamp), file, attrs)) { + // If equal, then non-dirty. + // If not equal, then we check the hash to avoid marking the file as `dirty` only because of a different timestamp. + // We cannot rely solely on the hash, as getting the last-modified timestamp is much cheaper than computing the file hash. + if ((if (attrs.isRegularFile) attrs.lastModifiedTime().toMillis() else FSOperations.lastModified(file)) == stamp.timestamp) { return false } return stamp.hash != FileHashUtil.getFileHash(file) } - - override fun force() { - super.force() - timestampStorage.force() - } - - override fun clean() { - super.clean() - timestampStorage.clean() - } - - override fun wipe(): Boolean { - return super.wipe() && timestampStorage.wipe() - } - - override fun close() { - super.close() - timestampStorage.close() - } } -internal class HashStampPerTarget(@JvmField val targetId: Int, @JvmField val hash: Long) +internal class HashStampPerTarget(@JvmField val targetId: Int, @JvmField val hash: Long, @JvmField val timestamp: Long) -internal data class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp +internal data class HashStamp(@JvmField val hash: Long, @JvmField val timestamp: Long) : StampsStorage.Stamp -private fun calcStorageRoot(dataStorageRoot: Path): Path = dataStorageRoot.resolve("hashes") +private fun getStorageRoot(dataStorageRoot: Path): Path = dataStorageRoot.resolve("hashes") private fun updateFilesStamp(oldState: Array?, targetId: Int, stamp: HashStamp): Array { - val newItem = HashStampPerTarget(targetId = targetId, hash = stamp.hash) + val newItem = HashStampPerTarget(targetId = targetId, hash = stamp.hash, timestamp = stamp.timestamp) if (oldState == null) { return arrayOf(newItem) } @@ -150,6 +126,7 @@ private object StateExternalizer : DataExternalizer> { for (target in value) { out.writeInt(target.targetId) out.writeLong(target.hash) + out.writeLong(target.timestamp) } } @@ -158,7 +135,8 @@ private object StateExternalizer : DataExternalizer> { return Array(size) { val id = `in`.readInt() val hash = `in`.readLong() - HashStampPerTarget(targetId = id, hash = hash) + val timestamp = `in`.readLong() + HashStampPerTarget(targetId = id, hash = hash, timestamp = timestamp) } } } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt index ce9ec5cbf444..e568971d3757 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/CompilationContextImpl.kt @@ -172,6 +172,11 @@ class CompilationContextImpl private constructor( enableCoroutinesDump: Boolean = true, customBuildPaths: BuildPaths? = null, ): CompilationContextImpl { + if (!options.useCompiledClassesFromProjectOutput) { + // disable compression - otherwise, our zstd/zip cannot compress efficiently + System.setProperty("jps.storage.do.compression", "false") + } + check(sequenceOf("platform/build-scripts", "bin/idea.properties", "build.txt").all { Files.exists(COMMUNITY_ROOT.communityRoot.resolve(it)) }) { diff --git a/platform/util/src/com/intellij/util/io/PersistentMapImpl.java b/platform/util/src/com/intellij/util/io/PersistentMapImpl.java index 3303abf1d29c..63e7284ecd99 100644 --- a/platform/util/src/com/intellij/util/io/PersistentMapImpl.java +++ b/platform/util/src/com/intellij/util/io/PersistentMapImpl.java @@ -17,7 +17,6 @@ import org.jetbrains.annotations.ApiStatus.Internal; import org.jetbrains.annotations.*; import java.io.DataInputStream; -import java.io.DataOutput; import java.io.File; import java.io.IOException; import java.lang.reflect.Field; @@ -246,9 +245,7 @@ public final class PersistentMapImpl implements PersistentMapBase deriveEmptyMap(@NotNull Path path) throws IOException { - return myOptions.with(() -> { - return new PersistentMapImpl<>(myBuilder.copyWithFile(path)); - }); + return myOptions.with(() -> new PersistentMapImpl<>(myBuilder.copyWithFile(path))); } @Override @@ -597,12 +594,7 @@ public final class PersistentMapImpl implements PersistentMapBase readValueId(id) != NULL_ADDR); } catch (ClosedStorageException ex) { throw ex; @@ -691,12 +683,7 @@ public final class PersistentMapImpl implements PersistentMapBase myValueExternalizer.save(out, valueRead), readResult); myEnumerator.lockStorageWrite(); try { @@ -1022,9 +1009,7 @@ public final class PersistentMapImpl implements PersistentMapBase children = Files.list(parentFile)) { - return children.filter(p -> { - return p.getFileName().toString().startsWith(fileName.toString()); - }).map(p -> p.toFile()).toArray(File[]::new); + return children.filter(p -> p.getFileName().toString().startsWith(fileName.toString())).map(Path::toFile).toArray(File[]::new); } }