From c6f555ef591f198e68b0fec10f5564117221b356 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Sat, 14 Sep 2024 16:53:20 +0200 Subject: [PATCH] experimental compact storage for JPS Cache (part 2 - finish ExperimentalOneToManyPathMapping and add test) GitOrigin-RevId: 6c8dadfc2fb7ad6c6ac39f31265c77cbbae9876b --- .gitignore | 1 + .../compiler/server/BuildManager.java | 5 + jps/jps-builders/api-dump.txt | 19 +- .../intellij.platform.jps.build.tests.iml | 49 +++++ .../storage/SourceToOutputMapping.java | 43 ++-- .../jps/incremental/BuildOperations.java | 16 +- .../jps/incremental/IncProjectBuilder.java | 26 ++- .../storage/AbstractStateStorage.java | 24 ++- .../incremental/storage/BuildDataManager.java | 29 +-- .../ExperimentalOneToManyPathMapping.kt | 78 +++---- .../ExperimentalSourceToOutputMapping.kt | 197 ++++++++++++++++++ .../storage/OneToManyPathMapping.kt | 22 ++ .../storage/OneToManyPathsMapping.java | 89 +++++--- .../storage/SourceToOutputMappingImpl.java | 27 +-- .../jetbrains/jps/builders/BuildResult.java | 26 ++- .../storage/SourceToOutputMappingFuzzTest.kt | 115 ++++++++++ .../groovy/jps-plugin/intellij.groovy.jps.iml | 1 + .../incremental/groovy/JpsGroovycRunner.java | 17 +- .../compiler/FormsBindingManager.java | 2 +- .../compiler/FormsInstrumenter.java | 2 +- 20 files changed, 606 insertions(+), 182 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt create mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt create mode 100644 jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt diff --git a/.gitignore b/.gitignore index 5323abbe8cb1..505d52c25f20 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .idea/libraries/Gradle__*.xml .idea/shelf .idea/workspace.xml +.jqwik-database /.diff /config /out diff --git a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java index 4d3ebffb05e5..e8b68d8b1378 100644 --- a/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java +++ b/java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java @@ -174,6 +174,8 @@ public final class BuildManager implements Disposable { s -> !(s.contains(IDEA_PROJECT_DIR_PATTERN) || s.endsWith(IWS_EXTENSION) || s.endsWith(IPR_EXTENSION)) : s -> !(Strings.endsWithIgnoreCase(s, IWS_EXTENSION) || Strings.endsWithIgnoreCase(s, IPR_EXTENSION) || StringUtil.containsIgnoreCase(s, IDEA_PROJECT_DIR_PATTERN)); + private static final String JPS_USE_EXPERIMENTAL_STORAGE = "jps.use.experimental.storage"; + private final String myFallbackSdkHome; private final String myFallbackSdkVersion; @@ -1409,6 +1411,9 @@ public final class BuildManager implements Disposable { } cmdLine.addParameter("-Djava.awt.headless=true"); + if (Boolean.getBoolean(JPS_USE_EXPERIMENTAL_STORAGE)) { + cmdLine.addParameter("-D" + JPS_USE_EXPERIMENTAL_STORAGE + "=true"); + } String jnaBootLibraryPath = System.getProperty("jna.boot.library.path"); if (jnaBootLibraryPath != null && wslPath == null) { diff --git a/jps/jps-builders/api-dump.txt b/jps/jps-builders/api-dump.txt index bcd9be3248c3..fa585d41e71a 100644 --- a/jps/jps-builders/api-dump.txt +++ b/jps/jps-builders/api-dump.txt @@ -1836,16 +1836,17 @@ org.jetbrains.jps.builders.storage.BuildDataPaths org.jetbrains.jps.builders.storage.SourceToOutputMapping - a:appendOutput(java.lang.String,java.lang.String):V - a:getOutputs(java.lang.String):java.util.Collection -- a:getOutputsIterator(java.lang.String):java.util.Iterator -- a:getSources():java.util.Collection +- getSources():java.util.Collection - a:getSourcesIterator():java.util.Iterator - a:remove(java.lang.String):V - a:removeOutput(java.lang.String,java.lang.String):V - a:setOutput(java.lang.String,java.lang.String):V -- a:setOutputs(java.lang.String,java.util.Collection):V +- a:setOutputs(java.lang.String,java.util.List):V a:org.jetbrains.jps.builders.storage.StorageProvider - ():V -- a:createStorage(java.nio.file.Path):org.jetbrains.jps.incremental.storage.StorageOwner +- createStorage(java.io.File):org.jetbrains.jps.incremental.storage.StorageOwner +- createStorage(java.io.File,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):org.jetbrains.jps.incremental.storage.StorageOwner +- createStorage(java.nio.file.Path):org.jetbrains.jps.incremental.storage.StorageOwner - createStorage(java.nio.file.Path,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):org.jetbrains.jps.incremental.storage.StorageOwner f:org.jetbrains.jps.cmdline.BuildRunner - (org.jetbrains.jps.cmdline.JpsModelLoader):V @@ -2609,7 +2610,7 @@ a:org.jetbrains.jps.incremental.storage.AbstractStateStorage - f:close():V - flush(Z):V - f:force():V -- getKeys():java.util.Collection +- pf:getKeyIterator(java.util.function.Function):java.util.Iterator - getKeysIterator():java.util.Iterator - getState(java.lang.Object):java.lang.Object - remove(java.lang.Object):V @@ -2686,14 +2687,14 @@ f:org.jetbrains.jps.incremental.storage.OneToManyPathsMapping - (java.nio.file.Path,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):V - appendData(java.lang.String,java.lang.String):V - appendData(java.lang.String,java.util.Collection):V -- getKeys():java.util.List - getKeysIterator():java.util.Iterator +- getOutputArray(java.lang.String):java.lang.String[] +- a:getOutputs(java.lang.String):java.util.Collection +- getOutputs(java.lang.String):java.util.List - getState(java.lang.String):java.util.Collection -- getStateIterator(java.lang.String):java.util.Iterator - remove(java.lang.String):V - removeData(java.lang.String,java.lang.String):V -- update(java.lang.String,java.lang.String):V -- update(java.lang.String,java.util.List):V +- setOutputs(java.lang.String,java.util.List):V f:org.jetbrains.jps.incremental.storage.OutputToTargetRegistry - org.jetbrains.jps.incremental.storage.AbstractStateStorage - getSafeToDeleteOutputs(java.util.Collection,I):java.util.Collection diff --git a/jps/jps-builders/intellij.platform.jps.build.tests.iml b/jps/jps-builders/intellij.platform.jps.build.tests.iml index 68c117a106c1..ea678fbc894c 100644 --- a/jps/jps-builders/intellij.platform.jps.build.tests.iml +++ b/jps/jps-builders/intellij.platform.jps.build.tests.iml @@ -20,5 +20,54 @@ + + + + + + bcbbc834752835d9c534647f925756de2cd6f51a1e94fcf6162fd55f3fc2c706 + + + b509448ac506d607319f182537f0b35d71007582ec741832a1f111e5b5b70b38 + + + 1ebd46a1df6575c3d46ba263a6575c88fe654f164f9e3d1f7fbc05fdc9dc726c + + + 67030794b96c3e854795c6e709b1e9adc54df93474cb8e8380a791d58db2560e + + + 4258ec1239879e520ac6df63de21b821691cf57479a14700c6ab8bd98cba1fa5 + + + 6eb6ffbfcf81f5eef65cffdd41186f961f1a0fcb3c44f17e4c09d5d3aa773b2c + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/storage/SourceToOutputMapping.java b/jps/jps-builders/src/org/jetbrains/jps/builders/storage/SourceToOutputMapping.java index 679d31dfc851..361c115a933e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/storage/SourceToOutputMapping.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/storage/SourceToOutputMapping.java @@ -1,49 +1,48 @@ -/* - * Copyright 2000-2012 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.builders.storage; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.jps.incremental.storage.SourceToOutputMappingCursor; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.List; public interface SourceToOutputMapping { - void setOutputs(@NotNull String srcPath, @NotNull Collection outputs) throws IOException; + void setOutputs(@NotNull String srcPath, @NotNull List outputs) throws IOException; void setOutput(@NotNull String srcPath, @NotNull String outputPath) throws IOException; void appendOutput(@NotNull String srcPath, @NotNull String outputPath) throws IOException; - void remove(@NotNull String srcPath) throws IOException; void removeOutput(@NotNull String sourcePath, @NotNull String outputPath) throws IOException; - - @NotNull - Collection getSources() throws IOException; + /** + * @deprecated Use {@link #getSourcesIterator()} + */ + @Deprecated(forRemoval = true) + default @NotNull Collection getSources() throws IOException { + List result = new ArrayList<>(); + Iterator iterator = getSourcesIterator(); + while (iterator.hasNext()) { + result.add(iterator.next()); + } + return result; + } @Nullable Collection getOutputs(@NotNull String srcPath) throws IOException; @NotNull - Iterator getOutputsIterator(@NotNull String srcPath) throws IOException; + Iterator getSourcesIterator() throws IOException; @NotNull - Iterator getSourcesIterator() throws IOException; + @ApiStatus.Internal + SourceToOutputMappingCursor cursor() throws IOException; } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java index c2c4f830f208..3b58bd4aca9d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java @@ -1,4 +1,4 @@ -// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.incremental; import com.intellij.openapi.util.io.FileUtilRt; @@ -178,14 +178,14 @@ public final class BuildOperations { }); - if (JavaBuilderUtil.isCompileJavaIncrementally(context)) { - final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); - if (logger.isEnabled()) { - logger.logDeletedFiles(deletedPaths); - } - } - if (!deletedPaths.isEmpty()) { + if (JavaBuilderUtil.isCompileJavaIncrementally(context)) { + final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); + if (logger.isEnabled()) { + logger.logDeletedFiles(deletedPaths); + } + } + context.processMessage(new FileDeletedEvent(deletedPaths)); } // attempting to delete potentially empty directories diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java index 8d57b3fa9b5b..3e5078cf8b67 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java @@ -728,14 +728,15 @@ public final class IncProjectBuilder { int targetId) throws IOException { Set dirsToDelete = targetType instanceof ModuleBasedBuildTargetType ? FileCollectionFactory.createCanonicalFileSet() : null; OutputToTargetRegistry outputToTargetRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetRegistry(); - for (String srcPath : mapping.getSources()) { - final Collection outs = mapping.getOutputs(srcPath); - if (outs != null && !outs.isEmpty()) { + for (SourceToOutputMappingCursor cursor = mapping.cursor(); cursor.hasNext(); ) { + cursor.next(); + String [] outs = cursor.getOutputPaths(); + if (outs.length > 0) { List deletedPaths = new ArrayList<>(); for (String out : outs) { BuildOperations.deleteRecursively(out, deletedPaths, dirsToDelete); } - outputToTargetRegistry.removeMapping(outs, targetId); + outputToTargetRegistry.removeMapping(Arrays.asList(outs), targetId); if (!deletedPaths.isEmpty()) { context.processMessage(new FileDeletedEvent(deletedPaths)); } @@ -1309,8 +1310,12 @@ public final class IncProjectBuilder { // Change of only one input of *.kotlin_module files didn't trigger recompilation of all inputs in old behaviour. // Now it does. It isn't yet obvious whether it is right or wrong behaviour. Let's leave old behaviour for a // while for safety and keeping kotlin incremental JPS tests green - List filteredOuts = - ContainerUtil.filter(outs, out -> !"kotlin_module".equals(StringUtil.substringAfterLast(out, "."))); + List filteredOuts = new ArrayList<>(); + for (String out : outs) { + if (!"kotlin_module".equals(StringUtil.substringAfterLast(out, "."))) { + filteredOuts.add(out); + } + } affectedOutputs.addAll(filteredOuts); } } @@ -1321,10 +1326,11 @@ public final class IncProjectBuilder { if (!affectedOutputs.isEmpty()) { for (SourceToOutputMapping srcToOut : mappings) { - for (String src : srcToOut.getSources()) { + for (SourceToOutputMappingCursor cursor = srcToOut.cursor(); cursor.hasNext(); ) { + String src = cursor.next(); if (!affectedSources.contains(src)) { - for (Iterator it = srcToOut.getOutputsIterator(src); it.hasNext(); ) { - if (affectedOutputs.contains(it.next())) { + for (String out : cursor.getOutputPaths()) { + if (affectedOutputs.contains(out)) { FSOperations.markDirtyIfNotDeleted(context, CompilationRound.CURRENT, new File(src)); break; } @@ -1508,7 +1514,7 @@ public final class IncProjectBuilder { if (target instanceof ModuleBuildTarget) { // check if the deleted source was associated with a form final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap(); - final Collection boundForms = sourceToFormMap.getState(deletedSource); + final Collection boundForms = sourceToFormMap.getOutputs(deletedSource); if (boundForms != null) { for (String formPath : boundForms) { final File formFile = new File(formPath); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java index 1f42880a293b..fb17f53e8782 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java @@ -12,9 +12,10 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.function.Function; public abstract class AbstractStateStorage implements StorageOwner { private static final boolean DO_COMPRESS = Boolean.parseBoolean(System.getProperty("jps.storage.do.compression", "true")); @@ -106,15 +107,7 @@ public abstract class AbstractStateStorage implements StorageOwner { } } - public Collection getKeys() throws IOException { - synchronized (dataLock) { - List result = new ArrayList<>(); - map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result)); - return result; - } - } - - public Iterator getKeysIterator() throws IOException { + public @NotNull Iterator getKeysIterator() throws IOException { synchronized (dataLock) { List result = new ArrayList<>(); map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result)); @@ -122,6 +115,17 @@ public abstract class AbstractStateStorage implements StorageOwner { } } + protected final @NotNull Iterator getKeyIterator(@NotNull Function mapper) throws IOException { + synchronized (dataLock) { + List result = new ArrayList<>(); + map.processExistingKeys(key -> { + result.add(mapper.apply(key)); + return true; + }); + return result.isEmpty() ? Collections.emptyIterator() : result.iterator(); + } + } + private @NotNull PersistentMapImpl createMap() throws IOException { Files.createDirectories(mapBuilder.getFile().getParent()); return new PersistentMapImpl<>(mapBuilder, new PersistentHashMapValueStorage.CreationTimeOptions(false, false, false, isCompressed)); diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java index 738fc20fc0f7..27ffdaa9a911 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java @@ -61,6 +61,7 @@ public final class BuildDataManager { private final NodeSourcePathMapper myDepGraphPathMapper; private final BuildDataPaths myDataPaths; private final BuildTargetsState myTargetsState; + @Nullable private final StorageManager storageManager; private final OutputToTargetRegistry myOutputToTargetRegistry; private final File myVersionFile; private final PathRelativizerService myRelativizer; @@ -85,6 +86,7 @@ public final class BuildDataManager { @Nullable StorageManager storageManager) throws IOException { myDataPaths = dataPaths; myTargetsState = targetsState; + this.storageManager = storageManager; try { if (storageManager == null) { mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer); @@ -139,9 +141,15 @@ public final class BuildDataManager { return myOutputToTargetRegistry; } - public SourceToOutputMapping getSourceToOutputMap(BuildTarget target) throws IOException { - SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER); - return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target)); + public @NotNull SourceToOutputMapping getSourceToOutputMap(@NotNull BuildTarget target) throws IOException { + int targetId = myTargetsState.getBuildTargetId(target); + if (storageManager == null) { + SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER); + return new SourceToOutputMappingWrapper(map, targetId); + } + else { + return new SourceToOutputMappingWrapper(ExperimentalSourceToOutputMapping.createSourceToOutputMap(storageManager, myRelativizer, target), targetId); + } } @ApiStatus.Internal @@ -464,7 +472,7 @@ public final class BuildDataManager { } @Override - public void setOutputs(@NotNull String srcPath, @NotNull Collection outputs) throws IOException { + public void setOutputs(@NotNull String srcPath, @NotNull List outputs) throws IOException { try { myDelegate.setOutputs(srcPath, outputs); } @@ -503,24 +511,19 @@ public final class BuildDataManager { myDelegate.removeOutput(sourcePath, outputPath); } - @Override - public @NotNull Collection getSources() throws IOException { - return myDelegate.getSources(); - } - @Override public @Nullable Collection getOutputs(@NotNull String srcPath) throws IOException { return myDelegate.getOutputs(srcPath); } @Override - public @NotNull Iterator getOutputsIterator(@NotNull String srcPath) throws IOException { - return myDelegate.getOutputsIterator(srcPath); + public @NotNull Iterator getSourcesIterator() throws IOException { + return myDelegate.getSourcesIterator(); } @Override - public @NotNull Iterator getSourcesIterator() throws IOException { - return myDelegate.getSourcesIterator(); + public @NotNull SourceToOutputMappingCursor cursor() throws IOException { + return myDelegate.cursor(); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt index ca0a47de2b09..c34fcbe93f88 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt @@ -4,53 +4,44 @@ package org.jetbrains.jps.incremental.storage import com.dynatrace.hash4j.hashing.Hashing -import org.h2.mvstore.type.DataType +import org.jetbrains.annotations.ApiStatus import org.jetbrains.jps.incremental.relativizer.PathRelativizerService import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType -import java.io.IOException -import kotlin.Throws -internal interface OneToManyPathMapping : StorageOwner { - @Throws(IOException::class) - fun getState(path: String): Collection? +@ApiStatus.Internal +open class ExperimentalOneToManyPathMapping protected constructor( + @JvmField protected val mapHandle: MapHandle>, + @JvmField protected val relativizer: PathRelativizerService, + private val valueOffset: Int = 0, +) : OneToManyPathMapping, StorageOwner by StorageOwnerByMap(mapHandle = mapHandle) { + constructor( + mapName: String, + storageManager: StorageManager, + relativizer: PathRelativizerService, + ) : this(storageManager.openMap(mapName, LongPairKeyDataType, StringListDataType), relativizer) - @Throws(IOException::class) - fun update(path: String, outPaths: List) - - @Throws(IOException::class) - fun remove(path: String) -} - -internal class ExperimentalOneToManyPathMapping( - mapName: String, - storageManager: StorageManager, - private val relativizer: PathRelativizerService, -) : OneToManyPathMapping, - StorageOwnerByMap>( - mapName = mapName, - storageManager = storageManager, - keyType = LongPairKeyDataType, - valueType = StringListDataType, - ) { - - private fun getKey(path: String): LongArray { - val stringKey = relativizer.toRelative(path).toByteArray() - return longArrayOf(Hashing.xxh3_64().hashBytesToLong(stringKey), Hashing.komihash5_0().hashBytesToLong(stringKey)) - } + protected fun getKey(path: String): LongArray = stringTo128BitHash(relativizer.toRelative(path)) @Suppress("ReplaceGetOrSet") - override fun getState(path: String): Collection? { + final override fun getOutputs(path: String): List? { val key = getKey(path) val list = mapHandle.map.get(key) ?: return null - return Array(list.size) { relativizer.toFull(list.get(it)) }.asList() + return Array(list.size - valueOffset) { relativizer.toFull(list.get(it + valueOffset)) }.asList() } - override fun update(path: String, outPaths: List) { - val key = getKey(path) + final override fun setOutputs(path: String, outPaths: List) { + val relativeSourcePath = relativizer.toRelative(path) + val key = stringTo128BitHash(relativeSourcePath) if (outPaths.isEmpty()) { mapHandle.map.remove(key) } + else if (valueOffset == 1) { + val listWithRelativePaths = Array(outPaths.size + 1) { + if (it == 0) relativeSourcePath else relativizer.toRelative(outPaths.get(it - 1)) + } + mapHandle.map.put(key, listWithRelativePaths) + } else { val listWithRelativePaths = Array(outPaths.size) { relativizer.toRelative(outPaths.get(it)) @@ -59,21 +50,18 @@ internal class ExperimentalOneToManyPathMapping( } } - override fun remove(path: String) { + final override fun remove(path: String) { mapHandle.map.remove(getKey(path)) } } -internal sealed class StorageOwnerByMap( - mapName: String, - storageManager: StorageManager, - keyType: DataType, - valueType: DataType, -) : StorageOwner { - @JvmField - protected val mapHandle = storageManager.openMap(mapName, keyType, valueType) +internal fun stringTo128BitHash(string: String): LongArray { + val bytes = string.toByteArray() + return longArrayOf(Hashing.xxh3_64().hashBytesToLong(bytes), Hashing.komihash5_0().hashBytesToLong(bytes)) +} - final override fun flush(memoryCachesOnly: Boolean) { +internal class StorageOwnerByMap(private val mapHandle: MapHandle) : StorageOwner { + override fun flush(memoryCachesOnly: Boolean) { if (memoryCachesOnly) { // set again to force to clear the cache (in kb) mapHandle.map.store.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024 @@ -83,11 +71,11 @@ internal sealed class StorageOwnerByMap( } } - final override fun clean() { + override fun clean() { mapHandle.map.clear() } - final override fun close() { + override fun close() { mapHandle.release() } } \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt new file mode 100644 index 000000000000..77004ffd543a --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt @@ -0,0 +1,197 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage + +import org.h2.mvstore.MVMap +import org.h2.mvstore.MVMap.Decision +import org.h2.mvstore.MVMap.DecisionMaker +import org.jetbrains.annotations.ApiStatus.Internal +import org.jetbrains.annotations.VisibleForTesting +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.storage.SourceToOutputMapping +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType +import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType + +@Internal +class ExperimentalSourceToOutputMapping private constructor( + mapHandle: MapHandle>, + relativizer: PathRelativizerService, +) : SourceToOutputMapping, ExperimentalOneToManyPathMapping(mapHandle = mapHandle, relativizer = relativizer, valueOffset = 1) { + companion object { + // we have a lot of targets - reduce GC and reuse map builder + private val mapBuilder = MVMap.Builder>().also { + it.setKeyType(LongPairKeyDataType) + it.setValueType(StringListDataType) + } + + @JvmStatic + fun createSourceToOutputMap( + storageManager: StorageManager, + relativizer: PathRelativizerService, + target: BuildTarget<*>, + ): ExperimentalSourceToOutputMapping { + return createSourceToOutputMap( + storageManager = storageManager, + relativizer = relativizer, + targetId = target.id, + targetTypeId = target.targetType.typeId, + ) + } + + @VisibleForTesting + @Internal + fun createSourceToOutputMap( + storageManager: StorageManager, + relativizer: PathRelativizerService, + targetId: String, + targetTypeId: String, + ): ExperimentalSourceToOutputMapping { + // we can use composite key and sort by target id, but as we compile targets in parallel: + // * avoid blocking - in-memory lock per map root, + // * avoid a huge B-tree and reduce rebalancing time due to contention. + return ExperimentalSourceToOutputMapping( + mapHandle = storageManager.openMap("$targetId|$targetTypeId|src-to-out", mapBuilder), + relativizer = relativizer, + ) + } + } + + //override fun getKey(sourcePath: String): LongArray { + // val stringKey = relativizer.toRelative(sourcePath).toByteArray() + // + // // We should sort by target id for data locality - + // // the high integer (high) is shifted to the higher 32 bits of the long value, + // // while the low integer (low) occupies the lower 32 bits. + // // When we do compare two long values, the higher 32 bits (which contain the high integer) dictate the order. + // val low = Hashing.komihash5_0().hashBytesToInt(stringKey) + // val high = targetId + // return longArrayOf((high.toLong() shl 32) or (low.toLong() and 0xFFFFFFFFL), Hashing.xxh3_64().hashBytesToLong(stringKey)) + //} + + override fun setOutput(sourcePath: String, outputPath: String) { + val relativeSourcePath = relativizer.toRelative(sourcePath) + mapHandle.map.put(stringTo128BitHash(relativeSourcePath), arrayOf(relativeSourcePath, relativizer.toRelative(outputPath))) + } + + override fun appendOutput(sourcePath: String, outputPath: String) { + val relativeSourcePath = relativizer.toRelative(sourcePath) + mapHandle.map.operate(stringTo128BitHash(relativeSourcePath), + null, + AddItemDecisionMaker(sourcePath = relativeSourcePath, toAdd = relativizer.toRelative(outputPath))) + } + + override fun removeOutput(sourcePath: String, outputPath: String) { + mapHandle.map.operate(getKey(sourcePath), null, RemoveItemDecisionMaker(relativizer.toRelative(outputPath))) + } + + override fun cursor(): SourceToOutputMappingCursor { + val cursor = mapHandle.map.cursor(null) + return object : SourceToOutputMappingCursor { + override fun hasNext(): Boolean = cursor.hasNext() + + override fun next(): String { + cursor.next() + return relativizer.toFull(cursor.value.first()) + } + + override val outputPaths: Array + get() { + val list = cursor.value + return Array(list.size - 1) { relativizer.toFull(list[it + 1]) } + } + } + } + + override fun getSourcesIterator(): Iterator = cursor() +} + +private val CHECK_COLLISIONS = System.getProperty("jps.source.to.output.mapping.check.collisions", "false").toBoolean() + +private class AddItemDecisionMaker(private val sourcePath: String, private val toAdd: String) : DecisionMaker>() { + override fun decide(existingValue: Array?, providedValue: Array?): Decision { + when { + existingValue == null -> { + return Decision.PUT + } + existingValue.isEmpty() -> { + if (CHECK_COLLISIONS) { + throw IllegalStateException("Value is empty") + } + return Decision.PUT + } + CHECK_COLLISIONS && existingValue[0] != sourcePath -> { + throw IllegalStateException("Collision for $sourcePath: ${existingValue[0]} and $sourcePath") + } + else -> { + for (i in 1 until existingValue.size) { + if (existingValue[i] == toAdd) { + return Decision.ABORT + } + } + return Decision.PUT + } + } + } + + // it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel + override fun ?> selectValue(existingValue: T?, ignore: T?): T? { + if (existingValue == null || existingValue.isEmpty()) { + @Suppress("UNCHECKED_CAST") + return arrayOf(sourcePath, toAdd) as T + } + else { + // we checked `contains` in `decide` + @Suppress("UNCHECKED_CAST") + return addElementToEnd(existingValue, toAdd) as T? + } + } +} + +private class RemoveItemDecisionMaker(private val toRemove: String) : DecisionMaker>() { + private var indexToRemove: Int = -1 + + override fun decide(existingValue: Array?, ignore: Array?): Decision { + return when { + existingValue == null -> Decision.ABORT + // empty value list is not normal, recover - just delete record + existingValue.size == 1 -> Decision.REMOVE + else -> { + for (i in 1 until existingValue.size) { + if (existingValue[i] == toRemove) { + indexToRemove = i + break + } + } + when { + indexToRemove == -1 -> Decision.ABORT + existingValue.size == 2 -> Decision.REMOVE + else -> Decision.PUT + } + } + } + } + + // it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel + override fun ?> selectValue(existingValue: T, providedValue: T?): T { + assert(indexToRemove != -1) + @Suppress("UNCHECKED_CAST") + return removeElementAtIndex(existingValue!!, indexToRemove) as T + } +} + +private fun removeElementAtIndex(old: Array, index: Int): Array { + val newSize = old.size - 1 + val result = arrayOfNulls(newSize) + System.arraycopy(old, 0, result, 0, index) + if (index < newSize) { + System.arraycopy(old, index + 1, result, index, newSize - index) + } + return result +} + +private fun addElementToEnd(old: Array, element: String): Array { + val result = arrayOfNulls(old.size + 1) + System.arraycopy(old, 0, result, 0, old.size) + result[old.size] = element + return result +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt new file mode 100644 index 000000000000..fc99c89dd584 --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt @@ -0,0 +1,22 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage + +import org.jetbrains.annotations.ApiStatus +import java.io.IOException + +internal interface OneToManyPathMapping : StorageOwner { + @Throws(IOException::class) + fun getOutputs(path: String): Collection? + + @Throws(IOException::class) + fun setOutputs(path: String, outPaths: List) + + @Throws(IOException::class) + fun remove(path: String) +} + +@ApiStatus.Internal +interface SourceToOutputMappingCursor : Iterator { + /** [next] must be called beforehand */ + val outputPaths: Array +} \ No newline at end of file diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java index 43f422c99327..441fb901481d 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java @@ -1,19 +1,17 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.incremental.storage; +import com.intellij.util.ArrayUtilRt; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.IOUtil; import com.intellij.util.io.PersistentMapBuilder; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Unmodifiable; import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; -import org.jetbrains.jps.javac.Iterators; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.IOException; +import java.io.*; import java.nio.file.Path; import java.util.*; @@ -29,11 +27,11 @@ public final class OneToManyPathsMapping extends AbstractStateStorage boundPaths) throws IOException { + public void setOutputs(@NotNull String keyPath, @NotNull List boundPaths) throws IOException { super.update(normalizePath(keyPath), normalizePaths(boundPaths)); } - public void update(@NotNull String keyPath, @NotNull String boundPath) throws IOException { + void setOutput(@NotNull String keyPath, @NotNull String boundPath) throws IOException { super.update(normalizePath(keyPath), List.of(normalizePath(boundPath))); } @@ -46,12 +44,24 @@ public final class OneToManyPathsMapping extends AbstractStateStorage)boundPaths)); } + /** + * @deprecated Use {@link #getOutputs(String)} + */ + @Deprecated(forRemoval = true) @Override public @Nullable Collection getState(@NotNull String keyPath) throws IOException { + return getOutputs(keyPath); + } + + @Override + public @Nullable List getOutputs(@NotNull String keyPath) throws IOException { List collection = (List)super.getState(relativizer.toRelative(keyPath)); - if (collection == null || collection.isEmpty()) { + if (collection == null) { return null; } + else if (collection.isEmpty()) { + return Collections.emptyList(); + } else { String[] result = new String[collection.size()]; for (int i = 0, size = collection.size(); i < size; i++) { @@ -61,9 +71,21 @@ public final class OneToManyPathsMapping extends AbstractStateStorage getStateIterator(@NotNull String keyPath) throws IOException { + public String @Nullable [] getOutputArray(@NotNull String keyPath) throws IOException { List collection = (List)super.getState(relativizer.toRelative(keyPath)); - return collection == null ? Collections.emptyIterator() : Iterators.map(collection.iterator(), relativizer::toFull); + if (collection == null) { + return null; + } + else if (collection.isEmpty()) { + return ArrayUtilRt.EMPTY_STRING_ARRAY; + } + else { + String[] result = new String[collection.size()]; + for (int i = 0, size = collection.size(); i < size; i++) { + result[i] = relativizer.toFull(collection.get(i)); + } + return result; + } } @Override @@ -72,22 +94,13 @@ public final class OneToManyPathsMapping extends AbstractStateStorage getKeys() throws IOException { - List collection = (List)super.getKeys(); - if (collection.isEmpty()) { - return List.of(); - } - - String[] result = new String[collection.size()]; - for (int i = 0; i < collection.size(); i++) { - result[i] = relativizer.toFull(collection.get(i)); - } - return Arrays.asList(result); + public @NotNull Iterator getKeysIterator() throws IOException { + return super.getKeyIterator(relativizer::toFull); } - @Override - public Iterator getKeysIterator() throws IOException { - return Iterators.map(super.getKeysIterator(), relativizer::toFull); + @ApiStatus.Internal + public @NotNull SourceToOutputMappingCursor cursor() throws IOException { + return new SourceToOutputMappingCursorImpl(getKeysIterator()); } public void removeData(@NotNull String keyPath, @NotNull String boundPath) throws IOException { @@ -151,4 +164,32 @@ public final class OneToManyPathsMapping extends AbstractStateStorage mySourceIterator; + private String sourcePath; + + private SourceToOutputMappingCursorImpl(Iterator sourceIterator) { mySourceIterator = sourceIterator; } + + @Override + public boolean hasNext() { + return mySourceIterator.hasNext(); + } + + @Override + public @NotNull String next() { + sourcePath = mySourceIterator.next(); + return sourcePath; + } + + @Override + public String @NotNull [] getOutputPaths() { + try { + return Objects.requireNonNullElse(getOutputArray(sourcePath), ArrayUtilRt.EMPTY_STRING_ARRAY); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java index 7661ecb19e87..6b1316f62a07 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java @@ -9,8 +9,8 @@ import org.jetbrains.jps.incremental.relativizer.PathRelativizerService; import java.io.IOException; import java.nio.file.Path; -import java.util.Collection; import java.util.Iterator; +import java.util.List; @ApiStatus.Internal public final class SourceToOutputMappingImpl implements SourceToOutputMapping, StorageOwner { @@ -21,13 +21,13 @@ public final class SourceToOutputMappingImpl implements SourceToOutputMapping, S } @Override - public void setOutputs(@NotNull String srcPath, @NotNull Collection outputs) throws IOException { - myMapping.update(srcPath, outputs); + public void setOutputs(@NotNull String srcPath, @NotNull List outputs) throws IOException { + myMapping.setOutputs(srcPath, outputs); } @Override public void setOutput(@NotNull String srcPath, @NotNull String outputPath) throws IOException { - myMapping.update(srcPath, outputPath); + myMapping.setOutput(srcPath, outputPath); } @Override @@ -46,18 +46,8 @@ public final class SourceToOutputMappingImpl implements SourceToOutputMapping, S } @Override - public @NotNull Collection getSources() throws IOException { - return myMapping.getKeys(); - } - - @Override - public @Nullable Collection getOutputs(@NotNull String srcPath) throws IOException { - return myMapping.getState(srcPath); - } - - @Override - public @NotNull Iterator getOutputsIterator(@NotNull String srcPath) throws IOException { - return myMapping.getStateIterator(srcPath); + public @Nullable List getOutputs(@NotNull String srcPath) throws IOException { + return myMapping.getOutputs(srcPath); } @Override @@ -65,6 +55,11 @@ public final class SourceToOutputMappingImpl implements SourceToOutputMapping, S return myMapping.getKeysIterator(); } + @Override + public @NotNull SourceToOutputMappingCursor cursor() throws IOException { + return myMapping.cursor(); + } + @Override public void flush(boolean memoryCachesOnly) { myMapping.flush(memoryCachesOnly); diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java index 6686e7fc3a36..295c1b90916b 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java @@ -1,13 +1,14 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.builders; import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; -import com.intellij.util.ObjectUtils; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntSet; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.java.dependencyView.Mappings; import org.jetbrains.jps.builders.storage.SourceToOutputMapping; @@ -20,10 +21,7 @@ import org.jetbrains.jps.incremental.storage.OutputToTargetRegistry; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; +import java.util.*; import static org.junit.Assert.*; @@ -64,15 +62,15 @@ public final class BuildResult implements MessageHandler { for (BuildTarget target : targets) { id2Target.put(pd.dataManager.getTargetsState().getBuildTargetId(target), target); } - Int2ObjectMap hashCodeToOutputPath = new Int2ObjectOpenHashMap<>(); + Int2ObjectOpenHashMap hashCodeToOutputPath = new Int2ObjectOpenHashMap<>(); for (BuildTarget target : targets) { stream.println("Begin Of SourceToOutput (target " + getTargetIdWithTypeId(target) + ")"); SourceToOutputMapping map = pd.dataManager.getSourceToOutputMap(target); - List sourcesList = new ArrayList<>(map.getSources()); - Collections.sort(sourcesList); - for (String source : sourcesList) { - List outputs = new ArrayList<>(ObjectUtils.notNull(map.getOutputs(source), Collections.emptySet())); - Collections.sort(outputs); + List sourceList = new ObjectArrayList<>(map.getSourcesIterator()); + sourceList.sort(null); + for (String source : sourceList) { + List outputs = new ArrayList<>(Objects.requireNonNullElse(map.getOutputs(source), Collections.emptyList())); + outputs.sort(null); for (String output : outputs) { hashCodeToOutputPath.put(FileUtil.pathHashCode(output), output); } @@ -84,7 +82,7 @@ public final class BuildResult implements MessageHandler { OutputToTargetRegistry registry = pd.dataManager.getOutputToTargetRegistry(); - List keys = new ArrayList<>(registry.getKeys()); + List keys = new IntArrayList(registry.getKeysIterator()); Collections.sort(keys); stream.println("Begin Of OutputToTarget"); for (Integer key : keys) { @@ -96,7 +94,7 @@ public final class BuildResult implements MessageHandler { targetsNames.add(target != null ? getTargetIdWithTypeId(target) : ""); }); Collections.sort(targetsNames); - stream.println(hashCodeToOutputPath.get(key) + " -> " + targetsNames); + stream.println(hashCodeToOutputPath.get(key.intValue()) + " -> " + targetsNames); } stream.println("End Of OutputToTarget"); } diff --git a/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt new file mode 100644 index 000000000000..084d54068b2d --- /dev/null +++ b/jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt @@ -0,0 +1,115 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.incremental.storage + +import net.jqwik.api.* +import net.jqwik.api.lifecycle.AfterProperty +import net.jqwik.api.lifecycle.BeforeProperty +import org.assertj.core.api.Assertions.assertThat +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import java.nio.file.Files +import java.nio.file.Path +import kotlin.random.Random + +class SourceToOutputMappingFuzzTest { + companion object { + init { + System.setProperty("jps.source.to.output.mapping.check.collisions", "true") + } + } + + private lateinit var mapping: ExperimentalSourceToOutputMapping + private var file: Path? = null + + @BeforeProperty + fun setUp() { + file = Files.createTempFile("mvstore", ".db") + val storageManager = StorageManager(file!!, 0) + mapping = ExperimentalSourceToOutputMapping.createSourceToOutputMap( + storageManager = storageManager, + relativizer = PathRelativizerService(), + targetId = "test-module", + targetTypeId = "java" + ) + } + + @AfterProperty + fun tearDown() { + try { + mapping.close() + } + finally { + file?.let { Files.deleteIfExists(it) } + } + } + + @Provide + fun pathStrings(): Arbitrary { + return Arbitraries.strings().alpha().numeric().withChars('/').ofMinLength(2).ofMaxLength(255) + } + + @Suppress("unused") + @Provide + fun pathStringLists(): Arbitrary> { + return pathStrings().list().ofMinSize(1).ofMaxSize(32) + } + + @Suppress("unused") + @Provide + fun pathStringListsList(): Arbitrary>> { + return pathStrings().list().ofMinSize(1).ofMaxSize(32).list().ofMinSize(1).ofMaxSize(64) + } + + @Property + fun setOutputs(@ForAll("pathStrings") source: String, @ForAll("pathStringLists") outputs: List) { + mapping.setOutputs(source, outputs) + val result = mapping.getOutputs(source) + assertThat(result).isNotNull() + assertThat(result).containsExactlyInAnyOrderElementsOf(outputs) + + checkCursorAndSourceIterator(source, outputs) + } + + @Property + fun appendOutput(@ForAll("pathStrings") source: String, @ForAll("pathStrings") output: String) { + mapping.appendOutput(source, output) + val outputs = mapping.getOutputs(source) + assertThat(outputs).isNotNull + assertThat(outputs).contains(output) + + checkCursorAndSourceIterator(source, outputs!!) + } + + @Property + fun cursor(@ForAll("pathStringLists") sources: List, @ForAll("pathStringListsList") outputs: List>) { + mapping.clean() + + val expectedMap = HashMap>() + for (source in sources) { + val list = outputs[Random.nextInt(outputs.size)] + expectedMap.put(source, list) + mapping.setOutputs(source, list) + } + + val actualMap = LinkedHashMap>() + val cursor = mapping.cursor() + while (cursor.hasNext()) { + actualMap.put(cursor.next(), cursor.outputPaths.asList()) + } + + assertThat(actualMap).isEqualTo(expectedMap) + } + + private fun checkCursorAndSourceIterator(source: String, outputs: List) { + val cursor = mapping.cursor() + val map = LinkedHashMap>() + while (cursor.hasNext()) { + val next = cursor.next() + if (next == source) { + map.put(source, cursor.outputPaths.asList()) + } + } + assertThat(map).isEqualTo(mapOf(source to outputs)) + + assertThat(mapping.sourcesIterator.asSequence().contains(source)).isTrue() + } +} \ No newline at end of file diff --git a/plugins/groovy/jps-plugin/intellij.groovy.jps.iml b/plugins/groovy/jps-plugin/intellij.groovy.jps.iml index c9ba0352b953..1d252090f0ce 100644 --- a/plugins/groovy/jps-plugin/intellij.groovy.jps.iml +++ b/plugins/groovy/jps-plugin/intellij.groovy.jps.iml @@ -34,5 +34,6 @@ + \ No newline at end of file diff --git a/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/JpsGroovycRunner.java b/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/JpsGroovycRunner.java index 561e51ccf325..49e9f464daaf 100644 --- a/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/JpsGroovycRunner.java +++ b/plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/JpsGroovycRunner.java @@ -28,6 +28,7 @@ import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode; import org.jetbrains.jps.incremental.java.JavaBuilder; import org.jetbrains.jps.incremental.messages.BuildMessage; import org.jetbrains.jps.incremental.messages.CompilerMessage; +import org.jetbrains.jps.incremental.storage.SourceToOutputMappingCursor; import org.jetbrains.jps.model.JpsDummyElement; import org.jetbrains.jps.model.java.JpsJavaExtensionService; import org.jetbrains.jps.model.java.JpsJavaSdkType; @@ -434,17 +435,15 @@ public abstract class JpsGroovycRunner outs = srcToOut.getOutputs(src); - if (outs != null) { - for (String out : outs) { - if (out.endsWith(".class") && out.startsWith(moduleOutputPath)) { - final String className = out.substring(moduleOutputPath.length(), out.length() - ".class".length()).replace('/', '.'); - class2Src.put(className, src); - } + for (String out : cursor.getOutputPaths()) { + if (out.endsWith(".class") && out.startsWith(moduleOutputPath)) { + final String className = out.substring(moduleOutputPath.length(), out.length() - ".class".length()).replace('/', '.'); + class2Src.put(className, src); } } } diff --git a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java index 6269fd4d47f1..977aa0aa93a8 100644 --- a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java +++ b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java @@ -162,7 +162,7 @@ public final class FormsBindingManager extends FormsBuilder { for (Map.Entry entry : filesToCompile.entrySet()) { final File srcFile = entry.getKey(); final ModuleBuildTarget target = entry.getValue(); - final Collection boundForms = sourceToFormMap.getState(srcFile.getPath()); + final Collection boundForms = sourceToFormMap.getOutputs(srcFile.getPath()); if (boundForms != null) { for (String formPath : boundForms) { final File formFile = new File(formPath); diff --git a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java index 0c3e7bbbf032..92c6e0a6d52f 100644 --- a/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java +++ b/plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java @@ -98,7 +98,7 @@ public final class FormsInstrumenter extends FormsBuilder { for (File form : forms) { formPaths.add(form.getPath()); } - sourceToFormMap.update(src.getPath(), formPaths); + sourceToFormMap.setOutputs(src.getPath(), formPaths); srcToForms.remove(src); } // clean mapping