experimental compact storage for JPS Cache (part 2 - finish ExperimentalOneToManyPathMapping and add test)

GitOrigin-RevId: 6c8dadfc2fb7ad6c6ac39f31265c77cbbae9876b

.gitignore

@@ -5,6 +5,7 @@
 .idea/libraries/Gradle__*.xml
 .idea/shelf
 .idea/workspace.xml
+.jqwik-database
 /.diff
 /config
 /out

java/compiler/impl/src/com/intellij/compiler/server/BuildManager.java

@@ -174,6 +174,8 @@ public final class BuildManager implements Disposable {
     s -> !(s.contains(IDEA_PROJECT_DIR_PATTERN) || s.endsWith(IWS_EXTENSION) || s.endsWith(IPR_EXTENSION)) :
     s -> !(Strings.endsWithIgnoreCase(s, IWS_EXTENSION) || Strings.endsWithIgnoreCase(s, IPR_EXTENSION) || StringUtil.containsIgnoreCase(s, IDEA_PROJECT_DIR_PATTERN));
 
+  private static final String JPS_USE_EXPERIMENTAL_STORAGE = "jps.use.experimental.storage";
+
   private final String myFallbackSdkHome;
   private final String myFallbackSdkVersion;
 
@@ -1409,6 +1411,9 @@ public final class BuildManager implements Disposable {
     }
 
     cmdLine.addParameter("-Djava.awt.headless=true");
+    if (Boolean.getBoolean(JPS_USE_EXPERIMENTAL_STORAGE)) {
+      cmdLine.addParameter("-D" + JPS_USE_EXPERIMENTAL_STORAGE + "=true");
+    }
 
     String jnaBootLibraryPath = System.getProperty("jna.boot.library.path");
     if (jnaBootLibraryPath != null && wslPath == null) {

jps/jps-builders/api-dump.txt

@@ -1836,16 +1836,17 @@ org.jetbrains.jps.builders.storage.BuildDataPaths
 org.jetbrains.jps.builders.storage.SourceToOutputMapping
 - a:appendOutput(java.lang.String,java.lang.String):V
 - a:getOutputs(java.lang.String):java.util.Collection
-- a:getOutputsIterator(java.lang.String):java.util.Iterator
+- getSources():java.util.Collection
-- a:getSources():java.util.Collection
 - a:getSourcesIterator():java.util.Iterator
 - a:remove(java.lang.String):V
 - a:removeOutput(java.lang.String,java.lang.String):V
 - a:setOutput(java.lang.String,java.lang.String):V
-- a:setOutputs(java.lang.String,java.util.Collection):V
+- a:setOutputs(java.lang.String,java.util.List):V
 a:org.jetbrains.jps.builders.storage.StorageProvider
 - <init>():V
-- a:createStorage(java.nio.file.Path):org.jetbrains.jps.incremental.storage.StorageOwner
+- createStorage(java.io.File):org.jetbrains.jps.incremental.storage.StorageOwner
+- createStorage(java.io.File,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):org.jetbrains.jps.incremental.storage.StorageOwner
+- createStorage(java.nio.file.Path):org.jetbrains.jps.incremental.storage.StorageOwner
 - createStorage(java.nio.file.Path,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):org.jetbrains.jps.incremental.storage.StorageOwner
 f:org.jetbrains.jps.cmdline.BuildRunner
 - <init>(org.jetbrains.jps.cmdline.JpsModelLoader):V
@@ -2609,7 +2610,7 @@ a:org.jetbrains.jps.incremental.storage.AbstractStateStorage
 - f:close():V
 - flush(Z):V
 - f:force():V
-- getKeys():java.util.Collection
+- pf:getKeyIterator(java.util.function.Function):java.util.Iterator
 - getKeysIterator():java.util.Iterator
 - getState(java.lang.Object):java.lang.Object
 - remove(java.lang.Object):V
@@ -2686,14 +2687,14 @@ f:org.jetbrains.jps.incremental.storage.OneToManyPathsMapping
 - <init>(java.nio.file.Path,org.jetbrains.jps.incremental.relativizer.PathRelativizerService):V
 - appendData(java.lang.String,java.lang.String):V
 - appendData(java.lang.String,java.util.Collection):V
-- getKeys():java.util.List
 - getKeysIterator():java.util.Iterator
+- getOutputArray(java.lang.String):java.lang.String[]
+- a:getOutputs(java.lang.String):java.util.Collection
+- getOutputs(java.lang.String):java.util.List
 - getState(java.lang.String):java.util.Collection
-- getStateIterator(java.lang.String):java.util.Iterator
 - remove(java.lang.String):V
 - removeData(java.lang.String,java.lang.String):V
-- update(java.lang.String,java.lang.String):V
+- setOutputs(java.lang.String,java.util.List):V
-- update(java.lang.String,java.util.List):V
 f:org.jetbrains.jps.incremental.storage.OutputToTargetRegistry
 - org.jetbrains.jps.incremental.storage.AbstractStateStorage
 - getSafeToDeleteOutputs(java.util.Collection,I):java.util.Collection

jps/jps-builders/intellij.platform.jps.build.tests.iml

@@ -20,5 +20,54 @@
     <orderEntry type="module" module-name="intellij.platform.ide.util.io" scope="TEST" />
     <orderEntry type="library" scope="TEST" name="assertJ" level="project" />
     <orderEntry type="library" scope="TEST" name="hash4j" level="project" />
+    <orderEntry type="module-library" scope="TEST">
+      <library name="jqwik" type="repository">
+        <properties maven-id="net.jqwik:jqwik:1.9.0">
+          <verification>
+            <artifact url="file://$MAVEN_REPOSITORY$/net/jqwik/jqwik/1.9.0/jqwik-1.9.0.jar">
+              <sha256sum>bcbbc834752835d9c534647f925756de2cd6f51a1e94fcf6162fd55f3fc2c706</sha256sum>
+            </artifact>
+            <artifact url="file://$MAVEN_REPOSITORY$/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar">
+              <sha256sum>b509448ac506d607319f182537f0b35d71007582ec741832a1f111e5b5b70b38</sha256sum>
+            </artifact>
+            <artifact url="file://$MAVEN_REPOSITORY$/net/jqwik/jqwik-api/1.9.0/jqwik-api-1.9.0.jar">
+              <sha256sum>1ebd46a1df6575c3d46ba263a6575c88fe654f164f9e3d1f7fbc05fdc9dc726c</sha256sum>
+            </artifact>
+            <artifact url="file://$MAVEN_REPOSITORY$/net/jqwik/jqwik-web/1.9.0/jqwik-web-1.9.0.jar">
+              <sha256sum>67030794b96c3e854795c6e709b1e9adc54df93474cb8e8380a791d58db2560e</sha256sum>
+            </artifact>
+            <artifact url="file://$MAVEN_REPOSITORY$/net/jqwik/jqwik-time/1.9.0/jqwik-time-1.9.0.jar">
+              <sha256sum>4258ec1239879e520ac6df63de21b821691cf57479a14700c6ab8bd98cba1fa5</sha256sum>
+            </artifact>
+            <artifact url="file://$MAVEN_REPOSITORY$/net/jqwik/jqwik-engine/1.9.0/jqwik-engine-1.9.0.jar">
+              <sha256sum>6eb6ffbfcf81f5eef65cffdd41186f961f1a0fcb3c44f17e4c09d5d3aa773b2c</sha256sum>
+            </artifact>
+          </verification>
+          <exclude>
+            <dependency maven-id="org.opentest4j:opentest4j" />
+            <dependency maven-id="org.junit.platform:junit-platform-commons" />
+            <dependency maven-id="org.junit.platform:junit-platform-engine" />
+          </exclude>
+        </properties>
+        <CLASSES>
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik/1.9.0/jqwik-1.9.0.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-api/1.9.0/jqwik-api-1.9.0.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-web/1.9.0/jqwik-web-1.9.0.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-time/1.9.0/jqwik-time-1.9.0.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-engine/1.9.0/jqwik-engine-1.9.0.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik/1.9.0/jqwik-1.9.0-sources.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2-sources.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-api/1.9.0/jqwik-api-1.9.0-sources.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-web/1.9.0/jqwik-web-1.9.0-sources.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-time/1.9.0/jqwik-time-1.9.0-sources.jar!/" />
+          <root url="jar://$MAVEN_REPOSITORY$/net/jqwik/jqwik-engine/1.9.0/jqwik-engine-1.9.0-sources.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="library" scope="TEST" name="mvstore" level="project" />
   </component>
 </module>

jps/jps-builders/src/org/jetbrains/jps/builders/storage/SourceToOutputMapping.java

@@ -1,49 +1,48 @@
-/*
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
- * Copyright 2000-2012 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package org.jetbrains.jps.builders.storage;
 
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.jps.incremental.storage.SourceToOutputMappingCursor;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.List;
 
 public interface SourceToOutputMapping {
-  void setOutputs(@NotNull String srcPath, @NotNull Collection<String> outputs) throws IOException;
+  void setOutputs(@NotNull String srcPath, @NotNull List<String> outputs) throws IOException;
 
   void setOutput(@NotNull String srcPath, @NotNull String outputPath) throws IOException;
 
   void appendOutput(@NotNull String srcPath, @NotNull String outputPath) throws IOException;
 
-
   void remove(@NotNull String srcPath) throws IOException;
 
   void removeOutput(@NotNull String sourcePath, @NotNull String outputPath) throws IOException;
 
-
+  /**
-  @NotNull
+   * @deprecated Use {@link #getSourcesIterator()}
-  Collection<String> getSources() throws IOException;
+   */
+  @Deprecated(forRemoval = true)
+  default @NotNull Collection<String> getSources() throws IOException {
+    List<String> result = new ArrayList<>();
+    Iterator<String> iterator = getSourcesIterator();
+    while (iterator.hasNext()) {
+      result.add(iterator.next());
+    }
+    return result;
+  }
 
   @Nullable
   Collection<String> getOutputs(@NotNull String srcPath) throws IOException;
 
   @NotNull
-  Iterator<String> getOutputsIterator(@NotNull String srcPath) throws IOException;
+  Iterator<String> getSourcesIterator() throws IOException;
 
   @NotNull
-  Iterator<String> getSourcesIterator() throws IOException;
+  @ApiStatus.Internal
+  SourceToOutputMappingCursor cursor() throws IOException;
 }

jps/jps-builders/src/org/jetbrains/jps/incremental/BuildOperations.java

@@ -1,4 +1,4 @@
-// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.jps.incremental;
 
 import com.intellij.openapi.util.io.FileUtilRt;
@@ -178,14 +178,14 @@ public final class BuildOperations {
 
       });
 
-      if (JavaBuilderUtil.isCompileJavaIncrementally(context)) {
-        final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger();
-        if (logger.isEnabled()) {
-          logger.logDeletedFiles(deletedPaths);
-        }
-      }
-
       if (!deletedPaths.isEmpty()) {
+        if (JavaBuilderUtil.isCompileJavaIncrementally(context)) {
+          final ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger();
+          if (logger.isEnabled()) {
+            logger.logDeletedFiles(deletedPaths);
+          }
+        }
+
         context.processMessage(new FileDeletedEvent(deletedPaths));
       }
       // attempting to delete potentially empty directories

jps/jps-builders/src/org/jetbrains/jps/incremental/IncProjectBuilder.java

@@ -728,14 +728,15 @@ public final class IncProjectBuilder {
                                        int targetId) throws IOException {
     Set<File> dirsToDelete = targetType instanceof ModuleBasedBuildTargetType<?> ? FileCollectionFactory.createCanonicalFileSet() : null;
     OutputToTargetRegistry outputToTargetRegistry = context.getProjectDescriptor().dataManager.getOutputToTargetRegistry();
-    for (String srcPath : mapping.getSources()) {
+    for (SourceToOutputMappingCursor cursor = mapping.cursor(); cursor.hasNext(); ) {
-      final Collection<String> outs = mapping.getOutputs(srcPath);
+      cursor.next();
-      if (outs != null && !outs.isEmpty()) {
+      String [] outs = cursor.getOutputPaths();
+      if (outs.length > 0) {
         List<String> deletedPaths = new ArrayList<>();
         for (String out : outs) {
           BuildOperations.deleteRecursively(out, deletedPaths, dirsToDelete);
         }
-        outputToTargetRegistry.removeMapping(outs, targetId);
+        outputToTargetRegistry.removeMapping(Arrays.asList(outs), targetId);
         if (!deletedPaths.isEmpty()) {
           context.processMessage(new FileDeletedEvent(deletedPaths));
         }
@@ -1309,8 +1310,12 @@ public final class IncProjectBuilder {
               // Change of only one input of *.kotlin_module files didn't trigger recompilation of all inputs in old behaviour.
               // Now it does. It isn't yet obvious whether it is right or wrong behaviour. Let's leave old behaviour for a
               // while for safety and keeping kotlin incremental JPS tests green
-              List<String> filteredOuts =
+              List<String> filteredOuts = new ArrayList<>();
-                ContainerUtil.filter(outs, out -> !"kotlin_module".equals(StringUtil.substringAfterLast(out, ".")));
+              for (String out : outs) {
+                if (!"kotlin_module".equals(StringUtil.substringAfterLast(out, "."))) {
+                  filteredOuts.add(out);
+                }
+              }
               affectedOutputs.addAll(filteredOuts);
             }
           }
@@ -1321,10 +1326,11 @@ public final class IncProjectBuilder {
 
     if (!affectedOutputs.isEmpty()) {
       for (SourceToOutputMapping srcToOut : mappings) {
-        for (String src : srcToOut.getSources()) {
+        for (SourceToOutputMappingCursor cursor = srcToOut.cursor(); cursor.hasNext(); ) {
+          String src = cursor.next();
           if (!affectedSources.contains(src)) {
-            for (Iterator<String> it = srcToOut.getOutputsIterator(src); it.hasNext(); ) {
+            for (String out : cursor.getOutputPaths()) {
-              if (affectedOutputs.contains(it.next())) {
+              if (affectedOutputs.contains(out)) {
                 FSOperations.markDirtyIfNotDeleted(context, CompilationRound.CURRENT, new File(src));
                 break;
               }
@@ -1508,7 +1514,7 @@ public final class IncProjectBuilder {
           if (target instanceof ModuleBuildTarget) {
             // check if the deleted source was associated with a form
             final OneToManyPathMapping sourceToFormMap = context.getProjectDescriptor().dataManager.getSourceToFormMap();
-            final Collection<String> boundForms = sourceToFormMap.getState(deletedSource);
+            final Collection<String> boundForms = sourceToFormMap.getOutputs(deletedSource);
             if (boundForms != null) {
               for (String formPath : boundForms) {
                 final File formFile = new File(formPath);

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java

@@ -12,9 +12,10 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.Function;
 
 public abstract class AbstractStateStorage<Key, T> implements StorageOwner {
   private static final boolean DO_COMPRESS = Boolean.parseBoolean(System.getProperty("jps.storage.do.compression", "true"));
@@ -106,15 +107,7 @@ public abstract class AbstractStateStorage<Key, T> implements StorageOwner {
     }
   }
 
-  public Collection<Key> getKeys() throws IOException {
+  public @NotNull Iterator<Key> getKeysIterator() throws IOException {
-    synchronized (dataLock) {
-      List<Key> result = new ArrayList<>();
-      map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result));
-      return result;
-    }
-  }
-
-  public Iterator<Key> getKeysIterator() throws IOException {
     synchronized (dataLock) {
       List<Key> result = new ArrayList<>();
       map.processExistingKeys(new CommonProcessors.CollectProcessor<>(result));
@@ -122,6 +115,17 @@ public abstract class AbstractStateStorage<Key, T> implements StorageOwner {
     }
   }
 
+  protected final @NotNull Iterator<Key> getKeyIterator(@NotNull Function<Key, Key> mapper) throws IOException {
+    synchronized (dataLock) {
+      List<Key> result = new ArrayList<>();
+      map.processExistingKeys(key -> {
+        result.add(mapper.apply(key));
+        return true;
+      });
+      return result.isEmpty() ? Collections.emptyIterator() : result.iterator();
+    }
+  }
+
   private @NotNull PersistentMapImpl<Key, T> createMap() throws IOException {
     Files.createDirectories(mapBuilder.getFile().getParent());
     return new PersistentMapImpl<>(mapBuilder, new PersistentHashMapValueStorage.CreationTimeOptions(false, false, false, isCompressed));

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/BuildDataManager.java

@@ -61,6 +61,7 @@ public final class BuildDataManager {
   private final NodeSourcePathMapper myDepGraphPathMapper;
   private final BuildDataPaths myDataPaths;
   private final BuildTargetsState myTargetsState;
+  @Nullable private final StorageManager storageManager;
   private final OutputToTargetRegistry myOutputToTargetRegistry;
   private final File myVersionFile;
   private final PathRelativizerService myRelativizer;
@@ -85,6 +86,7 @@ public final class BuildDataManager {
                           @Nullable StorageManager storageManager) throws IOException {
     myDataPaths = dataPaths;
     myTargetsState = targetsState;
+    this.storageManager = storageManager;
     try {
       if (storageManager == null) {
         mySrcToFormMap = new OneToManyPathsMapping(getSourceToFormsRoot().resolve("data"), relativizer);
@@ -139,9 +141,15 @@ public final class BuildDataManager {
     return myOutputToTargetRegistry;
   }
 
-  public SourceToOutputMapping getSourceToOutputMap(BuildTarget<?> target) throws IOException {
+  public @NotNull SourceToOutputMapping getSourceToOutputMap(@NotNull BuildTarget<?> target) throws IOException {
-    SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER);
+    int targetId = myTargetsState.getBuildTargetId(target);
-    return new SourceToOutputMappingWrapper(map, myTargetsState.getBuildTargetId(target));
+    if (storageManager == null) {
+      SourceToOutputMappingImpl map = getStorage(target, SRC_TO_OUT_MAPPING_PROVIDER);
+      return new SourceToOutputMappingWrapper(map, targetId);
+    }
+    else {
+      return new SourceToOutputMappingWrapper(ExperimentalSourceToOutputMapping.createSourceToOutputMap(storageManager, myRelativizer, target), targetId);
+    }
   }
 
   @ApiStatus.Internal
@@ -464,7 +472,7 @@ public final class BuildDataManager {
     }
 
     @Override
-    public void setOutputs(@NotNull String srcPath, @NotNull Collection<String> outputs) throws IOException {
+    public void setOutputs(@NotNull String srcPath, @NotNull List<String> outputs) throws IOException {
       try {
         myDelegate.setOutputs(srcPath, outputs);
       }
@@ -503,24 +511,19 @@ public final class BuildDataManager {
       myDelegate.removeOutput(sourcePath, outputPath);
     }
 
-    @Override
-    public @NotNull Collection<String> getSources() throws IOException {
-      return myDelegate.getSources();
-    }
-
     @Override
     public @Nullable Collection<String> getOutputs(@NotNull String srcPath) throws IOException {
       return myDelegate.getOutputs(srcPath);
     }
 
     @Override
-    public @NotNull Iterator<String> getOutputsIterator(@NotNull String srcPath) throws IOException {
+    public @NotNull Iterator<String> getSourcesIterator() throws IOException {
-      return myDelegate.getOutputsIterator(srcPath);
+      return myDelegate.getSourcesIterator();
     }
 
     @Override
-    public @NotNull Iterator<String> getSourcesIterator() throws IOException {
+    public @NotNull SourceToOutputMappingCursor cursor() throws IOException {
-      return myDelegate.getSourcesIterator();
+      return myDelegate.cursor();
     }
   }
 

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalOneToManyPathMapping.kt

@@ -4,53 +4,44 @@
 package org.jetbrains.jps.incremental.storage
 
 import com.dynatrace.hash4j.hashing.Hashing
-import org.h2.mvstore.type.DataType
+import org.jetbrains.annotations.ApiStatus
 import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
 import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
 import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType
-import java.io.IOException
-import kotlin.Throws
 
-internal interface OneToManyPathMapping : StorageOwner {
+@ApiStatus.Internal
-  @Throws(IOException::class)
+open class ExperimentalOneToManyPathMapping protected constructor(
-  fun getState(path: String): Collection<String>?
+  @JvmField protected val mapHandle: MapHandle<LongArray, Array<String>>,
+  @JvmField protected val relativizer: PathRelativizerService,
+  private val valueOffset: Int = 0,
+) : OneToManyPathMapping, StorageOwner by StorageOwnerByMap(mapHandle = mapHandle) {
+  constructor(
+    mapName: String,
+    storageManager: StorageManager,
+    relativizer: PathRelativizerService,
+  ) : this(storageManager.openMap(mapName, LongPairKeyDataType, StringListDataType), relativizer)
 
-  @Throws(IOException::class)
+  protected fun getKey(path: String): LongArray = stringTo128BitHash(relativizer.toRelative(path))
-  fun update(path: String, outPaths: List<String>)
-
-  @Throws(IOException::class)
-  fun remove(path: String)
-}
-
-internal class ExperimentalOneToManyPathMapping(
-  mapName: String,
-  storageManager: StorageManager,
-  private val relativizer: PathRelativizerService,
-) : OneToManyPathMapping,
-    StorageOwnerByMap<LongArray, Array<String>>(
-      mapName = mapName,
-      storageManager = storageManager,
-      keyType = LongPairKeyDataType,
-      valueType = StringListDataType,
-    ) {
-
-  private fun getKey(path: String): LongArray {
-    val stringKey = relativizer.toRelative(path).toByteArray()
-    return longArrayOf(Hashing.xxh3_64().hashBytesToLong(stringKey), Hashing.komihash5_0().hashBytesToLong(stringKey))
-  }
 
   @Suppress("ReplaceGetOrSet")
-  override fun getState(path: String): Collection<String>? {
+  final override fun getOutputs(path: String): List<String>? {
     val key = getKey(path)
     val list = mapHandle.map.get(key) ?: return null
-    return Array<String>(list.size) { relativizer.toFull(list.get(it)) }.asList()
+    return Array<String>(list.size - valueOffset) { relativizer.toFull(list.get(it + valueOffset)) }.asList()
   }
 
-  override fun update(path: String, outPaths: List<String>) {
+  final override fun setOutputs(path: String, outPaths: List<String>) {
-    val key = getKey(path)
+    val relativeSourcePath = relativizer.toRelative(path)
+    val key = stringTo128BitHash(relativeSourcePath)
     if (outPaths.isEmpty()) {
       mapHandle.map.remove(key)
     }
+    else if (valueOffset == 1) {
+      val listWithRelativePaths = Array(outPaths.size + 1) {
+        if (it == 0) relativeSourcePath else relativizer.toRelative(outPaths.get(it - 1))
+      }
+      mapHandle.map.put(key, listWithRelativePaths)
+    }
     else {
       val listWithRelativePaths = Array(outPaths.size) {
         relativizer.toRelative(outPaths.get(it))
@@ -59,21 +50,18 @@ internal class ExperimentalOneToManyPathMapping(
     }
   }
 
-  override fun remove(path: String) {
+  final override fun remove(path: String) {
     mapHandle.map.remove(getKey(path))
   }
 }
 
-internal sealed class StorageOwnerByMap<K : Any, V : Any>(
+internal fun stringTo128BitHash(string: String): LongArray {
-  mapName: String,
+  val bytes = string.toByteArray()
-  storageManager: StorageManager,
+  return longArrayOf(Hashing.xxh3_64().hashBytesToLong(bytes), Hashing.komihash5_0().hashBytesToLong(bytes))
-  keyType: DataType<K>,
+}
-  valueType: DataType<V>,
-) : StorageOwner {
-  @JvmField
-  protected val mapHandle = storageManager.openMap(mapName, keyType, valueType)
 
-  final override fun flush(memoryCachesOnly: Boolean) {
+internal class StorageOwnerByMap<K : Any, V : Any>(private val mapHandle: MapHandle<K, V>) : StorageOwner {
+  override fun flush(memoryCachesOnly: Boolean) {
     if (memoryCachesOnly) {
       // set again to force to clear the cache (in kb)
       mapHandle.map.store.cacheSize = MV_STORE_CACHE_SIZE_IN_MB * 1024
@@ -83,11 +71,11 @@ internal sealed class StorageOwnerByMap<K : Any, V : Any>(
     }
   }
 
-  final override fun clean() {
+  override fun clean() {
     mapHandle.map.clear()
   }
 
-  final override fun close() {
+  override fun close() {
     mapHandle.release()
   }
 }

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/ExperimentalSourceToOutputMapping.kt

@@ -0,0 +1,197 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.jps.incremental.storage
+
+import org.h2.mvstore.MVMap
+import org.h2.mvstore.MVMap.Decision
+import org.h2.mvstore.MVMap.DecisionMaker
+import org.jetbrains.annotations.ApiStatus.Internal
+import org.jetbrains.annotations.VisibleForTesting
+import org.jetbrains.jps.builders.BuildTarget
+import org.jetbrains.jps.builders.storage.SourceToOutputMapping
+import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
+import org.jetbrains.jps.incremental.storage.dataTypes.LongPairKeyDataType
+import org.jetbrains.jps.incremental.storage.dataTypes.StringListDataType
+
+@Internal
+class ExperimentalSourceToOutputMapping private constructor(
+  mapHandle: MapHandle<LongArray, Array<String>>,
+  relativizer: PathRelativizerService,
+) : SourceToOutputMapping, ExperimentalOneToManyPathMapping(mapHandle = mapHandle, relativizer = relativizer, valueOffset = 1) {
+  companion object {
+    // we have a lot of targets - reduce GC and reuse map builder
+    private val mapBuilder = MVMap.Builder<LongArray, Array<String>>().also {
+      it.setKeyType(LongPairKeyDataType)
+      it.setValueType(StringListDataType)
+    }
+
+    @JvmStatic
+    fun createSourceToOutputMap(
+      storageManager: StorageManager,
+      relativizer: PathRelativizerService,
+      target: BuildTarget<*>,
+    ): ExperimentalSourceToOutputMapping {
+      return createSourceToOutputMap(
+        storageManager = storageManager,
+        relativizer = relativizer,
+        targetId = target.id,
+        targetTypeId = target.targetType.typeId,
+      )
+    }
+
+    @VisibleForTesting
+    @Internal
+    fun createSourceToOutputMap(
+      storageManager: StorageManager,
+      relativizer: PathRelativizerService,
+      targetId: String,
+      targetTypeId: String,
+    ): ExperimentalSourceToOutputMapping {
+      // we can use composite key and sort by target id, but as we compile targets in parallel:
+      // * avoid blocking - in-memory lock per map root,
+      // * avoid a huge B-tree and reduce rebalancing time due to contention.
+      return ExperimentalSourceToOutputMapping(
+        mapHandle = storageManager.openMap("$targetId|$targetTypeId|src-to-out", mapBuilder),
+        relativizer = relativizer,
+      )
+    }
+  }
+
+  //override fun getKey(sourcePath: String): LongArray {
+  //  val stringKey = relativizer.toRelative(sourcePath).toByteArray()
+  //
+  //  // We should sort by target id for data locality -
+  //  // the high integer (high) is shifted to the higher 32 bits of the long value,
+  //  // while the low integer (low) occupies the lower 32 bits.
+  //  // When we do compare two long values, the higher 32 bits (which contain the high integer) dictate the order.
+  //  val low = Hashing.komihash5_0().hashBytesToInt(stringKey)
+  //  val high = targetId
+  //  return longArrayOf((high.toLong() shl 32) or (low.toLong() and 0xFFFFFFFFL), Hashing.xxh3_64().hashBytesToLong(stringKey))
+  //}
+
+  override fun setOutput(sourcePath: String, outputPath: String) {
+    val relativeSourcePath = relativizer.toRelative(sourcePath)
+    mapHandle.map.put(stringTo128BitHash(relativeSourcePath), arrayOf(relativeSourcePath, relativizer.toRelative(outputPath)))
+  }
+
+  override fun appendOutput(sourcePath: String, outputPath: String) {
+    val relativeSourcePath = relativizer.toRelative(sourcePath)
+    mapHandle.map.operate(stringTo128BitHash(relativeSourcePath),
+                          null,
+                          AddItemDecisionMaker(sourcePath = relativeSourcePath, toAdd = relativizer.toRelative(outputPath)))
+  }
+
+  override fun removeOutput(sourcePath: String, outputPath: String) {
+    mapHandle.map.operate(getKey(sourcePath), null, RemoveItemDecisionMaker(relativizer.toRelative(outputPath)))
+  }
+
+  override fun cursor(): SourceToOutputMappingCursor {
+    val cursor = mapHandle.map.cursor(null)
+    return object : SourceToOutputMappingCursor {
+      override fun hasNext(): Boolean = cursor.hasNext()
+
+      override fun next(): String {
+        cursor.next()
+        return relativizer.toFull(cursor.value.first<String>())
+      }
+
+      override val outputPaths: Array<String>
+        get() {
+          val list = cursor.value
+          return Array<String>(list.size - 1) { relativizer.toFull(list[it + 1]) }
+        }
+    }
+  }
+
+  override fun getSourcesIterator(): Iterator<String> = cursor()
+}
+
+private val CHECK_COLLISIONS = System.getProperty("jps.source.to.output.mapping.check.collisions", "false").toBoolean()
+
+private class AddItemDecisionMaker(private val sourcePath: String, private val toAdd: String) : DecisionMaker<Array<String>>() {
+  override fun decide(existingValue: Array<String>?, providedValue: Array<String>?): Decision {
+    when {
+      existingValue == null -> {
+        return Decision.PUT
+      }
+      existingValue.isEmpty() -> {
+        if (CHECK_COLLISIONS) {
+          throw IllegalStateException("Value is empty")
+        }
+        return Decision.PUT
+      }
+      CHECK_COLLISIONS && existingValue[0] != sourcePath -> {
+        throw IllegalStateException("Collision for $sourcePath: ${existingValue[0]} and $sourcePath")
+      }
+      else -> {
+        for (i in 1 until existingValue.size) {
+          if (existingValue[i] == toAdd) {
+            return Decision.ABORT
+          }
+        }
+        return Decision.PUT
+      }
+    }
+  }
+
+  // it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel
+  override fun <T : Array<String>?> selectValue(existingValue: T?, ignore: T?): T? {
+    if (existingValue == null || existingValue.isEmpty()) {
+      @Suppress("UNCHECKED_CAST")
+      return arrayOf(sourcePath, toAdd) as T
+    }
+    else {
+      // we checked `contains` in `decide`
+      @Suppress("UNCHECKED_CAST")
+      return addElementToEnd(existingValue, toAdd) as T?
+    }
+  }
+}
+
+private class RemoveItemDecisionMaker(private val toRemove: String) : DecisionMaker<Array<String>>() {
+  private var indexToRemove: Int = -1
+
+  override fun decide(existingValue: Array<String>?, ignore: Array<String>?): Decision {
+    return when {
+      existingValue == null -> Decision.ABORT
+      // empty value list is not normal, recover - just delete record
+      existingValue.size == 1 -> Decision.REMOVE
+      else -> {
+        for (i in 1 until existingValue.size) {
+          if (existingValue[i] == toRemove) {
+            indexToRemove = i
+            break
+          }
+        }
+        when {
+          indexToRemove == -1 -> Decision.ABORT
+          existingValue.size == 2 -> Decision.REMOVE
+          else -> Decision.PUT
+        }
+      }
+    }
+  }
+
+  // it is called when lock is obtained, so, we can assume that we will not lose a new value if appendOutput is called in parallel
+  override fun <T : Array<String>?> selectValue(existingValue: T, providedValue: T?): T {
+    assert(indexToRemove != -1)
+    @Suppress("UNCHECKED_CAST")
+    return removeElementAtIndex(existingValue!!, indexToRemove) as T
+  }
+}
+
+private fun removeElementAtIndex(old: Array<String>, index: Int): Array<String?> {
+  val newSize = old.size - 1
+  val result = arrayOfNulls<String>(newSize)
+  System.arraycopy(old, 0, result, 0, index)
+  if (index < newSize) {
+    System.arraycopy(old, index + 1, result, index, newSize - index)
+  }
+  return result
+}
+
+private fun addElementToEnd(old: Array<String>, element: String): Array<String?> {
+    val result = arrayOfNulls<String>(old.size + 1)
+    System.arraycopy(old, 0, result, 0, old.size)
+    result[old.size] = element
+    return result
+}

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathMapping.kt

@@ -0,0 +1,22 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.jps.incremental.storage
+
+import org.jetbrains.annotations.ApiStatus
+import java.io.IOException
+
+internal interface OneToManyPathMapping : StorageOwner {
+  @Throws(IOException::class)
+  fun getOutputs(path: String): Collection<String>?
+
+  @Throws(IOException::class)
+  fun setOutputs(path: String, outPaths: List<String>)
+
+  @Throws(IOException::class)
+  fun remove(path: String)
+}
+
+@ApiStatus.Internal
+interface SourceToOutputMappingCursor : Iterator<String> {
+  /** [next] must be called beforehand */
+  val outputPaths: Array<String>
+}

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java

@@ -1,19 +1,17 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.jps.incremental.storage;
 
+import com.intellij.util.ArrayUtilRt;
 import com.intellij.util.io.DataExternalizer;
 import com.intellij.util.io.IOUtil;
 import com.intellij.util.io.PersistentMapBuilder;
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.Unmodifiable;
 import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
-import org.jetbrains.jps.javac.Iterators;
 
-import java.io.DataInput;
+import java.io.*;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.nio.file.Path;
 import java.util.*;
 
@@ -29,11 +27,11 @@ public final class OneToManyPathsMapping extends AbstractStateStorage<String, Co
   }
 
   @Override
-  public void update(@NotNull String keyPath, @SuppressWarnings("NullableProblems") @NotNull List<String> boundPaths) throws IOException {
+  public void setOutputs(@NotNull String keyPath, @NotNull List<String> boundPaths) throws IOException {
     super.update(normalizePath(keyPath), normalizePaths(boundPaths));
   }
 
-  public void update(@NotNull String keyPath, @NotNull String boundPath) throws IOException {
+  void setOutput(@NotNull String keyPath, @NotNull String boundPath) throws IOException {
     super.update(normalizePath(keyPath), List.of(normalizePath(boundPath)));
   }
 
@@ -46,12 +44,24 @@ public final class OneToManyPathsMapping extends AbstractStateStorage<String, Co
     super.appendData(normalizePath(keyPath), normalizePaths((List<String>)boundPaths));
   }
 
+  /**
+   * @deprecated Use {@link #getOutputs(String)}
+   */
+  @Deprecated(forRemoval = true)
   @Override
   public @Nullable Collection<String> getState(@NotNull String keyPath) throws IOException {
+    return getOutputs(keyPath);
+  }
+
+  @Override
+  public @Nullable List<String> getOutputs(@NotNull String keyPath) throws IOException {
     List<String> collection = (List<String>)super.getState(relativizer.toRelative(keyPath));
-    if (collection == null || collection.isEmpty()) {
+    if (collection == null) {
       return null;
     }
+    else if (collection.isEmpty()) {
+      return Collections.emptyList();
+    }
     else {
       String[] result = new String[collection.size()];
       for (int i = 0, size = collection.size(); i < size; i++) {
@@ -61,9 +71,21 @@ public final class OneToManyPathsMapping extends AbstractStateStorage<String, Co
     }
   }
 
-  public @NotNull Iterator<String> getStateIterator(@NotNull String keyPath) throws IOException {
+  public String @Nullable [] getOutputArray(@NotNull String keyPath) throws IOException {
     List<String> collection = (List<String>)super.getState(relativizer.toRelative(keyPath));
-    return collection == null ? Collections.emptyIterator() : Iterators.map(collection.iterator(), relativizer::toFull);
+    if (collection == null) {
+      return null;
+    }
+    else if (collection.isEmpty()) {
+      return ArrayUtilRt.EMPTY_STRING_ARRAY;
+    }
+    else {
+      String[] result = new String[collection.size()];
+      for (int i = 0, size = collection.size(); i < size; i++) {
+        result[i] = relativizer.toFull(collection.get(i));
+      }
+      return result;
+    }
   }
 
   @Override
@@ -72,22 +94,13 @@ public final class OneToManyPathsMapping extends AbstractStateStorage<String, Co
   }
 
   @Override
-  public @NotNull List<String> getKeys() throws IOException {
+  public @NotNull Iterator<String> getKeysIterator() throws IOException {
-    List<String> collection = (List<String>)super.getKeys();
+    return super.getKeyIterator(relativizer::toFull);
-    if (collection.isEmpty()) {
-      return List.of();
-    }
-
-    String[] result = new String[collection.size()];
-    for (int i = 0; i < collection.size(); i++) {
-      result[i] = relativizer.toFull(collection.get(i));
-    }
-    return Arrays.asList(result);
   }
 
-  @Override
+  @ApiStatus.Internal
-  public Iterator<String> getKeysIterator() throws IOException {
+  public @NotNull SourceToOutputMappingCursor cursor() throws IOException {
-    return Iterators.map(super.getKeysIterator(), relativizer::toFull);
+    return new SourceToOutputMappingCursorImpl(getKeysIterator());
   }
 
   public void removeData(@NotNull String keyPath, @NotNull String boundPath) throws IOException {
@@ -151,4 +164,32 @@ public final class OneToManyPathsMapping extends AbstractStateStorage<String, Co
     }
     return Arrays.asList(normalized);
   }
+
+  private final class SourceToOutputMappingCursorImpl implements SourceToOutputMappingCursor {
+    private final Iterator<String> mySourceIterator;
+    private String sourcePath;
+
+    private SourceToOutputMappingCursorImpl(Iterator<String> sourceIterator) { mySourceIterator = sourceIterator; }
+
+    @Override
+    public boolean hasNext() {
+      return mySourceIterator.hasNext();
+    }
+
+    @Override
+    public @NotNull String next() {
+      sourcePath = mySourceIterator.next();
+      return sourcePath;
+    }
+
+    @Override
+    public String @NotNull [] getOutputPaths() {
+      try {
+        return Objects.requireNonNullElse(getOutputArray(sourcePath), ArrayUtilRt.EMPTY_STRING_ARRAY);
+      }
+      catch (IOException e) {
+        throw new UncheckedIOException(e);
+      }
+    }
+  }
 }

jps/jps-builders/src/org/jetbrains/jps/incremental/storage/SourceToOutputMappingImpl.java

@@ -9,8 +9,8 @@ import org.jetbrains.jps.incremental.relativizer.PathRelativizerService;
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Collection;
 import java.util.Iterator;
+import java.util.List;
 
 @ApiStatus.Internal
 public final class SourceToOutputMappingImpl implements SourceToOutputMapping, StorageOwner {
@@ -21,13 +21,13 @@ public final class SourceToOutputMappingImpl implements SourceToOutputMapping, S
   }
 
   @Override
-  public void setOutputs(@NotNull String srcPath, @NotNull Collection<String> outputs) throws IOException {
+  public void setOutputs(@NotNull String srcPath, @NotNull List<String> outputs) throws IOException {
-    myMapping.update(srcPath, outputs);
+    myMapping.setOutputs(srcPath, outputs);
   }
 
   @Override
   public void setOutput(@NotNull String srcPath, @NotNull String outputPath) throws IOException {
-    myMapping.update(srcPath, outputPath);
+    myMapping.setOutput(srcPath, outputPath);
   }
 
   @Override
@@ -46,18 +46,8 @@ public final class SourceToOutputMappingImpl implements SourceToOutputMapping, S
   }
 
   @Override
-  public @NotNull Collection<String> getSources() throws IOException {
+  public @Nullable List<String> getOutputs(@NotNull String srcPath) throws IOException {
-    return myMapping.getKeys();
+    return myMapping.getOutputs(srcPath);
-  }
-
-  @Override
-  public @Nullable Collection<String> getOutputs(@NotNull String srcPath) throws IOException {
-    return myMapping.getState(srcPath);
-  }
-
-  @Override
-  public @NotNull Iterator<String> getOutputsIterator(@NotNull String srcPath) throws IOException {
-    return myMapping.getStateIterator(srcPath);
   }
 
   @Override
@@ -65,6 +55,11 @@ public final class SourceToOutputMappingImpl implements SourceToOutputMapping, S
     return myMapping.getKeysIterator();
   }
 
+  @Override
+  public @NotNull SourceToOutputMappingCursor cursor() throws IOException {
+    return myMapping.cursor();
+  }
+
   @Override
   public void flush(boolean memoryCachesOnly) {
     myMapping.flush(memoryCachesOnly);

jps/jps-builders/testSrc/org/jetbrains/jps/builders/BuildResult.java

@@ -1,13 +1,14 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.jps.builders;
 
 import com.intellij.openapi.util.SystemInfo;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.util.ObjectUtils;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.ints.IntSet;
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jps.builders.java.dependencyView.Mappings;
 import org.jetbrains.jps.builders.storage.SourceToOutputMapping;
@@ -20,10 +21,7 @@ import org.jetbrains.jps.incremental.storage.OutputToTargetRegistry;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
 
 import static org.junit.Assert.*;
 
@@ -64,15 +62,15 @@ public final class BuildResult implements MessageHandler {
     for (BuildTarget<?> target : targets) {
       id2Target.put(pd.dataManager.getTargetsState().getBuildTargetId(target), target);
     }
-    Int2ObjectMap<String> hashCodeToOutputPath = new Int2ObjectOpenHashMap<>();
+    Int2ObjectOpenHashMap<String> hashCodeToOutputPath = new Int2ObjectOpenHashMap<>();
     for (BuildTarget<?> target : targets) {
       stream.println("Begin Of SourceToOutput (target " + getTargetIdWithTypeId(target) + ")");
       SourceToOutputMapping map = pd.dataManager.getSourceToOutputMap(target);
-      List<String> sourcesList = new ArrayList<>(map.getSources());
+      List<String> sourceList = new ObjectArrayList<>(map.getSourcesIterator());
-      Collections.sort(sourcesList);
+      sourceList.sort(null);
-      for (String source : sourcesList) {
+      for (String source : sourceList) {
-        List<String> outputs = new ArrayList<>(ObjectUtils.notNull(map.getOutputs(source), Collections.emptySet()));
+        List<String> outputs = new ArrayList<>(Objects.requireNonNullElse(map.getOutputs(source), Collections.emptyList()));
-        Collections.sort(outputs);
+        outputs.sort(null);
         for (String output : outputs) {
           hashCodeToOutputPath.put(FileUtil.pathHashCode(output), output);
         }
@@ -84,7 +82,7 @@ public final class BuildResult implements MessageHandler {
 
 
     OutputToTargetRegistry registry = pd.dataManager.getOutputToTargetRegistry();
-    List<Integer> keys = new ArrayList<>(registry.getKeys());
+    List<Integer> keys = new IntArrayList(registry.getKeysIterator());
     Collections.sort(keys);
     stream.println("Begin Of OutputToTarget");
     for (Integer key : keys) {
@@ -96,7 +94,7 @@ public final class BuildResult implements MessageHandler {
         targetsNames.add(target != null ? getTargetIdWithTypeId(target) : "<unknown " + value + ">");
       });
       Collections.sort(targetsNames);
-      stream.println(hashCodeToOutputPath.get(key) + " -> " + targetsNames);
+      stream.println(hashCodeToOutputPath.get(key.intValue()) + " -> " + targetsNames);
     }
     stream.println("End Of OutputToTarget");
   }

jps/jps-builders/testSrc/org/jetbrains/jps/incremental/storage/SourceToOutputMappingFuzzTest.kt

@@ -0,0 +1,115 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.jps.incremental.storage
+
+import net.jqwik.api.*
+import net.jqwik.api.lifecycle.AfterProperty
+import net.jqwik.api.lifecycle.BeforeProperty
+import org.assertj.core.api.Assertions.assertThat
+import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.random.Random
+
+class SourceToOutputMappingFuzzTest {
+  companion object {
+    init {
+      System.setProperty("jps.source.to.output.mapping.check.collisions", "true")
+    }
+  }
+
+  private lateinit var mapping: ExperimentalSourceToOutputMapping
+  private var file: Path? = null
+
+  @BeforeProperty
+  fun setUp() {
+    file = Files.createTempFile("mvstore", ".db")
+    val storageManager = StorageManager(file!!, 0)
+    mapping = ExperimentalSourceToOutputMapping.createSourceToOutputMap(
+      storageManager = storageManager,
+      relativizer = PathRelativizerService(),
+      targetId = "test-module",
+      targetTypeId = "java"
+    )
+  }
+
+  @AfterProperty
+  fun tearDown() {
+    try {
+      mapping.close()
+    }
+    finally {
+      file?.let { Files.deleteIfExists(it) }
+    }
+  }
+
+  @Provide
+  fun pathStrings(): Arbitrary<String> {
+    return Arbitraries.strings().alpha().numeric().withChars('/').ofMinLength(2).ofMaxLength(255)
+  }
+
+  @Suppress("unused")
+  @Provide
+  fun pathStringLists(): Arbitrary<List<String>> {
+    return pathStrings().list().ofMinSize(1).ofMaxSize(32)
+  }
+
+  @Suppress("unused")
+  @Provide
+  fun pathStringListsList(): Arbitrary<List<List<String>>> {
+    return pathStrings().list().ofMinSize(1).ofMaxSize(32).list().ofMinSize(1).ofMaxSize(64)
+  }
+
+  @Property
+  fun setOutputs(@ForAll("pathStrings") source: String, @ForAll("pathStringLists") outputs: List<String>) {
+    mapping.setOutputs(source, outputs)
+    val result = mapping.getOutputs(source)
+    assertThat(result).isNotNull()
+    assertThat(result).containsExactlyInAnyOrderElementsOf(outputs)
+
+    checkCursorAndSourceIterator(source, outputs)
+  }
+
+  @Property
+  fun appendOutput(@ForAll("pathStrings") source: String, @ForAll("pathStrings") output: String) {
+    mapping.appendOutput(source, output)
+    val outputs = mapping.getOutputs(source)
+    assertThat(outputs).isNotNull
+    assertThat(outputs).contains(output)
+
+    checkCursorAndSourceIterator(source, outputs!!)
+  }
+
+  @Property
+  fun cursor(@ForAll("pathStringLists") sources: List<String>, @ForAll("pathStringListsList") outputs: List<List<String>>) {
+    mapping.clean()
+
+    val expectedMap = HashMap<String, List<String>>()
+    for (source in sources) {
+      val list = outputs[Random.nextInt(outputs.size)]
+      expectedMap.put(source, list)
+      mapping.setOutputs(source, list)
+    }
+
+    val actualMap = LinkedHashMap<String, List<String>>()
+    val cursor = mapping.cursor()
+    while (cursor.hasNext()) {
+      actualMap.put(cursor.next(), cursor.outputPaths.asList())
+    }
+
+    assertThat(actualMap).isEqualTo(expectedMap)
+  }
+
+  private fun checkCursorAndSourceIterator(source: String, outputs: List<String>) {
+    val cursor = mapping.cursor()
+    val map = LinkedHashMap<String, List<String>>()
+    while (cursor.hasNext()) {
+      val next = cursor.next()
+      if (next == source) {
+        map.put(source, cursor.outputPaths.asList())
+      }
+    }
+    assertThat(map).isEqualTo(mapOf(source to outputs))
+
+    assertThat(mapping.sourcesIterator.asSequence().contains(source)).isTrue()
+  }
+}

plugins/groovy/jps-plugin/intellij.groovy.jps.iml

@@ -34,5 +34,6 @@
     <orderEntry type="module" module-name="intellij.platform.util.classLoader" />
     <orderEntry type="library" name="ASM" level="project" />
     <orderEntry type="module" module-name="intellij.platform.util.jdom" />
+    <orderEntry type="library" name="kotlin-stdlib" level="project" />
   </component>
 </module>

plugins/groovy/jps-plugin/src/org/jetbrains/jps/incremental/groovy/JpsGroovycRunner.java

@@ -28,6 +28,7 @@ import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode;
 import org.jetbrains.jps.incremental.java.JavaBuilder;
 import org.jetbrains.jps.incremental.messages.BuildMessage;
 import org.jetbrains.jps.incremental.messages.CompilerMessage;
+import org.jetbrains.jps.incremental.storage.SourceToOutputMappingCursor;
 import org.jetbrains.jps.model.JpsDummyElement;
 import org.jetbrains.jps.model.java.JpsJavaExtensionService;
 import org.jetbrains.jps.model.java.JpsJavaSdkType;
@@ -434,17 +435,15 @@ public abstract class JpsGroovycRunner<R extends BuildRootDescriptor, T extends
       context.getProjectDescriptor().getProject());
     for (T target : getTargets(chunk)) {
       String moduleOutputPath = finalOutputs.get(target);
-      final SourceToOutputMapping srcToOut = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
+      SourceToOutputMapping srcToOut = context.getProjectDescriptor().dataManager.getSourceToOutputMap(target);
-      for (String src : srcToOut.getSources()) {
+      for (SourceToOutputMappingCursor cursor = srcToOut.cursor(); cursor.hasNext(); ) {
+        String src = cursor.next();
         if (!toCompilePaths.contains(src) && GroovyBuilder.isGroovyFile(src) &&
             !configuration.getCompilerExcludes().isExcluded(new File(src))) {
-          final Collection<String> outs = srcToOut.getOutputs(src);
+          for (String out : cursor.getOutputPaths()) {
-          if (outs != null) {
+            if (out.endsWith(".class") && out.startsWith(moduleOutputPath)) {
-            for (String out : outs) {
+              final String className = out.substring(moduleOutputPath.length(), out.length() - ".class".length()).replace('/', '.');
-              if (out.endsWith(".class") && out.startsWith(moduleOutputPath)) {
+              class2Src.put(className, src);
-                final String className = out.substring(moduleOutputPath.length(), out.length() - ".class".length()).replace('/', '.');
-                class2Src.put(className, src);
-              }
             }
           }
         }

plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsBindingManager.java

@@ -162,7 +162,7 @@ public final class FormsBindingManager extends FormsBuilder {
       for (Map.Entry<File, ModuleBuildTarget> entry : filesToCompile.entrySet()) {
         final File srcFile = entry.getKey();
         final ModuleBuildTarget target = entry.getValue();
-        final Collection<String> boundForms = sourceToFormMap.getState(srcFile.getPath());
+        final Collection<String> boundForms = sourceToFormMap.getOutputs(srcFile.getPath());
         if (boundForms != null) {
           for (String formPath : boundForms) {
             final File formFile = new File(formPath);

plugins/ui-designer/jps-plugin/src/org/jetbrains/jps/uiDesigner/compiler/FormsInstrumenter.java

@@ -98,7 +98,7 @@ public final class FormsInstrumenter extends FormsBuilder {
           for (File form : forms) {
             formPaths.add(form.getPath());
           }
-          sourceToFormMap.update(src.getPath(), formPaths);
+          sourceToFormMap.setOutputs(src.getPath(), formPaths);
           srcToForms.remove(src);
         }
         // clean mapping