manage dependency graph on the builder side

GitOrigin-RevId: 40ba09b472b9df8d4d1bda52912c1e03e8f7304b

jps/jps-builders/api-dump.txt

@@ -1999,6 +1999,7 @@ org.jetbrains.jps.dependency.Graph
 - a:getSources():java.lang.Iterable
 - a:getSources(org.jetbrains.jps.dependency.ReferenceID):java.lang.Iterable
 org.jetbrains.jps.dependency.GraphConfiguration
+- s:create(org.jetbrains.jps.dependency.DependencyGraph,org.jetbrains.jps.dependency.NodeSourcePathMapper):org.jetbrains.jps.dependency.GraphConfiguration
 - a:getGraph():org.jetbrains.jps.dependency.DependencyGraph
 - a:getPathMapper():org.jetbrains.jps.dependency.NodeSourcePathMapper
 org.jetbrains.jps.dependency.GraphDataInput

jps/jps-builders/src/org/jetbrains/jps/bazel/BazelIncBuilder.java

@@ -6,14 +6,14 @@ import org.jetbrains.jps.bazel.impl.*;
 import org.jetbrains.jps.bazel.runner.BytecodeInstrumenter;
 import org.jetbrains.jps.bazel.runner.CompilerRunner;
 import org.jetbrains.jps.bazel.runner.RunnerFactory;
-import org.jetbrains.jps.dependency.Delta;
-import org.jetbrains.jps.dependency.DependencyGraph;
-import org.jetbrains.jps.dependency.Node;
-import org.jetbrains.jps.dependency.NodeSource;
+import org.jetbrains.jps.dependency.*;
+import org.jetbrains.jps.dependency.impl.DependencyGraphImpl;
 import org.jetbrains.jps.dependency.impl.GraphDataOutputImpl;
 import org.jetbrains.jps.dependency.impl.PathSource;
+import org.jetbrains.jps.dependency.impl.PersistentMVStoreMapletFactory;
 import org.jetbrains.jps.dependency.java.JVMClassNode;
 
+import java.io.Closeable;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
@@ -30,6 +30,7 @@ import static org.jetbrains.jps.javac.Iterators.*;
 
 public class BazelIncBuilder {
   private static final String SOURCE_SNAPSHOT_FILE_NAME = "src-snapshot.dat";
+  private static final String DEP_GRAPH_FILE_NAME = "dep-graph.mv";
 
   private static final List<RunnerFactory<? extends CompilerRunner>> ourCompilers = List.of(
     ResourcesCopy::new
@@ -41,27 +42,40 @@ public class BazelIncBuilder {
     NotNullInstrumenter::new, FormsInstrumenter::new
   );
 
+  private static GraphConfiguration setupGraphConfiguration(BuildContext context) throws IOException {
+    DependencyGraphImpl graph = new DependencyGraphImpl(
+      new PersistentMVStoreMapletFactory(context.getDataDir().resolve(DEP_GRAPH_FILE_NAME).toString(), Math.min(8, Runtime.getRuntime().availableProcessors()))
+    );
+    return GraphConfiguration.create(graph, context.getPathMapper());
+  }
+
+
   public ExitCode build(BuildContext context) {
     // todo: support cancellation checks
 
-    SourceSnapshotDelta snapshotDelta;
-    if (context.isRebuild()) {
-      snapshotDelta = new SourceSnapshotDeltaImpl(context.getSources());
-      snapshotDelta.markRecompileAll();
-    }
-    else {
-      snapshotDelta = new SourceSnapshotDeltaImpl(getOldSourceSnapshot(context), context.getSources());
-    }
-
     GraphUpdater graphUpdater = new GraphUpdater(context.getTargetName());
     DiagnosticSink diagnostic = context;
     ZipOutputBuilder outputBuilder = null;
+    SourceSnapshotDelta snapshotDelta = null;
+    DependencyGraph depGraph = null;
+    
     try {
-      DependencyGraph depGraph = context.getGraphConfig().getGraph();
+      if (context.isRebuild()) {
+        snapshotDelta = new SourceSnapshotDeltaImpl(context.getSources());
+        snapshotDelta.markRecompileAll(); // force rebuild
+      }
+      else {
+        snapshotDelta = new SourceSnapshotDeltaImpl(loadSourceSnapshot(context), context.getSources());
+      }
+
       if (snapshotDelta.isRecompileAll()) {
         context.cleanBuildState();
       }
-      else {
+
+      GraphConfiguration graphConfig = setupGraphConfiguration(context);
+      depGraph = graphConfig.getGraph();
+
+      if (!snapshotDelta.isRecompileAll()) {
         // todo: process changes in libs
 
         // expand compile scope
@@ -179,17 +193,12 @@ public class BazelIncBuilder {
         // report postponed errors, if necessary
         ((PostponedDiagnosticSink)diagnostic).drainTo(context);
       }
-      saveSourceSnapshot(context, snapshotDelta.asSnapshot());
-      // todo: save abi-jar
-      if (outputBuilder != null) {
-        try {
-          outputBuilder.close(true);
-        }
-        catch (IOException e) {
-          diagnostic.report(Message.create(null, e));
-        }
+      if (snapshotDelta != null) {
+        saveSourceSnapshot(context, snapshotDelta.asSnapshot());
       }
-      // todo: close graph and save all caches
+      // todo: save abi-jar
+      safeClose(outputBuilder, diagnostic);
+      safeClose(depGraph, diagnostic);
     }
   }
 
@@ -211,7 +220,7 @@ public class BazelIncBuilder {
   }
 
   private static void saveSourceSnapshot(BuildContext context, SourceSnapshot snapshot) {
-    Path snapshotPath = context.getBaseDir().resolve(SOURCE_SNAPSHOT_FILE_NAME);
+    Path snapshotPath = context.getDataDir().resolve(SOURCE_SNAPSHOT_FILE_NAME);
     try (var stream = new DataOutputStream(new DeflaterOutputStream(Files.newOutputStream(snapshotPath), new Deflater(Deflater.BEST_SPEED)))) {
       snapshot.write(new GraphDataOutputImpl(stream));
     }
@@ -220,8 +229,8 @@ public class BazelIncBuilder {
     }
   }
 
-  private static SourceSnapshot getOldSourceSnapshot(BuildContext context) {
-    Path oldSnapshot = context.getBaseDir().resolve(SOURCE_SNAPSHOT_FILE_NAME);
+  private static SourceSnapshot loadSourceSnapshot(BuildContext context) {
+    Path oldSnapshot = context.getDataDir().resolve(SOURCE_SNAPSHOT_FILE_NAME);
     try (var stream = new DataInputStream(new InflaterInputStream(Files.newInputStream(oldSnapshot, StandardOpenOption.READ)))) {
       return new SourceSnapshotImpl(stream, PathSource::new);
     }
@@ -230,4 +239,16 @@ public class BazelIncBuilder {
       return SourceSnapshot.EMPTY;
     }
   }
+
+  private static void safeClose(Closeable cl, DiagnosticSink diagnostic) {
+    if (cl == null) {
+      return;
+    }
+    try {
+      cl.close();
+    }
+    catch (Throwable e) {
+      diagnostic.report(Message.create(null, e));
+    }
+  }
 }

jps/jps-builders/src/org/jetbrains/jps/bazel/BuildContext.java

@@ -1,7 +1,9 @@
 // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.jps.bazel;
 
-import org.jetbrains.jps.dependency.GraphConfiguration;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.jps.dependency.NodeSourcePathMapper;
 
 import java.nio.file.Path;
 
@@ -12,9 +14,24 @@ public interface BuildContext extends DiagnosticSink {
 
   boolean isCanceled();
 
+  /**
+   * @return the BazelWorker working dir (can be a sandbox dir)
+   * Source and library inputs should be resolved against the base dir
+   */
+  @NotNull
   Path getBaseDir();
 
+  /**
+   * @return base directory where incremental data storages are to be stored
+   */
+  @NotNull
+  Path getDataDir();
+
+  @NotNull
   Path getOutputZip();
+  
+  @Nullable
+  Path getAbiOutputZip();
 
   SourceSnapshot getSources();
 
@@ -22,7 +39,7 @@ public interface BuildContext extends DiagnosticSink {
 
   BuilderArgs getBuilderArgs();
 
-  GraphConfiguration getGraphConfig();
+  NodeSourcePathMapper getPathMapper();
 
   BuildProcessLogger getBuildLogger();
   // wipe graph, delete all caches, snapshots, storages

jps/jps-builders/src/org/jetbrains/jps/bazel/impl/BuildContextImpl.java

@@ -0,0 +1,119 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.jps.bazel.impl;
+
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.jps.bazel.*;
+import org.jetbrains.jps.dependency.NodeSourcePathMapper;
+import org.jetbrains.jps.dependency.impl.PathSourceMapper;
+
+import java.nio.file.Path;
+
+public class BuildContextImpl implements BuildContext  {
+  private final String myTargetName;
+  private final Path myBaseDir;
+  private final PathSourceMapper myPathMapper;
+  @NotNull
+  private final Path myOutJar;
+  @Nullable
+  private final Path myAbiJar;
+  private final Path myDataDir;
+
+  public BuildContextImpl(String targetName, Path baseDir, Path outJar, @Nullable Path abiJar, String cachePrefix) {
+    myTargetName = targetName;
+    myBaseDir = baseDir;
+    myPathMapper = new PathSourceMapper(
+      relPath -> {
+        Path abs = baseDir.resolve(Path.of(relPath)).normalize();
+        return abs.toString().replace(baseDir.getFileSystem().getSeparator(), "/");
+      },
+      absPath -> {
+        Path relative = baseDir.relativize(Path.of(absPath)).normalize();
+        return relative.toString().replace(baseDir.getFileSystem().getSeparator(), "/");
+      }
+    );
+    myOutJar = outJar;
+    myAbiJar = abiJar;
+    myDataDir = outJar.resolveSibling(cachePrefix + truncateExtension(outJar.getFileName().toString()) + "-ic");
+  }
+
+  @Override
+  public String getTargetName() {
+    return myTargetName;
+  }
+
+  @Override
+  public boolean isRebuild() {
+    return false; // todo
+  }
+
+  @Override
+  public boolean isCanceled() {
+    return false;
+  }
+
+  @Override
+  public @NotNull Path getBaseDir() {
+    return myBaseDir;
+  }
+
+  @Override
+  public @NotNull Path getDataDir() {
+    return myDataDir;
+  }
+
+  @Override
+  public @NotNull Path getOutputZip() {
+    return myOutJar;
+  }
+
+  @Override
+  public @Nullable Path getAbiOutputZip() {
+    return myAbiJar;
+  }
+
+  @Override
+  public SourceSnapshot getSources() {
+    return null; // todo
+  }
+
+  @Override
+  public PathSnapshot getBinaryDependencies() {
+    return null; // todo
+  }
+
+  @Override
+  public BuilderArgs getBuilderArgs() {
+    return null; // todo
+  }
+
+  @Override
+  public NodeSourcePathMapper getPathMapper() {
+    return myPathMapper;
+  }
+
+  @Override
+  public BuildProcessLogger getBuildLogger() {
+    return BuildProcessLogger.EMPTY; // used for tests
+  }
+
+  @Override
+  public void cleanBuildState() {
+    // todo
+  }
+
+  @Override
+  public void report(Message msg) {
+    // todo
+  }
+
+  @Override
+  public boolean hasErrors() {
+    return false;
+  }
+
+  private static String truncateExtension(String filename) {
+    int idx = filename.lastIndexOf('.');
+    return idx >= 0? filename.substring(0, idx) : filename;
+  }
+}

jps/jps-builders/src/org/jetbrains/jps/bazel/impl/JavaCompilerRunner.java

@@ -13,7 +13,6 @@ import org.jetbrains.jps.bazel.runner.CompilerDataSink;
 import org.jetbrains.jps.bazel.runner.CompilerRunner;
 import org.jetbrains.jps.bazel.runner.OutputSink;
 import org.jetbrains.jps.builders.impl.java.JavacCompilerTool;
-import org.jetbrains.jps.dependency.GraphConfiguration;
 import org.jetbrains.jps.dependency.NodeSource;
 import org.jetbrains.jps.dependency.NodeSourcePathMapper;
 import org.jetbrains.jps.incremental.BinaryContent;
@@ -50,14 +49,12 @@ public class JavaCompilerRunner implements CompilerRunner {
   );
 
   private final BuildContext myContext;
-  private final GraphConfiguration myGraphConfig;
   private final List<String> myOptions;
   private final ModulePath myModulePath;
   private final Collection<File> myClassPath;
 
   public JavaCompilerRunner(BuildContext context) {
     myContext = context;
-    myGraphConfig = context.getGraphConfig();
     myOptions = getFilteredOptions(context);
 
     Collection<File> classpath = collect(map(context.getBinaryDependencies().getElements(), Path::toFile), new ArrayList<>()); // todo: convert relative path to abs path
@@ -106,8 +103,7 @@ public class JavaCompilerRunner implements CompilerRunner {
   // todo: install javac ast listener and consume data like in JpsReferenceDependenciesRegistrar
   @Override
   public ExitCode compile(Iterable<NodeSource> sources, DiagnosticSink diagnosticSink, OutputSink outSink) {
-
-    NodeSourcePathMapper pathMapper = myGraphConfig.getPathMapper();
+    NodeSourcePathMapper pathMapper = myContext.getPathMapper();
     OutputCollector outCollector = new OutputCollector(this, pathMapper, diagnosticSink, outSink);
     JavacCompilerTool javacTool = new JavacCompilerTool();
     // set non-null output, pointing to a non-existent dir. Need this to enable JavacFileManager creating OutputFileObjects
@@ -136,7 +132,7 @@ public class JavaCompilerRunner implements CompilerRunner {
   private static File findModuleInfo(BuildContext context) {
     @Nullable
     NodeSource moduleInfo = find(context.getSources().getElements(), ns -> ns.toString().endsWith(MODULE_INFO_FILE_SUFFIX));
-    return moduleInfo != null? context.getGraphConfig().getPathMapper().toPath(moduleInfo).toFile() : null;
+    return moduleInfo != null? context.getPathMapper().toPath(moduleInfo).toFile() : null;
   }
 
   private static class OutputCollector implements DiagnosticOutputConsumer, OutputFileConsumer {

jps/jps-builders/src/org/jetbrains/jps/bazel/impl/Utils.java

@@ -0,0 +1,33 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.jps.bazel.impl;
+
+import com.dynatrace.hash4j.hashing.Hashing;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+
+/** @noinspection NonFinalUtilityClass*/
+public class Utils {
+
+  public static String digest(byte[] bytes) {
+    return Long.toHexString(Hashing.xxh3_64().hashBytesToLong(bytes));
+  }
+
+  public static String digest(File file) {
+    return digest(file.toPath(), null);
+  }
+  
+  public static String digest(Path path, @Nullable BasicFileAttributes attrs) {
+    try {
+      return Long.toHexString((attrs != null? attrs.lastModifiedTime().toMillis() : Files.getLastModifiedTime(path).toMillis())); // todo
+    }
+    catch (IOException ignored) {
+      return "";
+    }
+  }
+
+}

jps/jps-builders/src/org/jetbrains/jps/bazel/runner/CompilerRunner.java

@@ -21,7 +21,7 @@ public interface CompilerRunner extends Runner{
     if (!context.isRebuild()) {
       BuildProcessLogger logger = context.getBuildLogger();
       if (logger.isEnabled() && !isEmpty(toCompile)) {
-        NodeSourcePathMapper pathMapper = context.getGraphConfig().getPathMapper();
+        NodeSourcePathMapper pathMapper = context.getPathMapper();
         logger.logCompiledPaths(map(toCompile, pathMapper::toPath), getName(), "Compiling files:");
       }
     }

jps/jps-builders/src/org/jetbrains/jps/dependency/GraphConfiguration.java

@@ -1,4 +1,4 @@
-// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.jps.dependency;
 
 import org.jetbrains.annotations.NotNull;
@@ -10,4 +10,18 @@ public interface GraphConfiguration {
 
   @NotNull
   DependencyGraph getGraph();
+
+  static GraphConfiguration create(@NotNull DependencyGraph graph, @NotNull NodeSourcePathMapper pathMapper) {
+    return new GraphConfiguration() {
+      @Override
+      public @NotNull NodeSourcePathMapper getPathMapper() {
+        return pathMapper;
+      }
+
+      @Override
+      public @NotNull DependencyGraph getGraph() {
+        return graph;
+      }
+    };
+  }
 }

jps/jps-builders/src/org/jetbrains/jps/dependency/impl/PersistentMVStoreMapletFactory.java

@@ -30,7 +30,7 @@ import java.util.Set;
 import java.util.function.Function;
 
 // suitable for relatively small amounts of stored data
-final class PersistentMVStoreMapletFactory implements MapletFactory, Closeable, Flushable {
+public final class PersistentMVStoreMapletFactory implements MapletFactory, Closeable, Flushable {
   private static final int BASE_CACHE_SIZE = 512 * (SystemProperties.getBooleanProperty(GlobalOptions.COMPILE_PARALLEL_OPTION, false)? 2 : 1);
   private static final int ALLOWED_STORE_COMPACTION_TIME_MS = -1; // -1 for full-compact, 0 to disable compaction
   private final MVSEnumerator myEnumerator;
@@ -41,7 +41,7 @@ final class PersistentMVStoreMapletFactory implements MapletFactory, Closeable,
   private final int myCacheSize;
   private final MVStore myStore;
 
-  PersistentMVStoreMapletFactory(String filePath, int maxBuilderThreads) throws IOException {
+  public PersistentMVStoreMapletFactory(String filePath, int maxBuilderThreads) throws IOException {
     Files.createDirectories(Path.of(filePath).getParent());
     // todo: need transaction store for transactions?
     myStore = new MVStore.Builder()