From ca3986ea11d8abebc81097a0b719fcd272afbed6 Mon Sep 17 00:00:00 2001 From: Eugene Zhuravlev Date: Sat, 3 May 2025 15:58:34 +0200 Subject: [PATCH] manage dependency graph on the builder side GitOrigin-RevId: 40ba09b472b9df8d4d1bda52912c1e03e8f7304b --- jps/jps-builders/api-dump.txt | 1 + .../jetbrains/jps/bazel/BazelIncBuilder.java | 77 +++++++----- .../org/jetbrains/jps/bazel/BuildContext.java | 21 +++- .../jps/bazel/impl/BuildContextImpl.java | 119 ++++++++++++++++++ .../jps/bazel/impl/JavaCompilerRunner.java | 8 +- .../org/jetbrains/jps/bazel/impl/Utils.java | 33 +++++ .../jps/bazel/runner/CompilerRunner.java | 2 +- .../jps/dependency/GraphConfiguration.java | 16 ++- .../impl/PersistentMVStoreMapletFactory.java | 4 +- 9 files changed, 241 insertions(+), 40 deletions(-) create mode 100644 jps/jps-builders/src/org/jetbrains/jps/bazel/impl/BuildContextImpl.java create mode 100644 jps/jps-builders/src/org/jetbrains/jps/bazel/impl/Utils.java diff --git a/jps/jps-builders/api-dump.txt b/jps/jps-builders/api-dump.txt index cabcec1c0eb3..311eaf45a2e2 100644 --- a/jps/jps-builders/api-dump.txt +++ b/jps/jps-builders/api-dump.txt @@ -1999,6 +1999,7 @@ org.jetbrains.jps.dependency.Graph - a:getSources():java.lang.Iterable - a:getSources(org.jetbrains.jps.dependency.ReferenceID):java.lang.Iterable org.jetbrains.jps.dependency.GraphConfiguration +- s:create(org.jetbrains.jps.dependency.DependencyGraph,org.jetbrains.jps.dependency.NodeSourcePathMapper):org.jetbrains.jps.dependency.GraphConfiguration - a:getGraph():org.jetbrains.jps.dependency.DependencyGraph - a:getPathMapper():org.jetbrains.jps.dependency.NodeSourcePathMapper org.jetbrains.jps.dependency.GraphDataInput diff --git a/jps/jps-builders/src/org/jetbrains/jps/bazel/BazelIncBuilder.java b/jps/jps-builders/src/org/jetbrains/jps/bazel/BazelIncBuilder.java index 057724db4760..41213455bb98 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/bazel/BazelIncBuilder.java +++ b/jps/jps-builders/src/org/jetbrains/jps/bazel/BazelIncBuilder.java @@ -6,14 +6,14 @@ import org.jetbrains.jps.bazel.impl.*; import org.jetbrains.jps.bazel.runner.BytecodeInstrumenter; import org.jetbrains.jps.bazel.runner.CompilerRunner; import org.jetbrains.jps.bazel.runner.RunnerFactory; -import org.jetbrains.jps.dependency.Delta; -import org.jetbrains.jps.dependency.DependencyGraph; -import org.jetbrains.jps.dependency.Node; -import org.jetbrains.jps.dependency.NodeSource; +import org.jetbrains.jps.dependency.*; +import org.jetbrains.jps.dependency.impl.DependencyGraphImpl; import org.jetbrains.jps.dependency.impl.GraphDataOutputImpl; import org.jetbrains.jps.dependency.impl.PathSource; +import org.jetbrains.jps.dependency.impl.PersistentMVStoreMapletFactory; import org.jetbrains.jps.dependency.java.JVMClassNode; +import java.io.Closeable; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -30,6 +30,7 @@ import static org.jetbrains.jps.javac.Iterators.*; public class BazelIncBuilder { private static final String SOURCE_SNAPSHOT_FILE_NAME = "src-snapshot.dat"; + private static final String DEP_GRAPH_FILE_NAME = "dep-graph.mv"; private static final List> ourCompilers = List.of( ResourcesCopy::new @@ -41,27 +42,40 @@ public class BazelIncBuilder { NotNullInstrumenter::new, FormsInstrumenter::new ); + private static GraphConfiguration setupGraphConfiguration(BuildContext context) throws IOException { + DependencyGraphImpl graph = new DependencyGraphImpl( + new PersistentMVStoreMapletFactory(context.getDataDir().resolve(DEP_GRAPH_FILE_NAME).toString(), Math.min(8, Runtime.getRuntime().availableProcessors())) + ); + return GraphConfiguration.create(graph, context.getPathMapper()); + } + + public ExitCode build(BuildContext context) { // todo: support cancellation checks - SourceSnapshotDelta snapshotDelta; - if (context.isRebuild()) { - snapshotDelta = new SourceSnapshotDeltaImpl(context.getSources()); - snapshotDelta.markRecompileAll(); - } - else { - snapshotDelta = new SourceSnapshotDeltaImpl(getOldSourceSnapshot(context), context.getSources()); - } - GraphUpdater graphUpdater = new GraphUpdater(context.getTargetName()); DiagnosticSink diagnostic = context; ZipOutputBuilder outputBuilder = null; + SourceSnapshotDelta snapshotDelta = null; + DependencyGraph depGraph = null; + try { - DependencyGraph depGraph = context.getGraphConfig().getGraph(); + if (context.isRebuild()) { + snapshotDelta = new SourceSnapshotDeltaImpl(context.getSources()); + snapshotDelta.markRecompileAll(); // force rebuild + } + else { + snapshotDelta = new SourceSnapshotDeltaImpl(loadSourceSnapshot(context), context.getSources()); + } + if (snapshotDelta.isRecompileAll()) { context.cleanBuildState(); } - else { + + GraphConfiguration graphConfig = setupGraphConfiguration(context); + depGraph = graphConfig.getGraph(); + + if (!snapshotDelta.isRecompileAll()) { // todo: process changes in libs // expand compile scope @@ -179,17 +193,12 @@ public class BazelIncBuilder { // report postponed errors, if necessary ((PostponedDiagnosticSink)diagnostic).drainTo(context); } - saveSourceSnapshot(context, snapshotDelta.asSnapshot()); - // todo: save abi-jar - if (outputBuilder != null) { - try { - outputBuilder.close(true); - } - catch (IOException e) { - diagnostic.report(Message.create(null, e)); - } + if (snapshotDelta != null) { + saveSourceSnapshot(context, snapshotDelta.asSnapshot()); } - // todo: close graph and save all caches + // todo: save abi-jar + safeClose(outputBuilder, diagnostic); + safeClose(depGraph, diagnostic); } } @@ -211,7 +220,7 @@ public class BazelIncBuilder { } private static void saveSourceSnapshot(BuildContext context, SourceSnapshot snapshot) { - Path snapshotPath = context.getBaseDir().resolve(SOURCE_SNAPSHOT_FILE_NAME); + Path snapshotPath = context.getDataDir().resolve(SOURCE_SNAPSHOT_FILE_NAME); try (var stream = new DataOutputStream(new DeflaterOutputStream(Files.newOutputStream(snapshotPath), new Deflater(Deflater.BEST_SPEED)))) { snapshot.write(new GraphDataOutputImpl(stream)); } @@ -220,8 +229,8 @@ public class BazelIncBuilder { } } - private static SourceSnapshot getOldSourceSnapshot(BuildContext context) { - Path oldSnapshot = context.getBaseDir().resolve(SOURCE_SNAPSHOT_FILE_NAME); + private static SourceSnapshot loadSourceSnapshot(BuildContext context) { + Path oldSnapshot = context.getDataDir().resolve(SOURCE_SNAPSHOT_FILE_NAME); try (var stream = new DataInputStream(new InflaterInputStream(Files.newInputStream(oldSnapshot, StandardOpenOption.READ)))) { return new SourceSnapshotImpl(stream, PathSource::new); } @@ -230,4 +239,16 @@ public class BazelIncBuilder { return SourceSnapshot.EMPTY; } } + + private static void safeClose(Closeable cl, DiagnosticSink diagnostic) { + if (cl == null) { + return; + } + try { + cl.close(); + } + catch (Throwable e) { + diagnostic.report(Message.create(null, e)); + } + } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/bazel/BuildContext.java b/jps/jps-builders/src/org/jetbrains/jps/bazel/BuildContext.java index 302571012937..c2db4c6e4d60 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/bazel/BuildContext.java +++ b/jps/jps-builders/src/org/jetbrains/jps/bazel/BuildContext.java @@ -1,7 +1,9 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.bazel; -import org.jetbrains.jps.dependency.GraphConfiguration; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.jps.dependency.NodeSourcePathMapper; import java.nio.file.Path; @@ -12,9 +14,24 @@ public interface BuildContext extends DiagnosticSink { boolean isCanceled(); + /** + * @return the BazelWorker working dir (can be a sandbox dir) + * Source and library inputs should be resolved against the base dir + */ + @NotNull Path getBaseDir(); + /** + * @return base directory where incremental data storages are to be stored + */ + @NotNull + Path getDataDir(); + + @NotNull Path getOutputZip(); + + @Nullable + Path getAbiOutputZip(); SourceSnapshot getSources(); @@ -22,7 +39,7 @@ public interface BuildContext extends DiagnosticSink { BuilderArgs getBuilderArgs(); - GraphConfiguration getGraphConfig(); + NodeSourcePathMapper getPathMapper(); BuildProcessLogger getBuildLogger(); // wipe graph, delete all caches, snapshots, storages diff --git a/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/BuildContextImpl.java b/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/BuildContextImpl.java new file mode 100644 index 000000000000..d87572fe0b7f --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/BuildContextImpl.java @@ -0,0 +1,119 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.bazel.impl; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.jps.bazel.*; +import org.jetbrains.jps.dependency.NodeSourcePathMapper; +import org.jetbrains.jps.dependency.impl.PathSourceMapper; + +import java.nio.file.Path; + +public class BuildContextImpl implements BuildContext { + private final String myTargetName; + private final Path myBaseDir; + private final PathSourceMapper myPathMapper; + @NotNull + private final Path myOutJar; + @Nullable + private final Path myAbiJar; + private final Path myDataDir; + + public BuildContextImpl(String targetName, Path baseDir, Path outJar, @Nullable Path abiJar, String cachePrefix) { + myTargetName = targetName; + myBaseDir = baseDir; + myPathMapper = new PathSourceMapper( + relPath -> { + Path abs = baseDir.resolve(Path.of(relPath)).normalize(); + return abs.toString().replace(baseDir.getFileSystem().getSeparator(), "/"); + }, + absPath -> { + Path relative = baseDir.relativize(Path.of(absPath)).normalize(); + return relative.toString().replace(baseDir.getFileSystem().getSeparator(), "/"); + } + ); + myOutJar = outJar; + myAbiJar = abiJar; + myDataDir = outJar.resolveSibling(cachePrefix + truncateExtension(outJar.getFileName().toString()) + "-ic"); + } + + @Override + public String getTargetName() { + return myTargetName; + } + + @Override + public boolean isRebuild() { + return false; // todo + } + + @Override + public boolean isCanceled() { + return false; + } + + @Override + public @NotNull Path getBaseDir() { + return myBaseDir; + } + + @Override + public @NotNull Path getDataDir() { + return myDataDir; + } + + @Override + public @NotNull Path getOutputZip() { + return myOutJar; + } + + @Override + public @Nullable Path getAbiOutputZip() { + return myAbiJar; + } + + @Override + public SourceSnapshot getSources() { + return null; // todo + } + + @Override + public PathSnapshot getBinaryDependencies() { + return null; // todo + } + + @Override + public BuilderArgs getBuilderArgs() { + return null; // todo + } + + @Override + public NodeSourcePathMapper getPathMapper() { + return myPathMapper; + } + + @Override + public BuildProcessLogger getBuildLogger() { + return BuildProcessLogger.EMPTY; // used for tests + } + + @Override + public void cleanBuildState() { + // todo + } + + @Override + public void report(Message msg) { + // todo + } + + @Override + public boolean hasErrors() { + return false; + } + + private static String truncateExtension(String filename) { + int idx = filename.lastIndexOf('.'); + return idx >= 0? filename.substring(0, idx) : filename; + } +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/JavaCompilerRunner.java b/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/JavaCompilerRunner.java index 80a2cdcaf526..5a168940683a 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/JavaCompilerRunner.java +++ b/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/JavaCompilerRunner.java @@ -13,7 +13,6 @@ import org.jetbrains.jps.bazel.runner.CompilerDataSink; import org.jetbrains.jps.bazel.runner.CompilerRunner; import org.jetbrains.jps.bazel.runner.OutputSink; import org.jetbrains.jps.builders.impl.java.JavacCompilerTool; -import org.jetbrains.jps.dependency.GraphConfiguration; import org.jetbrains.jps.dependency.NodeSource; import org.jetbrains.jps.dependency.NodeSourcePathMapper; import org.jetbrains.jps.incremental.BinaryContent; @@ -50,14 +49,12 @@ public class JavaCompilerRunner implements CompilerRunner { ); private final BuildContext myContext; - private final GraphConfiguration myGraphConfig; private final List myOptions; private final ModulePath myModulePath; private final Collection myClassPath; public JavaCompilerRunner(BuildContext context) { myContext = context; - myGraphConfig = context.getGraphConfig(); myOptions = getFilteredOptions(context); Collection classpath = collect(map(context.getBinaryDependencies().getElements(), Path::toFile), new ArrayList<>()); // todo: convert relative path to abs path @@ -106,8 +103,7 @@ public class JavaCompilerRunner implements CompilerRunner { // todo: install javac ast listener and consume data like in JpsReferenceDependenciesRegistrar @Override public ExitCode compile(Iterable sources, DiagnosticSink diagnosticSink, OutputSink outSink) { - - NodeSourcePathMapper pathMapper = myGraphConfig.getPathMapper(); + NodeSourcePathMapper pathMapper = myContext.getPathMapper(); OutputCollector outCollector = new OutputCollector(this, pathMapper, diagnosticSink, outSink); JavacCompilerTool javacTool = new JavacCompilerTool(); // set non-null output, pointing to a non-existent dir. Need this to enable JavacFileManager creating OutputFileObjects @@ -136,7 +132,7 @@ public class JavaCompilerRunner implements CompilerRunner { private static File findModuleInfo(BuildContext context) { @Nullable NodeSource moduleInfo = find(context.getSources().getElements(), ns -> ns.toString().endsWith(MODULE_INFO_FILE_SUFFIX)); - return moduleInfo != null? context.getGraphConfig().getPathMapper().toPath(moduleInfo).toFile() : null; + return moduleInfo != null? context.getPathMapper().toPath(moduleInfo).toFile() : null; } private static class OutputCollector implements DiagnosticOutputConsumer, OutputFileConsumer { diff --git a/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/Utils.java b/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/Utils.java new file mode 100644 index 000000000000..7954d07989da --- /dev/null +++ b/jps/jps-builders/src/org/jetbrains/jps/bazel/impl/Utils.java @@ -0,0 +1,33 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.jps.bazel.impl; + +import com.dynatrace.hash4j.hashing.Hashing; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +/** @noinspection NonFinalUtilityClass*/ +public class Utils { + + public static String digest(byte[] bytes) { + return Long.toHexString(Hashing.xxh3_64().hashBytesToLong(bytes)); + } + + public static String digest(File file) { + return digest(file.toPath(), null); + } + + public static String digest(Path path, @Nullable BasicFileAttributes attrs) { + try { + return Long.toHexString((attrs != null? attrs.lastModifiedTime().toMillis() : Files.getLastModifiedTime(path).toMillis())); // todo + } + catch (IOException ignored) { + return ""; + } + } + +} diff --git a/jps/jps-builders/src/org/jetbrains/jps/bazel/runner/CompilerRunner.java b/jps/jps-builders/src/org/jetbrains/jps/bazel/runner/CompilerRunner.java index 323e5996d2cc..8533cf6b5973 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/bazel/runner/CompilerRunner.java +++ b/jps/jps-builders/src/org/jetbrains/jps/bazel/runner/CompilerRunner.java @@ -21,7 +21,7 @@ public interface CompilerRunner extends Runner{ if (!context.isRebuild()) { BuildProcessLogger logger = context.getBuildLogger(); if (logger.isEnabled() && !isEmpty(toCompile)) { - NodeSourcePathMapper pathMapper = context.getGraphConfig().getPathMapper(); + NodeSourcePathMapper pathMapper = context.getPathMapper(); logger.logCompiledPaths(map(toCompile, pathMapper::toPath), getName(), "Compiling files:"); } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/dependency/GraphConfiguration.java b/jps/jps-builders/src/org/jetbrains/jps/dependency/GraphConfiguration.java index f2693364f8d3..4fbdf8e30581 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/dependency/GraphConfiguration.java +++ b/jps/jps-builders/src/org/jetbrains/jps/dependency/GraphConfiguration.java @@ -1,4 +1,4 @@ -// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.jps.dependency; import org.jetbrains.annotations.NotNull; @@ -10,4 +10,18 @@ public interface GraphConfiguration { @NotNull DependencyGraph getGraph(); + + static GraphConfiguration create(@NotNull DependencyGraph graph, @NotNull NodeSourcePathMapper pathMapper) { + return new GraphConfiguration() { + @Override + public @NotNull NodeSourcePathMapper getPathMapper() { + return pathMapper; + } + + @Override + public @NotNull DependencyGraph getGraph() { + return graph; + } + }; + } } diff --git a/jps/jps-builders/src/org/jetbrains/jps/dependency/impl/PersistentMVStoreMapletFactory.java b/jps/jps-builders/src/org/jetbrains/jps/dependency/impl/PersistentMVStoreMapletFactory.java index b7dea92f3bf0..08ec57b00c09 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/dependency/impl/PersistentMVStoreMapletFactory.java +++ b/jps/jps-builders/src/org/jetbrains/jps/dependency/impl/PersistentMVStoreMapletFactory.java @@ -30,7 +30,7 @@ import java.util.Set; import java.util.function.Function; // suitable for relatively small amounts of stored data -final class PersistentMVStoreMapletFactory implements MapletFactory, Closeable, Flushable { +public final class PersistentMVStoreMapletFactory implements MapletFactory, Closeable, Flushable { private static final int BASE_CACHE_SIZE = 512 * (SystemProperties.getBooleanProperty(GlobalOptions.COMPILE_PARALLEL_OPTION, false)? 2 : 1); private static final int ALLOWED_STORE_COMPACTION_TIME_MS = -1; // -1 for full-compact, 0 to disable compaction private final MVSEnumerator myEnumerator; @@ -41,7 +41,7 @@ final class PersistentMVStoreMapletFactory implements MapletFactory, Closeable, private final int myCacheSize; private final MVStore myStore; - PersistentMVStoreMapletFactory(String filePath, int maxBuilderThreads) throws IOException { + public PersistentMVStoreMapletFactory(String filePath, int maxBuilderThreads) throws IOException { Files.createDirectories(Path.of(filePath).getParent()); // todo: need transaction store for transactions? myStore = new MVStore.Builder()