BazelIncBuilder initial: main build logic

GitOrigin-RevId: 516f59741554a64b42537f280f23f0de1da2be3e
This commit is contained in:
Eugene Zhuravlev
2025-04-23 16:04:55 +02:00
committed by intellij-monorepo-bot
parent 50795e0f46
commit ce4b5f5116
30 changed files with 1323 additions and 0 deletions

View File

@@ -0,0 +1,183 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
import com.intellij.openapi.util.Pair;
import org.jetbrains.jps.bazel.impl.*;
import org.jetbrains.jps.bazel.runner.BytecodeInstrumenter;
import org.jetbrains.jps.bazel.runner.CompilerRunner;
import org.jetbrains.jps.dependency.Delta;
import org.jetbrains.jps.dependency.DependencyGraph;
import org.jetbrains.jps.dependency.Node;
import org.jetbrains.jps.dependency.NodeSource;
import org.jetbrains.jps.dependency.impl.GraphDataOutputImpl;
import org.jetbrains.jps.dependency.impl.PathSource;
import org.jetbrains.jps.dependency.java.JVMClassNode;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import static org.jetbrains.jps.javac.Iterators.*;
public class BazelIncBuilder {
private static final String SOURCE_SNAPSHOT_FILE_NAME = "src-snapshot.dat";
private static final List<CompilerRunner> ourCompilers = List.of(
new ResourcesCopy()
);
private static final List<CompilerRunner> ourRoundCompilers = List.of(
new KotlinCompilerRunner(), new JavaCompilerRunner()
);
private static final List<BytecodeInstrumenter> ourInstrumenters = List.of(
new NotNullInstrumenter(), new FormsInstrumenter()
);
public ExitCode build(BuildContext context) {
// todo: support cancellation checks
SourceSnapshotDelta snapshotDelta;
if (context.isRebuild()) {
snapshotDelta = new SnapshotDeltaImpl(context.getSources());
snapshotDelta.markRecompileAll();
}
else {
snapshotDelta = new SnapshotDeltaImpl(getOldSourceSnapshot(context), context.getSources());
}
GraphUpdater graphUpdater = new GraphUpdater(context.getTargetName());
DiagnosticSink diagnostic = context;
try {
if (snapshotDelta.isRecompileAll()) {
context.cleanBuildState();
}
else {
DependencyGraph depGraph = context.getGraphConfig().getGraph();
// todo: process changes in libs
// expand compile scope
Delta sourceOnlyDelta = depGraph.createDelta(snapshotDelta.getSourcesToRecompile(), snapshotDelta.getDeletedSources(), true);
snapshotDelta = graphUpdater.updateDependencyGraph(depGraph, snapshotDelta, sourceOnlyDelta, /*errorsDetected: */ false);
}
ZipOutputBuilder outputBuilder = new ZipOutputBuilderImpl(context.getOutputZip());
DependencyGraph depGraph = context.getGraphConfig().getGraph();
BuilderArgs builderArgs = context.getBuilderArgs();
boolean isInitialRound = true;
do {
diagnostic = isInitialRound? new PostponedDiagnosticSink() : context; // for initial round postpone error reporting
OutputSinkImpl outSink = new OutputSinkImpl(diagnostic, outputBuilder, ourInstrumenters);
if (isInitialRound) {
for (NodeSource source : filter(flat(snapshotDelta.getDeletedSources(), snapshotDelta.getSourcesToRecompile()), s -> find(ourCompilers, compiler -> compiler.canCompile(s)) != null)) {
// source paths are assumed to be relative to source roots, so under the output root the sirectory structure is the same
outputBuilder.deleteEntry(source.toString());
}
for (CompilerRunner runner : ourCompilers) {
runner.compile(snapshotDelta.getSourcesToRecompile(), builderArgs, diagnostic, outSink);
if (diagnostic.hasErrors()) {
break;
}
}
}
if (!diagnostic.hasErrors()) {
// delete outputs corresponding to deleted or recompiled sources
for (Node<?,?> node : flat(map(flat(snapshotDelta.getDeletedSources(), snapshotDelta.getSourcesToRecompile()), depGraph::getNodes))) {
if (node instanceof JVMClassNode) {
outputBuilder.deleteEntry(((JVMClassNode<?, ?>)node).getOutFilePath());
}
}
for (CompilerRunner runner : ourRoundCompilers) {
ExitCode code = runner.compile(snapshotDelta.getSourcesToRecompile(), builderArgs, diagnostic, outSink);
if (code == ExitCode.CANCEL) {
return code;
}
if (code == ExitCode.ERROR && !diagnostic.hasErrors()) {
// ensure we have some error message
diagnostic.report(Message.error(runner, runner.getName() + " completed with errors"));
}
if (diagnostic.hasErrors()) {
break;
}
}
}
SourceSnapshotDelta nextSnapshotDelta = graphUpdater.updateDependencyGraph(depGraph, snapshotDelta, createGraphDelta(depGraph, snapshotDelta, outSink), diagnostic.hasErrors());
if (!diagnostic.hasErrors()) {
snapshotDelta = nextSnapshotDelta;
}
else {
if (snapshotDelta.isRecompileAll() || !nextSnapshotDelta.hasChanges()) {
return ExitCode.ERROR;
}
// keep previous snapshot delta, just augment it with the newly found sources for recompilation
if (nextSnapshotDelta.isRecompileAll()) {
snapshotDelta.markRecompileAll();
}
else {
for (NodeSource source : nextSnapshotDelta.getSourcesToRecompile()) {
snapshotDelta.markRecompile(source);
}
}
if (!isInitialRound) {
return ExitCode.ERROR;
}
// for initial round, partial compilation and when analysis has expanded the scope, attempt automatic error recovery by repeating the compilation with the expanded scope
}
isInitialRound = false;
}
while (snapshotDelta.hasChanges());
// todo: save output jar and abi-jar
//outputBuilder.write(context.getOutputZip());
return ExitCode.OK;
}
finally {
if (diagnostic instanceof PostponedDiagnosticSink) {
// report postponed errors, if necessary
((PostponedDiagnosticSink)diagnostic).drainTo(context);
}
saveSourceSnapshot(context, snapshotDelta.asSnapshot());
// todo: close graph and save all caches
}
}
private static Delta createGraphDelta(DependencyGraph depGraph, SourceSnapshotDelta snapshotDelta, OutputSinkImpl outSink) {
Delta delta = depGraph.createDelta(snapshotDelta.getSourcesToRecompile(), snapshotDelta.getDeletedSources(), false);
for (Pair<Node<?, ?>, Iterable<NodeSource>> pair : outSink.getNodes()) {
delta.associate(pair.getFirst(), pair.getSecond());
}
return delta;
}
private static void saveSourceSnapshot(BuildContext context, SourceSnapshot snapshot) {
Path snapshotPath = context.getBaseDir().resolve(SOURCE_SNAPSHOT_FILE_NAME);
try (var stream = new DataOutputStream(new DeflaterOutputStream(Files.newOutputStream(snapshotPath), new Deflater(Deflater.BEST_SPEED)))) {
snapshot.write(new GraphDataOutputImpl(stream));
}
catch (Throwable e) {
context.report(Message.create(null, e));
}
}
private static SourceSnapshot getOldSourceSnapshot(BuildContext context) {
Path oldSnapshot = context.getBaseDir().resolve(SOURCE_SNAPSHOT_FILE_NAME);
try (var stream = new DataInputStream(new InflaterInputStream(Files.newInputStream(oldSnapshot, StandardOpenOption.READ)))) {
return new SourceSnapshotImpl(stream, PathSource::new);
}
catch (Throwable e) {
context.report(Message.create(null, e));
return SourceSnapshot.EMPTY;
}
}
}

View File

@@ -0,0 +1,25 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
import org.jetbrains.jps.dependency.GraphConfiguration;
import java.nio.file.Path;
public interface BuildContext extends DiagnosticSink {
String getTargetName();
boolean isRebuild();
Path getBaseDir();
Path getOutputZip();
SourceSnapshot getSources();
BuilderArgs getBuilderArgs();
GraphConfiguration getGraphConfig();
// wipe graph, delete all caches, snapshots, storages
void cleanBuildState();
}

View File

@@ -0,0 +1,5 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
public interface BuilderArgs {
}

View File

@@ -0,0 +1,9 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
public interface DiagnosticSink {
void report(Message msg);
boolean hasErrors();
}

View File

@@ -0,0 +1,6 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
public enum ExitCode {
OK, CANCEL, ERROR
}

View File

@@ -0,0 +1,104 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
import org.jetbrains.jps.bazel.impl.SnapshotDeltaImpl;
import org.jetbrains.jps.dependency.*;
import org.jetbrains.jps.dependency.impl.DifferentiateParametersBuilder;
import org.jetbrains.jps.incremental.dependencies.LibraryDef;
import org.jetbrains.jps.javac.Iterators;
import java.io.File;
import java.util.HashSet;
import java.util.Set;
public final class GraphUpdater {
private static final String MODULE_INFO_FILE_NAME = "module-info.java";
private final String myTargetName;
private final Set<NodeSource> myAllAffectedSources = new HashSet<>();
public GraphUpdater(String targetName) {
myTargetName = targetName;
}
public SourceSnapshotDelta updateDependencyGraph(DependencyGraph depGraph, SourceSnapshotDelta snapshotDelta, Delta delta, boolean errorsDetected) {
if (snapshotDelta.isRecompileAll()) {
if (errorsDetected || delta.isSourceOnly()) {
// do nothing
return new SnapshotDeltaImpl(snapshotDelta.getBaseSnapshot());
}
}
DifferentiateParameters params = DifferentiateParametersBuilder.create(myTargetName)
.compiledWithErrors(errorsDetected)
.calculateAffected(!snapshotDelta.isRecompileAll())
.processConstantsIncrementally(true)
.withAffectionFilter(s -> !LibraryDef.isLibraryPath(s))
.withChunkStructureFilter(s -> true).get();
DifferentiateResult diffResult = depGraph.differentiate(delta, params);
if (snapshotDelta.isRecompileAll()) {
depGraph.integrate(diffResult); // save full graph state
return new SnapshotDeltaImpl(snapshotDelta.getBaseSnapshot());
}
SourceSnapshotDelta nextSnapshotDelta = new SnapshotDeltaImpl(snapshotDelta.getBaseSnapshot());
if (!diffResult.isIncremental()) {
// recompile whole target, no integrate necessary
nextSnapshotDelta.markRecompileAll();
return nextSnapshotDelta;
}
if (!errorsDetected && params.isCalculateAffected()) {
// some compilers (and compiler plugins) may produce different outputs for the same set of inputs.
// This might cause corresponding graph Nodes to be considered as always 'changed'. In some scenarios this may lead to endless build loops
// This fallback logic detects such loops and recompiles the whole module chunk instead.
Set<NodeSource> affectedForChunk = Iterators.collect(Iterators.filter(diffResult.getAffectedSources(), params.belongsToCurrentCompilationChunk()::test), new HashSet<>());
if (!affectedForChunk.isEmpty() && !myAllAffectedSources.addAll(affectedForChunk)) {
// all affected files in this round have already been affected in previous rounds. This might indicate a build cycle => recompiling whole chunk
// todo: diagnostic
//LOG.info("Build cycle detected for " + chunk.getName() + "; recompiling whole module chunk");
// turn on non-incremental mode for the current target => next time the whole target is recompiled and affected files won't be calculated anymore
nextSnapshotDelta.markRecompileAll();
return nextSnapshotDelta;
}
}
for (NodeSource src : diffResult.getAffectedSources()) {
if (isJavaModuleInfo(src)) {
// recompile whole target, no integrate necessary
nextSnapshotDelta.markRecompileAll();
return nextSnapshotDelta;
}
nextSnapshotDelta.markRecompile(src);
}
if (delta.isSourceOnly()) {
// the delta does not correspond to real compilation session, files already marked for recompilation should be marked for recompilation in the next snapshot too
for (NodeSource source : snapshotDelta.getSourcesToRecompile()) {
nextSnapshotDelta.markRecompile(source);
}
}
if (!errorsDetected) {
depGraph.integrate(diffResult);
}
return nextSnapshotDelta;
}
private static boolean isJavaModuleInfo(NodeSource src) {
String path = src.toString();
if (!path.endsWith(MODULE_INFO_FILE_NAME)) {
return false;
}
if (path.length() > MODULE_INFO_FILE_NAME.length()) {
char separator = path.charAt(path.length() - MODULE_INFO_FILE_NAME.length() - 1);
return separator == '/' || separator == File.separatorChar;
}
return true;
}
}

View File

@@ -0,0 +1,74 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
import org.jetbrains.jps.bazel.runner.Runner;
/**
* @author Eugene Zhuravlev
* Date: 23 Apr 2025
*/
public
interface Message {
enum Kind {
ERROR, WARNING, INFO
}
Kind getKind();
String getText();
Runner getSource();
//-----------------------------------------------------------
static Message error(Runner reporter, String text) {
return create(reporter, Kind.ERROR, text);
}
static Message warning(Runner reporter, String text) {
return create(reporter, Kind.WARNING, text);
}
static Message info(Runner reporter, String text) {
return create(reporter, Kind.INFO, text);
}
static Message create(Runner reporter, Kind messageKind, String text) {
return new Message() {
@Override
public Kind getKind() {
return messageKind;
}
@Override
public String getText() {
return text;
}
@Override
public Runner getSource() {
return reporter;
}
};
}
static Message create(Runner reporter, Throwable ex) {
String text = ex.getMessage();
return new Message() {
@Override
public Kind getKind() {
return Kind.ERROR;
}
@Override
public String getText() {
return text;
}
@Override
public Runner getSource() {
return reporter;
}
};
}
}

View File

@@ -0,0 +1,39 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.dependency.GraphDataOutput;
import org.jetbrains.jps.dependency.NodeSource;
import org.jetbrains.jps.javac.Iterators;
import java.io.IOException;
import java.util.List;
public interface SourceSnapshot {
SourceSnapshot EMPTY = new SourceSnapshot() {
@Override
public @NotNull Iterable<@NotNull NodeSource> getSources() {
return List.of();
}
@Override
public @NotNull String getDigest(NodeSource src) {
return "";
}
};
@NotNull
Iterable<@NotNull NodeSource> getSources();
@NotNull
String getDigest(NodeSource src);
default void write(GraphDataOutput out) throws IOException {
Iterable<@NotNull NodeSource> sources = getSources();
out.writeInt(Iterators.count(sources));
for (NodeSource src : sources) {
out.writeUTF(getDigest(src));
src.write(out);
}
}
}

View File

@@ -0,0 +1,37 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.dependency.NodeSource;
public interface SourceSnapshotDelta {
@NotNull
SourceSnapshot getBaseSnapshot();
@NotNull
Iterable<@NotNull NodeSource> getDeletedSources();
@NotNull
Iterable<@NotNull NodeSource> getSourcesToRecompile();
boolean isRecompile(@NotNull NodeSource src);
void markRecompile(@NotNull NodeSource src);
boolean isRecompileAll();
default void markRecompileAll() {
for (NodeSource s : getBaseSnapshot().getSources()) {
markRecompile(s);
}
}
boolean hasChanges();
/**
* Provides a SourceSnapshot view for the delta where digests for files marked for recompilation are ignored
*/
SourceSnapshot asSnapshot();
}

View File

@@ -0,0 +1,19 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel;
import java.io.DataOutput;
public interface ZipOutputBuilder {
Iterable<String> getEntryNames();
boolean isDirectory(String entryName);
byte[] getContent(String entryName);
void putEntry(String entryName, byte[] content);
void deleteEntry(String entryName);
void write(DataOutput out);
}

View File

@@ -0,0 +1,21 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.jps.bazel.DiagnosticSink;
import org.jetbrains.jps.bazel.Message;
public abstract class DiagnosticSinkImpl implements DiagnosticSink {
protected boolean myHasErrors;
@Override
public void report(Message msg) {
if (msg.getKind() == Message.Kind.ERROR) {
myHasErrors = true;
}
}
@Override
public final boolean hasErrors() {
return myHasErrors;
}
}

View File

@@ -0,0 +1,20 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import com.intellij.compiler.instrumentation.InstrumentationClassFinder;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.bazel.runner.BytecodeInstrumenter;
import org.jetbrains.org.objectweb.asm.ClassReader;
import org.jetbrains.org.objectweb.asm.ClassWriter;
public class FormsInstrumenter implements BytecodeInstrumenter {
@Override
public String getName() {
return "Forms Instrumenter";
}
@Override
public byte @Nullable [] instrument(String filePath, ClassReader reader, ClassWriter writer, InstrumentationClassFinder finder) throws Exception {
return new byte[0]; // todo
}
}

View File

@@ -0,0 +1,28 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.jps.bazel.BuilderArgs;
import org.jetbrains.jps.bazel.DiagnosticSink;
import org.jetbrains.jps.bazel.ExitCode;
import org.jetbrains.jps.bazel.runner.CompilerRunner;
import org.jetbrains.jps.bazel.runner.OutputSink;
import org.jetbrains.jps.dependency.NodeSource;
public class JavaCompilerRunner implements CompilerRunner {
@Override
public String getName() {
return "Javac Runner";
}
@Override
public boolean canCompile(NodeSource src) {
return src.toString().endsWith(".java");
}
// todo: implement JavaCompilerToolExtension to listen to javac constants and registering them into outputConsumer
// todo: install javac ast lisneter and consume data like in JpsReferenceDependenciesRegistrar
@Override
public ExitCode compile(Iterable<NodeSource> sources, BuilderArgs args, DiagnosticSink diagnostic, OutputSink out) {
return ExitCode.OK; // todo
}
}

View File

@@ -0,0 +1,26 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.jps.bazel.BuilderArgs;
import org.jetbrains.jps.bazel.DiagnosticSink;
import org.jetbrains.jps.bazel.ExitCode;
import org.jetbrains.jps.bazel.runner.CompilerRunner;
import org.jetbrains.jps.bazel.runner.OutputSink;
import org.jetbrains.jps.dependency.NodeSource;
public class KotlinCompilerRunner implements CompilerRunner {
@Override
public String getName() {
return "Kotlinc Runner";
}
@Override
public boolean canCompile(NodeSource src) {
return src.toString().endsWith(".kt");
}
@Override
public ExitCode compile(Iterable<NodeSource> sources, BuilderArgs args, DiagnosticSink diagnostic, OutputSink out) {
return ExitCode.OK; // todo
}
}

View File

@@ -0,0 +1,21 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import com.intellij.compiler.instrumentation.InstrumentationClassFinder;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.bazel.runner.BytecodeInstrumenter;
import org.jetbrains.org.objectweb.asm.ClassReader;
import org.jetbrains.org.objectweb.asm.ClassWriter;
public class NotNullInstrumenter implements BytecodeInstrumenter {
@Override
public String getName() {
return "NotNull Instrumenter";
}
@Override
public byte @Nullable [] instrument(String filePath, ClassReader reader, ClassWriter writer, InstrumentationClassFinder finder) throws Exception {
return new byte[0]; // todo
}
}

View File

@@ -0,0 +1,44 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.bazel.runner.OutputSink;
public class OutputFileImpl implements OutputSink.OutputFile {
private static final byte[] EMPTY_CONTENT = new byte[0];
private final String myPath;
private final Kind myKind;
private final byte[] myContent;
private final boolean myFromGeneratedSource;
public OutputFileImpl(@NotNull String path, @NotNull Kind kind, byte @NotNull [] content) {
this(path, kind, content, false);
}
public OutputFileImpl(@NotNull String path, @NotNull Kind kind, byte @NotNull [] content, boolean fromGeneratedSource) {
myPath = path;
myKind = kind;
myContent = content.length == 0? EMPTY_CONTENT : content;
myFromGeneratedSource = fromGeneratedSource;
}
@Override
public Kind getKind() {
return myKind;
}
@Override
public @NotNull String getPath() {
return myPath;
}
@Override
public byte @NotNull [] getContent() {
return myContent;
}
@Override
public boolean isFromGeneratedSource() {
return myFromGeneratedSource;
}
}

View File

@@ -0,0 +1,233 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import com.intellij.compiler.instrumentation.FailSafeClassReader;
import com.intellij.compiler.instrumentation.InstrumentationClassFinder;
import com.intellij.compiler.instrumentation.InstrumenterClassWriter;
import com.intellij.openapi.util.Pair;
import kotlin.metadata.*;
import org.jetbrains.jps.bazel.DiagnosticSink;
import org.jetbrains.jps.bazel.Message;
import org.jetbrains.jps.bazel.ZipOutputBuilder;
import org.jetbrains.jps.bazel.runner.BytecodeInstrumenter;
import org.jetbrains.jps.bazel.runner.CompilerDataSink;
import org.jetbrains.jps.bazel.runner.OutputSink;
import org.jetbrains.jps.dependency.Node;
import org.jetbrains.jps.dependency.NodeSource;
import org.jetbrains.jps.dependency.Usage;
import org.jetbrains.jps.dependency.java.*;
import org.jetbrains.jps.javac.Iterators;
import org.jetbrains.org.objectweb.asm.ClassReader;
import org.jetbrains.org.objectweb.asm.ClassWriter;
import java.util.*;
public class OutputSinkImpl implements OutputSink, CompilerDataSink {
private static final String IMPORT_WILDCARD_SUFFIX = ".*";
private final DiagnosticSink myDiagnostic;
private final ZipOutputBuilder myOutBuilder;
private final List<BytecodeInstrumenter> myInstrumenters;
// -----------------------------------------------------------
private final Map<String, Pair<Collection<String>, Collection<String>>> myImportRefs = new HashMap<>();
private final Map<String, Collection<ConstantRef>> myConstantRefs = new HashMap<>();
private final Map<String, Set<Usage>> myAdditionalUsages = new HashMap<>();
private final Map<NodeSource, Set<Usage>> myPerSourceAdditionalUsages = new HashMap<>();
private final List<Pair<Node<?, ?>, Iterable<NodeSource>>> myNodes = new ArrayList<>();
private final Map<NodeSource, Set<Usage>> mySelfUsages = new HashMap<>();
public OutputSinkImpl(DiagnosticSink diagnostic, ZipOutputBuilder outBuilder, List<BytecodeInstrumenter> instrumenters) {
myDiagnostic = diagnostic;
myOutBuilder = outBuilder;
myInstrumenters = instrumenters;
}
@Override
public void addFile(OutputFile outFile, Iterable<NodeSource> originSources) {
// todo: make sure the outFile.getPath() is relative to output root
// todo: parse/instrument files and create nodes asynchronously?
processAndSave(outFile, originSources);
}
private void processAndSave(OutputFile outFile, Iterable<NodeSource> originSources) {
byte[] content = outFile.getContent();
try {
if (outFile.getKind() == OutputFile.Kind.bytecode) {
ClassReader reader = new FailSafeClassReader(content);
associate(outFile.getPath(), originSources, reader, outFile.isFromGeneratedSource());
InstrumentationClassFinder finder = getInstrumentationClassFinder();
if (finder != null) {
for (BytecodeInstrumenter instrumenter : myInstrumenters) {
try {
if (reader == null) {
reader = new FailSafeClassReader(content);
}
int version = InstrumenterClassWriter.getClassFileVersion(reader);
ClassWriter writer = new InstrumenterClassWriter(reader, InstrumenterClassWriter.getAsmClassWriterFlags(version), finder);
final byte[] instrumented = instrumenter.instrument(outFile.getPath(), reader, writer, finder);
if (instrumented != null) {
content = instrumented;
finder.cleanCachedData(reader.getClassName());
reader = null;
}
}
catch (Exception e) {
// todo: better diagnostics?
myDiagnostic.report(Message.error(instrumenter, e.getMessage()));
}
}
}
}
}
finally {
myOutBuilder.putEntry(outFile.getPath(), content);
}
}
private InstrumentationClassFinder getInstrumentationClassFinder() {
return null; // todo
}
private void associate(String classFileName, Iterable<NodeSource> sources, ClassReader cr, boolean isGenerated) {
JvmClassNodeBuilder builder = JvmClassNodeBuilder.create(classFileName, cr, isGenerated);
JvmNodeReferenceID nodeID = builder.getReferenceID();
String nodeName = nodeID.getNodeName();
addConstantUsages(builder, nodeName, myConstantRefs.remove(nodeName));
Pair<Collection<String>, Collection<String>> imports = myImportRefs.remove(nodeName);
if (imports != null) {
addImportUsages(builder, imports.getFirst(), imports.getSecond());
}
Set<Usage> additionalUsages = myAdditionalUsages.remove(nodeName);
if (additionalUsages != null) {
for (Usage usage : additionalUsages) {
builder.addUsage(usage);
}
}
var node = builder.getResult();
Iterable<LookupNameUsage> lookups = Iterators.flat(Iterators.map(node.getMetadata(KotlinMeta.class), meta -> {
KmDeclarationContainer container = meta.getDeclarationContainer();
final JvmNodeReferenceID owner;
LookupNameUsage clsUsage = null;
if (container instanceof KmPackage) {
owner = new JvmNodeReferenceID(JvmClass.getPackageName(node.getName()));
}
else if (container instanceof KmClass) {
owner = new JvmNodeReferenceID(((KmClass)container).getName());
String ownerName = owner.getNodeName();
String scopeName = JvmClass.getPackageName(ownerName);
String symbolName = scopeName.isEmpty()? ownerName : ownerName.substring(scopeName.length() + 1);
clsUsage = new LookupNameUsage(scopeName, symbolName);
}
else {
owner = null;
}
if (owner == null) {
return Collections.emptyList();
}
Iterable<LookupNameUsage> memberLookups =
Iterators.map(Iterators.unique(Iterators.flat(Iterators.map(container.getFunctions(), KmFunction::getName), Iterators.map(container.getProperties(), KmProperty::getName))), name -> new LookupNameUsage(owner, name));
return clsUsage == null? memberLookups : Iterators.flat(Iterators.asIterable(clsUsage), memberLookups);
}));
for (LookupNameUsage lookup : lookups) {
for (NodeSource src : sources) {
mySelfUsages.computeIfAbsent(src, s -> new HashSet<>()).add(lookup);
}
}
myNodes.add(new Pair<>(node, sources));
}
public List<Pair<Node<?, ?>, Iterable<NodeSource>>> getNodes() {
if (!myPerSourceAdditionalUsages.isEmpty()) {
for (Map.Entry<NodeSource, Set<Usage>> entry : myPerSourceAdditionalUsages.entrySet()) {
NodeSource src = entry.getKey();
Set<Usage> usages = entry.getValue();
Set<Usage> selfUsages = mySelfUsages.get(src);
if (selfUsages != null) {
usages.removeAll(selfUsages);
}
myNodes.add(new Pair<>(new FileNode(src.toString(), usages), List.of(src)));
}
myPerSourceAdditionalUsages.clear();
}
return myNodes;
}
@Override
public void registerImports(String className, Collection<String> classImports, Collection<String> staticImports) {
final String key = className.replace('.', '/');
if (!classImports.isEmpty() || !staticImports.isEmpty()) {
myImportRefs.put(key, Pair.create(classImports, staticImports));
}
else {
myImportRefs.remove(key);
}
}
@Override
public void registerConstantReferences(String className, Collection<ConstantRef> cRefs) {
final String key = className.replace('.', '/');
if (!cRefs.isEmpty()) {
myConstantRefs.put(key, cRefs);
}
else {
myConstantRefs.remove(key);
}
}
@Override
public void registerUsage(String className, Usage usage) {
myAdditionalUsages.computeIfAbsent(className.replace('.', '/'), k -> Collections.synchronizedSet(new HashSet<>())).add(usage);
}
@Override
public void registerUsage(NodeSource source, Usage usage) {
myPerSourceAdditionalUsages.computeIfAbsent(source, k -> Collections.synchronizedSet(new HashSet<>())).add(usage);
}
private static void addImportUsages(JvmClassNodeBuilder builder, Collection<String> classImports, Collection<String> staticImports) {
for (final String anImport : classImports) {
if (anImport.endsWith(IMPORT_WILDCARD_SUFFIX)) {
builder.addUsage(new ImportPackageOnDemandUsage(anImport.substring(0, anImport.length() - IMPORT_WILDCARD_SUFFIX.length()).replace('.', '/')));
}
else {
builder.addUsage(new ClassUsage(anImport.replace('.', '/')));
}
}
for (String anImport : staticImports) {
if (anImport.endsWith(IMPORT_WILDCARD_SUFFIX)) {
final String iname = anImport.substring(0, anImport.length() - IMPORT_WILDCARD_SUFFIX.length()).replace('.', '/');
builder.addUsage(new ClassUsage(iname));
builder.addUsage(new ImportStaticOnDemandUsage(iname));
}
else {
final int i = anImport.lastIndexOf('.');
if (i > 0 && i < anImport.length() - 1) {
final String iname = anImport.substring(0, i).replace('.', '/');
final String memberName = anImport.substring(i + 1);
builder.addUsage(new ClassUsage(iname));
builder.addUsage(new ImportStaticMemberUsage(iname, memberName));
}
}
}
}
private static void addConstantUsages(JvmClassNodeBuilder builder, String nodeName, Collection<? extends ConstantRef> cRefs) {
if (cRefs != null) {
for (ConstantRef ref : cRefs) {
final String constantOwner = ref.getOwner().replace('.', '/');
if (!constantOwner.equals(nodeName)) {
builder.addUsage(new FieldUsage(constantOwner, ref.getName(), ref.getDescriptor()));
}
}
}
}
}

View File

@@ -0,0 +1,26 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.jps.bazel.DiagnosticSink;
import org.jetbrains.jps.bazel.Message;
import java.util.ArrayList;
import java.util.List;
public final class PostponedDiagnosticSink extends DiagnosticSinkImpl {
private final List<Message> myMessages = new ArrayList<>();
@Override
public void report(Message msg) {
super.report(msg);
myMessages.add(msg);
}
public void drainTo(DiagnosticSink sink) {
for (Message message : myMessages) {
sink.report(message);
}
myMessages.clear();
myHasErrors = false;
}
}

View File

@@ -0,0 +1,27 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.jps.bazel.BuilderArgs;
import org.jetbrains.jps.bazel.DiagnosticSink;
import org.jetbrains.jps.bazel.ExitCode;
import org.jetbrains.jps.bazel.runner.CompilerRunner;
import org.jetbrains.jps.bazel.runner.OutputSink;
import org.jetbrains.jps.dependency.NodeSource;
public class ResourcesCopy implements CompilerRunner {
@Override
public String getName() {
return "Resources Copy";
}
@Override
public boolean canCompile(NodeSource src) {
String path = src.toString();
return !path.endsWith(".kt") && !path.endsWith(".java");
}
@Override
public ExitCode compile(Iterable<NodeSource> sources, BuilderArgs args, DiagnosticSink diagnostic, OutputSink out) {
return ExitCode.OK; // todo
}
}

View File

@@ -0,0 +1,100 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.bazel.SourceSnapshot;
import org.jetbrains.jps.bazel.SourceSnapshotDelta;
import org.jetbrains.jps.dependency.NodeSource;
import org.jetbrains.jps.dependency.diff.Difference;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import static org.jetbrains.jps.javac.Iterators.*;
public final class SnapshotDeltaImpl implements SourceSnapshotDelta {
private static final String RECOMPILED_SOURCE_DIGEST = "";
private final SourceSnapshot myBaseSnapshot;
private final Set<NodeSource> myDeleted = new HashSet<>();
private final Set<NodeSource> myRecompileMarked = new HashSet<>();
private boolean myIsWholeTargetRecompile;
public SnapshotDeltaImpl(SourceSnapshot base) {
myBaseSnapshot = base;
}
public SnapshotDeltaImpl(SourceSnapshot pastSnapshot, SourceSnapshot presentSnapshot) {
this(presentSnapshot);
if (!isEmpty(pastSnapshot.getSources())) {
Difference.Specifier<@NotNull NodeSource, Difference> diff = Difference.deepDiff(
pastSnapshot.getSources(), presentSnapshot.getSources(), NodeSource::equals, NodeSource::hashCode, (pastSrc, presentSrc) -> () -> Objects.equals(pastSnapshot.getDigest(pastSrc), presentSnapshot.getDigest(presentSrc))
);
collect(diff.removed(), myDeleted);
collect(flat(map(diff.changed(), Difference.Change::getPast), diff.added()), myRecompileMarked);
}
else {
myIsWholeTargetRecompile = true;
}
}
@Override
public @NotNull SourceSnapshot getBaseSnapshot() {
return myBaseSnapshot;
}
@Override
public boolean hasChanges() {
return myIsWholeTargetRecompile || !myRecompileMarked.isEmpty() || !myDeleted.isEmpty();
}
@Override
public @NotNull Iterable<@NotNull NodeSource> getDeletedSources() {
return myDeleted;
}
@Override
public @NotNull Iterable<@NotNull NodeSource> getSourcesToRecompile() {
return myIsWholeTargetRecompile? getBaseSnapshot().getSources() : myRecompileMarked;
}
@Override
public boolean isRecompile(@NotNull NodeSource src) {
return myIsWholeTargetRecompile || myRecompileMarked.contains(src);
}
@Override
public void markRecompile(@NotNull NodeSource src) {
if (!myIsWholeTargetRecompile) {
myRecompileMarked.add(src);
}
}
@Override
public boolean isRecompileAll() {
return myIsWholeTargetRecompile;
}
@Override
public void markRecompileAll() {
myIsWholeTargetRecompile = true;
myRecompileMarked.clear();
}
@Override
public SourceSnapshot asSnapshot() {
SourceSnapshot baseSnapshot = getBaseSnapshot();
return !hasChanges()? baseSnapshot : new SourceSnapshot() {
@Override
public @NotNull Iterable<@NotNull NodeSource> getSources() {
return flat(myDeleted, baseSnapshot.getSources());
}
@Override
public @NotNull String getDigest(NodeSource src) {
return isRecompile(src) || myDeleted.contains(src)? RECOMPILED_SOURCE_DIGEST : baseSnapshot.getDigest(src);
}
};
}
}

View File

@@ -0,0 +1,53 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.bazel.SourceSnapshot;
import org.jetbrains.jps.dependency.DataReader;
import org.jetbrains.jps.dependency.GraphDataInput;
import org.jetbrains.jps.dependency.GraphDataOutput;
import org.jetbrains.jps.dependency.NodeSource;
import org.jetbrains.jps.dependency.impl.GraphDataInputImpl;
import java.io.DataInput;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class SourceSnapshotImpl implements SourceSnapshot {
private final Map<NodeSource, String> mySources;
public SourceSnapshotImpl(Map<NodeSource, String> digestSources) {
mySources = Map.copyOf(digestSources);
}
public SourceSnapshotImpl(DataInput in, DataReader<? extends NodeSource> sourceReader) throws IOException {
GraphDataInput _inp = in instanceof GraphDataInput ? ((GraphDataInput) in) : new GraphDataInputImpl(in);
Map<NodeSource, String> sources = new HashMap<>();
int count = _inp.readInt();
while (count-- > 0) {
String digest = _inp.readUTF();
sources.put(sourceReader.load(_inp), digest);
}
mySources = Map.copyOf(sources);
}
@Override
public @NotNull Iterable<@NotNull NodeSource> getSources() {
return mySources.keySet();
}
@Override
public @NotNull String getDigest(NodeSource src) {
return mySources.getOrDefault(src, "");
}
@Override
public void write(GraphDataOutput out) throws IOException {
out.writeInt(mySources.size());
for (Map.Entry<NodeSource, String> entry : mySources.entrySet()) {
out.writeUTF(entry.getValue());
entry.getKey().write(out);
}
}
}

View File

@@ -0,0 +1,99 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jps.bazel.ZipOutputBuilder;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.Map;
import java.util.TreeMap;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
public class ZipOutputBuilderImpl implements ZipOutputBuilder {
private static final byte[] EMPTY_BYTES = new byte[0];
private final Map<String, EntryData> myEntries = new TreeMap<>();
public ZipOutputBuilderImpl(Path outputZip) {
// todo: init from the previous zip
}
@Override
public Iterable<String> getEntryNames() {
return myEntries.keySet();
}
@Override
public boolean isDirectory(String entryName) {
return isDirectoryName(entryName);
}
@Override
public byte[] getContent(String entryName) {
try {
return myEntries.getOrDefault(entryName, EntryData.EMPTY).getContent();
}
catch (IOException e) {
// todo: diagnostics
return EMPTY_BYTES;
}
}
@Override
public void putEntry(String entryName, byte[] content) {
// todo: create intermediate directory entries
myEntries.put(entryName, EntryData.create(content));
}
@Override
public void deleteEntry(String entryName) {
if (myEntries.remove(entryName) != null) {
// todo: update parent intermediate entry
}
}
@Override
public void write(DataOutput out) {
// todo
}
@Nullable
private static String getParent(String entryName) {
int idx = isDirectoryName(entryName)? entryName.lastIndexOf('/', entryName.length() - 2) : entryName.lastIndexOf('/');
return idx >= 0? entryName.substring(0, idx + 1) : null;
}
private static boolean isDirectoryName(String entryName) {
return entryName.endsWith("/");
}
private interface EntryData {
EntryData EMPTY = () -> EMPTY_BYTES;
byte[] getContent() throws IOException;
static EntryData create(byte[] content) {
return () -> content;
}
static EntryData create(ZipFile zf, ZipEntry ze) {
return new EntryData() {
private byte[] loaded;
@Override
public byte[] getContent() throws IOException {
if (loaded == null) {
try (InputStream is = zf.getInputStream(ze)) {
loaded = is.readAllBytes();
}
}
return loaded;
}
};
}
}
}

View File

@@ -0,0 +1,6 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package org.jetbrains.jps.bazel.impl;
import org.jetbrains.annotations.ApiStatus;

View File

@@ -0,0 +1,6 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package org.jetbrains.jps.bazel;
import org.jetbrains.annotations.ApiStatus;

View File

@@ -0,0 +1,14 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.runner;
import com.intellij.compiler.instrumentation.InstrumentationClassFinder;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.org.objectweb.asm.ClassReader;
import org.jetbrains.org.objectweb.asm.ClassWriter;
public interface BytecodeInstrumenter extends Runner{
/**
* @return null, if instrumentation did not happen, otherwise an instrumented content
*/
byte @Nullable [] instrument(String filePath, ClassReader reader, ClassWriter writer, InstrumentationClassFinder finder) throws Exception;
}

View File

@@ -0,0 +1,46 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.runner;
import org.jetbrains.jps.dependency.NodeSource;
import org.jetbrains.jps.dependency.Usage;
import java.util.Collection;
public interface CompilerDataSink {
interface ConstantRef {
String getOwner();
String getName();
String getDescriptor();
static ConstantRef create(String ownerClass, String fieldName, String descriptor) {
return new ConstantRef() {
@Override
public String getOwner() {
return ownerClass;
}
@Override
public String getName() {
return fieldName;
}
@Override
public String getDescriptor() {
return descriptor;
}
};
}
}
void registerImports(String className, Collection<String> classImports, Collection<String> staticImports);
void registerConstantReferences(String className, Collection<ConstantRef> cRefs);
default void registerUsage(String className, Usage usage) {
}
default void registerUsage(NodeSource source, Usage usage) {
}
}

View File

@@ -0,0 +1,13 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.runner;
import org.jetbrains.jps.bazel.BuilderArgs;
import org.jetbrains.jps.bazel.DiagnosticSink;
import org.jetbrains.jps.bazel.ExitCode;
import org.jetbrains.jps.dependency.NodeSource;
public interface CompilerRunner extends Runner{
boolean canCompile(NodeSource src);
ExitCode compile(Iterable<NodeSource> sources, BuilderArgs args, DiagnosticSink diagnostic, OutputSink out);
}

View File

@@ -0,0 +1,27 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.runner;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jps.dependency.NodeSource;
public interface OutputSink {
interface OutputFile {
enum Kind {
bytecode, source, other
}
Kind getKind();
@NotNull String getPath();
byte @NotNull [] getContent();
default boolean isFromGeneratedSource() {
return false;
}
}
void addFile(OutputFile outFile, Iterable<NodeSource> originSources);
}

View File

@@ -0,0 +1,6 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package org.jetbrains.jps.bazel.runner;
public interface Runner {
String getName();
}

View File

@@ -0,0 +1,6 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package org.jetbrains.jps.bazel.runner;
import org.jetbrains.annotations.ApiStatus;