From 7e970c6cb35077de598c3a77d19100dddc1bb06c Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Sun, 5 Jan 2025 10:13:02 +0100 Subject: [PATCH] bazel jps compiler (part 2) GitOrigin-RevId: 8e61f22cab4bbac11555cb4f2234735cb469b3c0 --- build/jvm-rules/.gitignore | 2 +- build/jvm-rules/BUILD.bazel | 15 +- build/jvm-rules/MODULE.bazel | 25 +- build/jvm-rules/libs.lock.json | 12 +- build/jvm-rules/libs.yaml | 4 +- build/jvm-rules/rules/common-attrs.bzl | 1 + build/jvm-rules/rules/impl/associates.bzl | 14 - build/jvm-rules/rules/impl/builder-args.bzl | 51 +- build/jvm-rules/rules/impl/compile.bzl | 240 +--- build/jvm-rules/rules/library.bzl | 1 - build/jvm-rules/rules/test.bzl | 1 - build/jvm-rules/rules/trace.bzl | 10 - .../src/compiler-util/JvmBuilderFlags.kt | 17 +- build/jvm-rules/src/jps-builder/BUILD.bazel | 39 +- .../src/jps-builder/BazelJpsModelLoader.kt | 183 ++- .../src/jps-builder/BazelJpsServiceManager.kt | 10 +- .../src/jps-builder/BazelSharedThreadPool.kt | 44 +- build/jvm-rules/src/jps-builder/JpsBuilder.kt | 337 +++-- .../src/jps-builder/JpsProjectBuilder.kt | 1240 ----------------- .../src/jps-builder/StorageInitializer.kt | 162 +++ .../src/jps-builder/TestJpsBuildWorker.kt | 71 +- .../jps-builder/impl/BazelBuildRootIndex.kt | 92 ++ .../impl/BazelBuildTargetStateManager.kt | 62 + .../impl/BazelModuleBuildTarget.kt | 63 + .../src/jps-builder/impl/BuildTaskLauncher.kt | 47 +- .../src/jps-builder/impl/JpsModuleBuilder.kt | 882 ++++++++++++ ...ludeIndex.kt => NoopModuleExcludeIndex.kt} | 3 +- build/jvm-rules/src/jps-builder/impl/load.kt | 84 ++ .../src/jps-builder/impl/pathRelativizer.kt | 69 + build/jvm-rules/src/jps-builder/logger.kt | 66 + .../src/jps-builder/packager/BUILD.bazel | 20 + .../packager/ConsoleMessageHandler.kt | 62 + .../packager/KotlinAnnotationVisitor.kt | 95 ++ .../jvm-rules/src/jps-builder/packager/abi.kt | 268 ++++ .../src/jps-builder/packager/packager.kt | 175 +++ .../src/jps-builder/state/configuration.kt | 36 + build/jvm-rules/src/jps-builder/state/hash.kt | 16 + .../jvm-rules/src/jps-builder/state/state.kt | 31 + .../src/jps-builder/state/targetState.kt | 80 ++ .../src/kotlin-builder/KotlinBuilder.kt | 10 +- .../kotlin-builder/KotlinJvmTaskExecutor.kt | 2 +- .../src/kotlin-builder/compilation/plugins.kt | 4 +- .../abi/JvmAbiMetadataProcessor.kt | 347 ++--- build/jvm-rules/src/misc/JvmWorker.kt | 15 +- .../{AsyncLogger.kt => AsyncFileLogger.kt} | 2 +- .../src/worker-framework/BUILD.bazel | 5 +- .../worker-framework/WorkRequestHandler.kt | 132 +- .../WorkRequestHandlerTest.kt | 199 +-- .../src/worker-framework/protocol.kt | 133 ++ build/jvm-rules/zip/BUILD.bazel | 1 + build/jvm-rules/zip/src/ZipFileWriter.kt | 21 +- lib/BUILD.bazel | 91 +- lib/MODULE.bazel | 140 +- .../build-scripts/codeOptimizer/BUILD.bazel | 5 +- 54 files changed, 3352 insertions(+), 2385 deletions(-) delete mode 100644 build/jvm-rules/rules/trace.bzl delete mode 100644 build/jvm-rules/src/jps-builder/JpsProjectBuilder.kt create mode 100644 build/jvm-rules/src/jps-builder/StorageInitializer.kt create mode 100644 build/jvm-rules/src/jps-builder/impl/BazelBuildRootIndex.kt create mode 100644 build/jvm-rules/src/jps-builder/impl/BazelBuildTargetStateManager.kt create mode 100644 build/jvm-rules/src/jps-builder/impl/BazelModuleBuildTarget.kt create mode 100644 build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt rename build/jvm-rules/src/jps-builder/impl/{BazelModuleExcludeIndex.kt => NoopModuleExcludeIndex.kt} (77%) create mode 100644 build/jvm-rules/src/jps-builder/impl/load.kt create mode 100644 build/jvm-rules/src/jps-builder/impl/pathRelativizer.kt create mode 100644 build/jvm-rules/src/jps-builder/logger.kt create mode 100644 build/jvm-rules/src/jps-builder/packager/BUILD.bazel create mode 100644 build/jvm-rules/src/jps-builder/packager/ConsoleMessageHandler.kt create mode 100644 build/jvm-rules/src/jps-builder/packager/KotlinAnnotationVisitor.kt create mode 100644 build/jvm-rules/src/jps-builder/packager/abi.kt create mode 100644 build/jvm-rules/src/jps-builder/packager/packager.kt create mode 100644 build/jvm-rules/src/jps-builder/state/configuration.kt create mode 100644 build/jvm-rules/src/jps-builder/state/hash.kt create mode 100644 build/jvm-rules/src/jps-builder/state/state.kt create mode 100644 build/jvm-rules/src/jps-builder/state/targetState.kt rename build/jvm-rules/src/worker-framework/{AsyncLogger.kt => AsyncFileLogger.kt} (95%) create mode 100644 build/jvm-rules/src/worker-framework/protocol.kt diff --git a/build/jvm-rules/.gitignore b/build/jvm-rules/.gitignore index 8cbab2ef802b..ff2ccbe2d23d 100644 --- a/build/jvm-rules/.gitignore +++ b/build/jvm-rules/.gitignore @@ -1,4 +1,4 @@ .idea bazel-* .bazelbsp/ -src/jps-builder/jps-build-standalone* \ No newline at end of file +src/jps-builder/jps-build-standalone.jar \ No newline at end of file diff --git a/build/jvm-rules/BUILD.bazel b/build/jvm-rules/BUILD.bazel index aea98e162e11..c514ce0327b4 100644 --- a/build/jvm-rules/BUILD.bazel +++ b/build/jvm-rules/BUILD.bazel @@ -2,7 +2,7 @@ load("@rules_java//java:defs.bzl", "java_import") load("@rules_kotlin//kotlin:core.bzl", _kt_kotlinc_options = "kt_kotlinc_options") load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import", "kt_jvm_library") load("//:jvm.bzl", "jvm_import", "kt_kotlinc_options") -load(":rules/trace.bzl", "trace") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "int_flag") label_flag( name = "default-kotlinc-opts", @@ -23,15 +23,20 @@ kt_kotlinc_options( ) # --@rules_jvm//:kt_trace -trace( +bool_flag( name = "kt_trace", build_setting_default = False, ) +# --@rules_jvm//:jps_threshold=500 +int_flag( + name = "jps_threshold", + build_setting_default = -1, +) _kt_kotlinc_options( name = "rules_jvm_bootstrap_kotlinc_options", include_stdlibs = "none", - visibility = ["//src:__subpackages__"], + visibility = ["//src:__subpackages__", "//zip:__pkg__"], ) ### auto-generated section `libraries` start @@ -77,7 +82,7 @@ jvm_import( visibility = ["//visibility:public"], runtime_deps = [ "@lib//:jetbrains-annotations", - ":kotlinx-coroutines-core", + ":kotlinx-coroutines-core" ], ) @@ -123,4 +128,4 @@ jvm_import( visibility = ["//visibility:public"], ) -### auto-generated section `libraries` end +### auto-generated section `libraries` end \ No newline at end of file diff --git a/build/jvm-rules/MODULE.bazel b/build/jvm-rules/MODULE.bazel index b317f81f7db4..825ad3b73590 100644 --- a/build/jvm-rules/MODULE.bazel +++ b/build/jvm-rules/MODULE.bazel @@ -6,6 +6,7 @@ bazel_dep(name = "rules_kotlin", version = "2.0.0-jb.5") bazel_dep(name = "rules_graalvm", version = "0.11.3") bazel_dep(name = "lib") bazel_dep(name = "protobuf", version = "29.2") +bazel_dep(name = "bazel_skylib", version = "1.7.1") # https://github.com/bazelbuild/bazel/issues/6681#issuecomment-2188972754 bazel_dep(name = "contrib_rules_jvm", version = "0.27.0") @@ -191,28 +192,28 @@ http_file( http_file( name = "poko-compiler-plugin", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/dev/drewhamilton/poko/poko-compiler-plugin/0.18.1/poko-compiler-plugin-0.18.1.jar", - sha256 = "4def45d16f7b077c8f4fa31f46f940bff14ecb985408fd74ac26c7910576cf00", - downloaded_file_path = "poko-compiler-plugin-0.18.1.jar", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/dev/drewhamilton/poko/poko-compiler-plugin/0.18.2/poko-compiler-plugin-0.18.2.jar", + sha256 = "08edf341dfa4dd0a9e15b83a2a74850baad3a4a3ca2f93aed05ca6a481b1f394", + downloaded_file_path = "poko-compiler-plugin-0.18.2.jar", ) http_file( name = "poko-compiler-plugin-sources", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/dev/drewhamilton/poko/poko-compiler-plugin/0.18.1/poko-compiler-plugin-0.18.1-sources.jar", - sha256 = "e33785f02bafc11751914ff1892c0a302a6408edb0ca6076436a71150b201e55", - downloaded_file_path = "poko-compiler-plugin-0.18.1-sources.jar", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/dev/drewhamilton/poko/poko-compiler-plugin/0.18.2/poko-compiler-plugin-0.18.2-sources.jar", + sha256 = "02f7ac954f3a9574b37a54d7bdc79190325b9ee38a3244256a84d6d760eb6f1f", + downloaded_file_path = "poko-compiler-plugin-0.18.2-sources.jar", ) http_file( name = "kotlinx-coroutines-core-jvm", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.9.0/kotlinx-coroutines-core-jvm-1.9.0.jar", - sha256 = "ad89c2892235e670f222d819cb3d81188143cb19a05b59df9889ae4269f5c70a", - downloaded_file_path = "kotlinx-coroutines-core-jvm-1.9.0.jar", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.10.1/kotlinx-coroutines-core-jvm-1.10.1.jar", + sha256 = "069c5988633230e074ec0d39321ec3cdaa4547c49e90ba936c63d8fc91c8c00d", + downloaded_file_path = "kotlinx-coroutines-core-jvm-1.10.1.jar", ) http_file( name = "kotlinx-coroutines-core-jvm-sources", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.9.0/kotlinx-coroutines-core-jvm-1.9.0-sources.jar", - sha256 = "01e0f26a274d8f2f2ae702b161bc0980cea817bfcaf66ca4944434ea7b601771", - downloaded_file_path = "kotlinx-coroutines-core-jvm-1.9.0-sources.jar", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.10.1/kotlinx-coroutines-core-jvm-1.10.1-sources.jar", + sha256 = "3faa298de8000d68744a179f7d413ab1672acf1d77cb8d6be57a78544657edd2", + downloaded_file_path = "kotlinx-coroutines-core-jvm-1.10.1-sources.jar", ) ### auto-generated section `libraries` end diff --git a/build/jvm-rules/libs.lock.json b/build/jvm-rules/libs.lock.json index 8cf29d2fcae1..95fc009cfb1d 100644 --- a/build/jvm-rules/libs.lock.json +++ b/build/jvm-rules/libs.lock.json @@ -3,9 +3,9 @@ "7441a6394dca477b3f8b7c86f17bef035f705a8aaa11f53065d380300158177c", "68f0f47297335e2c3fb5a904ade91aadef884b3aa8f558a4efac16ba803b8b27" ], - "dev.drewhamilton.poko:poko-compiler-plugin:0.18.1": [ - "4def45d16f7b077c8f4fa31f46f940bff14ecb985408fd74ac26c7910576cf00", - "e33785f02bafc11751914ff1892c0a302a6408edb0ca6076436a71150b201e55" + "dev.drewhamilton.poko:poko-compiler-plugin:0.18.2": [ + "08edf341dfa4dd0a9e15b83a2a74850baad3a4a3ca2f93aed05ca6a481b1f394", + "02f7ac954f3a9574b37a54d7bdc79190325b9ee38a3244256a84d6d760eb6f1f" ], "org.jetbrains.kotlin:kotlin-build-tools-api:2.1.20-Beta1": [ "09ddebb845f30310395d91c084f65d2133484080dfac1977502776b77345f3d7", @@ -43,8 +43,8 @@ "f48b6323092cb30f860cb5c880d6107982d82941166073871a36349dfd3f7909", "4aa4da6df01507c0fe9b48ff4f3d2a5eb5edf5f894850169c90efc89330a9d8b" ], - "org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.9.0": [ - "ad89c2892235e670f222d819cb3d81188143cb19a05b59df9889ae4269f5c70a", - "01e0f26a274d8f2f2ae702b161bc0980cea817bfcaf66ca4944434ea7b601771" + "org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.10.1": [ + "069c5988633230e074ec0d39321ec3cdaa4547c49e90ba936c63d8fc91c8c00d", + "3faa298de8000d68744a179f7d413ab1672acf1d77cb8d6be57a78544657edd2" ] } \ No newline at end of file diff --git a/build/jvm-rules/libs.yaml b/build/jvm-rules/libs.yaml index b59401145b35..7c29e773c508 100644 --- a/build/jvm-rules/libs.yaml +++ b/build/jvm-rules/libs.yaml @@ -28,7 +28,7 @@ - id: org.jetbrains.kotlin:kotlin-compose-compiler-plugin version: 2.1.20-Beta1 - id: dev.drewhamilton.poko:poko-compiler-plugin - version: 0.18.1 + version: 0.18.2 # other libs - id: org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm - version: 1.9.0 + version: 1.10.1 diff --git a/build/jvm-rules/rules/common-attrs.bzl b/build/jvm-rules/rules/common-attrs.bzl index e33eb133e7ad..2735b5c8ae3e 100644 --- a/build/jvm-rules/rules/common-attrs.bzl +++ b/build/jvm-rules/rules/common-attrs.bzl @@ -133,6 +133,7 @@ common_attr = add_dicts( ), "_reduced_classpath": attr.bool(default = False), "_trace": attr.label(default = "//:kt_trace"), + "_jps_threshold": attr.label(default = "//:jps_threshold"), }, ) diff --git a/build/jvm-rules/rules/impl/associates.bzl b/build/jvm-rules/rules/impl/associates.bzl index 7d99f481f1d7..ca9e71d8594f 100644 --- a/build/jvm-rules/rules/impl/associates.bzl +++ b/build/jvm-rules/rules/impl/associates.bzl @@ -45,20 +45,6 @@ def get_associates(ctx): module_name = module_names.keys()[0], ) -def flatten_jars(nested_jars_depset): - """Returns a list of strings containing the compile_jars for depset of targets. - - This ends up unwinding the nesting of depsets, since compile_jars contains depsets inside - the nested_jars targets, which themselves are depsets. This function is intended to be called - lazily form within Args.add_all(map_each) as it collapses depsets. - """ - compile_jars_depsets = [ - target[JavaInfo].compile_jars - for target in nested_jars_depset.to_list() - if target[JavaInfo].compile_jars - ] - return [file.path for file in depset(transitive = compile_jars_depsets).to_list()] - def _derive_module_name(ctx): """Gets the `module_name` attribute if it's set in the ctx, otherwise derive a unique module name using the elements found in the label.""" diff --git a/build/jvm-rules/rules/impl/builder-args.bzl b/build/jvm-rules/rules/impl/builder-args.bzl index 31975a2544a5..ee3a6a6194d1 100644 --- a/build/jvm-rules/rules/impl/builder-args.bzl +++ b/build/jvm-rules/rules/impl/builder-args.bzl @@ -1,8 +1,11 @@ -load("//:rules/trace.bzl", "TraceInfo") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@rules_java//java:defs.bzl", "JavaInfo") +load("//:rules/impl/associates.bzl", "get_associates") +load("//:rules/impl/kotlinc-options.bzl", "KotlincOptions", "kotlinc_options_to_flags") visibility("private") -def init_builder_args(ctx, rule_kind, module_name): +def init_builder_args(ctx, rule_kind, associates, transitiveInputs, plugins, compile_jars): """Initialize an arg object for a task that will be executed by the Kotlin Builder.""" args = ctx.actions.args() args.set_param_file_format("multiline") @@ -10,11 +13,49 @@ def init_builder_args(ctx, rule_kind, module_name): args.add("--target_label", ctx.label) args.add("--rule_kind", rule_kind) - args.add("--kotlin_module_name", module_name) + args.add("--kotlin_module_name", associates.module_name) - trace = ctx.attr._trace[TraceInfo].trace + kotlinc_options = ctx.attr.kotlinc_opts[KotlincOptions] - if ctx.attr._trace[TraceInfo].trace: + if kotlinc_options.report_unused_deps: + deps_artifacts = _collect_jdeps_from_artifacts(ctx.attr.deps + associates.targets) + transitiveInputs.append(deps_artifacts) + args.add_all("--deps_artifacts", deps_artifacts) + + if associates: + args.add_all("--friends", associates.jars, map_each = _flatten_jars) + + if ctx.attr._trace[BuildSettingInfo].value: args.add("--trace") + kotlinc_options_to_flags(kotlinc_options, args) + + args.add_all("--opt-in", kotlinc_options.opt_in) + args.add_all("--classpath", compile_jars) + + for id, classpath in plugins.compile_phase.classpath.items(): + args.add("--plugin-id", id) + args.add_joined("--plugin-classpath", classpath, omit_if_empty = False, join_with = ":") + transitiveInputs.append(classpath) + return args + +def _collect_jdeps_from_artifacts(all_targets): + """Collect Jdeps artifacts if required.""" + + # collect `.jdeps` outputs from targets that contain JavaInfo and have a `jdeps` output + return depset([t[JavaInfo].outputs.jdeps for t in all_targets if JavaInfo in t and t[JavaInfo].outputs.jdeps]) + +def _flatten_jars(nested_jars_depset): + """Returns a list of strings containing the compile_jars for depset of targets. + + This ends up unwinding the nesting of depsets, since compile_jars contains depsets inside + the nested_jars targets, which themselves are depsets. This function is intended to be called + lazily form within Args.add_all(map_each) as it collapses depsets. + """ + compile_jars_depsets = [ + target[JavaInfo].compile_jars + for target in nested_jars_depset.to_list() + if target[JavaInfo].compile_jars + ] + return [file.path for file in depset(transitive = compile_jars_depsets).to_list()] diff --git a/build/jvm-rules/rules/impl/compile.bzl b/build/jvm-rules/rules/impl/compile.bzl index 81dd8f4fee98..56465cbcd80b 100644 --- a/build/jvm-rules/rules/impl/compile.bzl +++ b/build/jvm-rules/rules/impl/compile.bzl @@ -1,3 +1,5 @@ +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") + # Copyright 2018 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,11 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -load( - "@rules_java//java:defs.bzl", - "JavaInfo", - "java_common", -) +load("@rules_java//java:defs.bzl", "JavaInfo", "java_common") load( "@rules_kotlin//kotlin/internal:defs.bzl", _JAVA_RUNTIME_TOOLCHAIN_TYPE = "JAVA_RUNTIME_TOOLCHAIN_TYPE", @@ -27,11 +25,10 @@ load( ) load("@rules_kotlin//kotlin/internal:opts.bzl", "JavacOptions") load("//:rules/common-attrs.bzl", "add_dicts") -load("//:rules/impl/associates.bzl", "flatten_jars", "get_associates") +load("//:rules/impl/associates.bzl", "get_associates") load("//:rules/impl/builder-args.bzl", "init_builder_args") -load("//:rules/impl/kotlinc-options.bzl", "KotlincOptions", "kotlinc_options_to_flags") +load("//:rules/impl/kotlinc-options.bzl", "KotlincOptions") -# UTILITY ############################################################################################################## def find_java_toolchain(ctx, target): if _JAVA_TOOLCHAIN_TYPE in ctx.toolchains: return ctx.toolchains[_JAVA_TOOLCHAIN_TYPE].java @@ -45,12 +42,6 @@ def find_java_runtime_toolchain(ctx, target): def _java_info(target): return target[JavaInfo] if JavaInfo in target else None -def _deps_artifacts(all_targets): - """Collect Jdeps artifacts if required.""" - - # collect `.jdeps` outputs from targets that contain JavaInfo and have a `jdeps` output - return depset([t[JavaInfo].outputs.jdeps for t in all_targets if JavaInfo in t and t[JavaInfo].outputs.jdeps]) - def _partitioned_srcs(srcs): """Creates a struct of srcs sorted by extension. Fails if there are no sources.""" kt_srcs = [] @@ -134,12 +125,6 @@ def _collect_plugins_for_export(local, exports): ], ) -def _format_compile_plugin_options(o): - """Format compiler option into id:value for cmd line.""" - return [ - "%s:%s" % (o.id, o.value), - ] - def _new_plugins_from(targets): """Returns a struct containing the plugin metadata for the given targets. @@ -214,7 +199,6 @@ def _new_plugin_from(plugin_id_to_configuration, plugins_for_phase): options = options, ) -# INTERNAL ACTIONS ##################################################################################################### def _fold_jars_action(ctx, rule_kind, toolchains, output_jar, input_jars, action_type = ""): """Set up an action to Fold the input jars into a normalized output jar.""" args = ctx.actions.args() @@ -271,8 +255,6 @@ def _run_merge_jdeps_action(ctx, report_unused_deps, jdeps, output, deps): progress_message = progress_message, ) -empty_depset = depset() - def _run_jvm_builder_action( mnemonic, executable, @@ -280,62 +262,22 @@ def _run_jvm_builder_action( rule_kind, toolchains, srcs, - generated_src_jars, - associates, + transitiveInputs, compile_deps, - annotation_processors, - transitive_runtime_jars, - plugins, args, outputs): """Creates a KotlinBuilder action invocation.""" - kotlinc_options = ctx.attr.kotlinc_opts[KotlincOptions] - - for f, path in outputs.items(): - args.add("--" + f, path) - - kotlinc_options_to_flags(kotlinc_options, args) - - args.add_all("--opt_in", kotlinc_options.opt_in) - - args.add_all("--classpath", compile_deps.compile_jars) - if ctx.attr._reduced_classpath: args.add("--reduced-classpath-mode", "true") args.add_all("--direct-dependencies", _java_infos_to_compile_jars(compile_deps.deps)) - deps_artifacts = _deps_artifacts(ctx.attr.deps + associates.targets) if kotlinc_options.report_unused_deps else empty_depset - if kotlinc_options.report_unused_deps: - args.add_all("--deps_artifacts", deps_artifacts) - - if associates: - args.add_all("--friend_paths", associates.jars, map_each = flatten_jars) - if ctx.coverage_instrumented(): - args.add("--instrument_coverage") - - all_classpath = depset() - for id, classpath in plugins.compile_phase.classpath.items(): - args.add("--plugin-id", id) - args.add_joined("--plugin-classpath", classpath, omit_if_empty = False, join_with = ":") - all_classpath = depset(transitive = [classpath, all_classpath]) - - # args.add_all("--compiler_plugin_options", plugins.compile_phase.options, map_each = _format_compile_plugin_options) - progress_message = "Compile %%{label} { kt: %d, java: %d }" % (len(srcs.kt), len(srcs.java)) ctx.actions.run( mnemonic = mnemonic, - inputs = depset( - srcs.all_srcs + generated_src_jars, - transitive = [ - compile_deps.compile_jars, - transitive_runtime_jars, - deps_artifacts, - all_classpath, - ], - ), + inputs = depset(srcs.all_srcs, transitive = transitiveInputs), use_default_shell_env = True, - outputs = [f for f in outputs.values()], + outputs = outputs, executable = ctx.attr._kotlin_builder.files_to_run.executable, execution_requirements = { "supports-workers": "1", @@ -349,8 +291,6 @@ def _run_jvm_builder_action( }, ) -# MAIN ACTIONS ######################################################################################################### - def kt_jvm_produce_jar_actions(ctx, rule_kind): """This macro sets up a compile action for a Kotlin jar. @@ -372,15 +312,22 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): ) perTargetPlugins = ctx.attr.plugins if hasattr(ctx.attr, "plugins") else [] - transitive_runtime_jars = _targets_to_transitive_runtime_jars(perTargetPlugins + ctx.attr.deps) plugins = _new_plugins_from(perTargetPlugins + _exported_plugins(deps = ctx.attr.deps)) # merge outputs into final runtime jar output_jar = ctx.actions.declare_file(ctx.label.name + ".jar") outputs_struct = None - kotlinc_options = ctx.attr.kotlinc_opts[KotlincOptions] - if kotlinc_options.jps_threshold != -1 and len(srcs.kt) >= kotlinc_options.jps_threshold: + jps_threshold = ctx.attr._jps_threshold[BuildSettingInfo].value + if jps_threshold == -1: + kotlinc_options = ctx.attr.kotlinc_opts[KotlincOptions] + jps_threshold = kotlinc_options.jps_threshold + + transitiveInputs = [compile_deps.compile_jars] + _collect_runtime_jars(perTargetPlugins, transitiveInputs) + _collect_runtime_jars(ctx.attr.deps, transitiveInputs) + + if jps_threshold != -1 and len(srcs.kt) >= jps_threshold: outputs_struct = _run_jps_builder( ctx = ctx, output_jar = output_jar, @@ -390,7 +337,7 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): generated_ksp_src_jars = [], associates = associates, compile_deps = compile_deps, - transitive_runtime_jars = transitive_runtime_jars, + transitiveInputs = transitiveInputs, plugins = plugins, ) else: @@ -404,7 +351,7 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): associates = associates, compile_deps = compile_deps, annotation_processors = [], - transitive_runtime_jars = transitive_runtime_jars, + transitiveInputs = transitiveInputs, plugins = plugins, ) @@ -420,13 +367,6 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): java_toolchain = toolchains.java, ) - generated_source_jar = java_common.pack_sources( - ctx.actions, - output_source_jar = ctx.actions.declare_file(ctx.label.name + "-gensrc.jar"), - source_jars = generated_src_jars, - java_toolchain = toolchains.java, - ) if generated_src_jars else None - java_info = JavaInfo( output_jar = output_jar, compile_jar = compile_jar, @@ -436,19 +376,10 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): runtime_deps = [_java_info(d) for d in ctx.attr.runtime_deps], exports = [_java_info(d) for d in getattr(ctx.attr, "exports", [])], neverlink = getattr(ctx.attr, "neverlink", False), - generated_source_jar = generated_source_jar, - ) - - instrumented_files = coverage_common.instrumented_files_info( - ctx, - source_attributes = ["srcs"], - dependency_attributes = ["deps", "exports", "associates"], - extensions = ["kt", "java"], ) return struct( java = java_info, - instrumented_files = instrumented_files, kt = _KtJvmInfo( srcs = ctx.files.srcs, module_name = associates.module_name, @@ -536,7 +467,7 @@ def _run_kt_java_builder_actions( associates, compile_deps, annotation_processors, - transitive_runtime_jars, + transitiveInputs, plugins): """Runs the necessary KotlinBuilder and JavaBuilder actions to compile a jar @@ -546,7 +477,7 @@ def _run_kt_java_builder_actions( has_kt_sources = srcs.kt or srcs.src_jars java_infos = [] - outputs = None + outputs = [] kt_compile_jar = None kotlinc_options = ctx.attr.kotlinc_opts[KotlincOptions] @@ -559,38 +490,36 @@ def _run_kt_java_builder_actions( # build Kotlin if has_kt_sources: + args = init_builder_args(ctx, rule_kind, associates, transitiveInputs, plugins = plugins, compile_jars = compile_deps.compile_jars) + kt_output_jar = ctx.actions.declare_file(ctx.label.name + "-kt.jar") if has_java_sources else output_jar + outputs.append(kt_output_jar) + if not "kt_abi_plugin_incompatible" in ctx.attr.tags: kt_compile_jar = ctx.actions.declare_file(ctx.label.name + ("-kt.abi.jar" if has_java_sources else ".abi.jar")) - outputs = { - "output": kt_output_jar, - "abi_jar": kt_compile_jar, - } + outputs.append(kt_compile_jar) + args.add("--abi-out", kt_compile_jar) else: kt_compile_jar = kt_output_jar - outputs = { - "output": kt_output_jar, - } + + args.add("--out", kt_output_jar) kt_jdeps = None if emit_jdeps: kt_jdeps = ctx.actions.declare_file(ctx.label.name + "-kt.jdeps") - outputs["kotlin_output_jdeps"] = kt_jdeps + args.add("--jdeps-out", kt_jdeps) + outputs.append(kt_jdeps) _run_jvm_builder_action( mnemonic = "KotlinCompile", executable = ctx.attr._kotlin_builder.files_to_run.executable, - args = init_builder_args(ctx, rule_kind, associates.module_name), + args = args, ctx = ctx, rule_kind = rule_kind, toolchains = toolchains, srcs = srcs, - generated_src_jars = generated_ksp_src_jars, - associates = associates, + transitiveInputs = transitiveInputs, compile_deps = compile_deps, - annotation_processors = [], - transitive_runtime_jars = transitive_runtime_jars, - plugins = plugins, outputs = outputs, ) @@ -662,13 +591,13 @@ def _run_kt_java_builder_actions( ctx.actions.symlink(output = output_jar, target_file = toolchains.kt.empty_jar) annotation_processing = None - if annotation_processors: - outputs_list = [java_info.outputs for java_info in java_infos] - annotation_processing = _create_annotation_processing( - annotation_processors = annotation_processors, - ap_class_jar = [jars.class_jar for outputs in outputs_list for jars in outputs.jars][0], - ap_source_jar = ap_generated_src_jar, - ) + # if annotation_processors: + # outputs_list = [java_info.outputs for java_info in java_infos] + # annotation_processing = _create_annotation_processing( + # annotation_processors = annotation_processors, + # ap_class_jar = [jars.class_jar for outputs in outputs_list for jars in outputs.jars][0], + # ap_source_jar = ap_generated_src_jar, + # ) return struct( compile_jar = compile_jar, @@ -686,43 +615,36 @@ def _run_jps_builder( generated_ksp_src_jars, associates, compile_deps, - transitive_runtime_jars, + transitiveInputs, plugins): """Runs the necessary JpsBuilder actions to compile a jar Returns: A struct containing the a list of output_jars and a struct annotation_processing jars """ - outputs = None - kt_compile_jar = None - kotlinc_options = ctx.attr.kotlinc_opts[KotlincOptions] + args = init_builder_args(ctx, rule_kind, associates, transitiveInputs, plugins = plugins, compile_jars = compile_deps.compile_jars) + args.add("--out", output_jar) + + outputs = [] + abi_jar = output_jar + if not "kt_abi_plugin_incompatible" in ctx.attr.tags: + abi_jar = ctx.actions.declare_file(ctx.label.name + ".abi.jar") + outputs = [output_jar, abi_jar] + args.add("--abi-out", abi_jar) + else: + outputs = [output_jar] # todo JDEPS for JVM - emit_jdeps = False # emit_jdeps = kotlinc_options.emit_jdeps - - kt_output_jar = output_jar - - # todo ABI for JPS - if False: - kt_compile_jar = ctx.actions.declare_file(ctx.label.name + ".abi.jar") - outputs = { - "output": kt_output_jar, - "abi_jar": kt_compile_jar, - } - else: - kt_compile_jar = kt_output_jar - outputs = { - "output": kt_output_jar, - } - - kt_jdeps = None + #kotlinc_options = ctx.attr.kotlinc_opts[KotlincOptions] + jdeps = None + emit_jdeps = False if emit_jdeps: - kt_jdeps = ctx.actions.declare_file(ctx.label.name + "-kt.jdeps") - outputs["kotlin_output_jdeps"] = kt_jdeps + jdeps = ctx.actions.declare_file(ctx.label.name + ".jdeps") + outputs.append(jdeps) + args.add("--jdeps-out", jdeps) - args = init_builder_args(ctx, rule_kind, associates.module_name) javac_opts = ctx.attr.javac_opts[JavacOptions] if ctx.attr.javac_opts else None if javac_opts and javac_opts.add_exports: args.add_all("--add-export", javac_opts.add_exports) @@ -735,53 +657,19 @@ def _run_jps_builder( rule_kind = rule_kind, toolchains = toolchains, srcs = srcs, - generated_src_jars = generated_ksp_src_jars, - associates = associates, + transitiveInputs = transitiveInputs, compile_deps = compile_deps, - annotation_processors = [], - transitive_runtime_jars = transitive_runtime_jars, - plugins = plugins, outputs = outputs, ) - java_infos = [ - JavaInfo( - output_jar = kt_output_jar, - compile_jar = kt_compile_jar, - jdeps = kt_jdeps, - deps = compile_deps.deps, - runtime_deps = [d[JavaInfo] for d in ctx.attr.runtime_deps], - exports = [d[JavaInfo] for d in getattr(ctx.attr, "exports", [])], - neverlink = getattr(ctx.attr, "neverlink", False), - ), - ] - return struct( - compile_jar = kt_compile_jar, + compile_jar = abi_jar, generated_src_jars = generated_ksp_src_jars, annotation_processing = None, - output_jdeps = _get_or_create_single_jdeps_output(kotlinc_options.report_unused_deps, java_infos, ctx, compile_deps) if emit_jdeps else None, + output_jdeps = jdeps, ) -def _create_annotation_processing(annotation_processors, ap_class_jar, ap_source_jar): - """Creates the annotation_processing field for Kt to match what JavaInfo - - The Bazel Plugin IDE logic is based on this assumption in order to locate the Annotation - Processor generated source code. - - See https://docs.bazel.build/versions/master/skylark/lib/JavaInfo.html#annotation_processing - """ - if annotation_processors: - return struct( - enabled = True, - class_jar = ap_class_jar, - source_jar = ap_source_jar, - ) - return None - -def _targets_to_transitive_runtime_jars(targets): - transitive = [] +def _collect_runtime_jars(targets, transitive): for t in targets: if JavaInfo in t: - transitive.append(t[JavaInfo].plugins.processor_jars) - return depset(transitive = transitive) + transitive.append(t[JavaInfo].plugins.processor_jars) \ No newline at end of file diff --git a/build/jvm-rules/rules/library.bzl b/build/jvm-rules/rules/library.bzl index 429746def80a..f34f8c55433f 100644 --- a/build/jvm-rules/rules/library.bzl +++ b/build/jvm-rules/rules/library.bzl @@ -13,7 +13,6 @@ def _make_providers(ctx, providers): return [ providers.java, providers.kt, - providers.instrumented_files, DefaultInfo( files = depset(files), runfiles = ctx.runfiles( diff --git a/build/jvm-rules/rules/test.bzl b/build/jvm-rules/rules/test.bzl index 586a59d374af..cec0b993656a 100644 --- a/build/jvm-rules/rules/test.bzl +++ b/build/jvm-rules/rules/test.bzl @@ -119,7 +119,6 @@ def _jvm_test(ctx): return [ providers.java, providers.kt, - providers.instrumented_files, DefaultInfo( files = depset(files), runfiles = ctx.runfiles( diff --git a/build/jvm-rules/rules/trace.bzl b/build/jvm-rules/rules/trace.bzl deleted file mode 100644 index 4bfdc9cb6c2c..000000000000 --- a/build/jvm-rules/rules/trace.bzl +++ /dev/null @@ -1,10 +0,0 @@ -TraceInfo = provider(doc = "", fields = ["trace"]) - -def _impl(ctx): - trace = ctx.build_setting_value - return TraceInfo(trace = trace) - -trace = rule( - implementation = _impl, - build_setting = config.bool(flag = True), -) diff --git a/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt b/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt index 3b6f28d64ebc..67b140048679 100644 --- a/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt +++ b/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt @@ -7,7 +7,7 @@ import java.util.regex.Pattern private val FLAG_FILE_RE: Regex = Pattern.compile("""^--flagfile=((.*)-(\d+).params)$""").toRegex() -fun parseArgs(args: List): ArgMap { +fun parseArgs(args: Array): ArgMap { check(args.isNotEmpty()) { "expected at least a single arg got: ${args.joinToString(" ")}" } @@ -15,7 +15,7 @@ fun parseArgs(args: List): ArgMap { return createArgMap( args = FLAG_FILE_RE.matchEntire(args[0])?.groups?.get(1)?.let { Files.readAllLines(Path.of(it.value)) - } ?: args, + } ?: args.asList(), enumClass = JvmBuilderFlags::class.java, ) } @@ -29,7 +29,9 @@ enum class JvmBuilderFlags { PLUGIN_ID, PLUGIN_CLASSPATH, - OUTPUT, + OUT, + ABI_OUT, + RULE_KIND, KOTLIN_MODULE_NAME, @@ -46,10 +48,9 @@ enum class JvmBuilderFlags { WARN, - FRIEND_PATHS, - KOTLIN_OUTPUT_JDEPS, + FRIENDS, + JDEPS_OUT, TRACE, - ABI_JAR, STRICT_KOTLIN_DEPS, REDUCED_CLASSPATH_MODE, @@ -68,8 +69,8 @@ fun configureCommonCompilerArgs(kotlinArgs: K2JVMCompilerArguments, args: ArgMap // kotlin bug - not compatible with a new X compiler-plugin syntax //compilationArgs.disableDefaultScriptingPlugin = true - args.optional(JvmBuilderFlags.FRIEND_PATHS)?.let { value -> - kotlinArgs.friendPaths = value.map { workingDir.resolve(it).toString() }.toTypedArray() + args.optional(JvmBuilderFlags.FRIENDS)?.let { value -> + kotlinArgs.friendPaths = value.map { workingDir.resolve(it).normalize().toString() }.toTypedArray() } args.optional(JvmBuilderFlags.OPT_IN)?.let { diff --git a/build/jvm-rules/src/jps-builder/BUILD.bazel b/build/jvm-rules/src/jps-builder/BUILD.bazel index 67dc61c03832..3cd1734e3a61 100644 --- a/build/jvm-rules/src/jps-builder/BUILD.bazel +++ b/build/jvm-rules/src/jps-builder/BUILD.bazel @@ -4,24 +4,24 @@ load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") kt_jvm_library( name = "worker-lib", - srcs = glob(["**/*.kt"]), + srcs = glob(["*.kt", "impl/*.kt", "state/*.kt"]), kotlinc_opts = "//:rules_jvm_bootstrap_kotlinc_options", deps = [ "@lib//:kotlin-stdlib", - ":jps-standalone", "@lib//:fastutil-min", "//src/compiler-util", "//src/worker-framework", - "//zip:build-zip", "@rules_java//java/runfiles", + ":jps-standalone", "//:kotlin-jps-plugin", + "//src/jps-builder/packager", + "@lib//:hash4j", + "@lib//:mvstore", ], runtime_deps = [ # jps needs these libs "@lib//:asm", "@lib//:aalto-xml", - "@lib//:mvstore", - "@lib//:hash4j", "@lib//:caffeine", "@lib//:platform-jps-build-qdox-java-parser", "@lib//:jps-javac-extension", @@ -37,23 +37,31 @@ kt_jvm_library( java_import( name = "jps-standalone", - jars = ["jps-build-standalone-2.jar"], + jars = ["jps-build-standalone.jar"], + visibility = ["//src/jps-builder:__subpackages__"], ) java_binary( name = "worker-jvm", runtime_deps = [":worker-lib"], + data = [ + "@kotlinc//:kotlinc_dist", + "@kotlin-serialization-compiler-plugin//file", + ], main_class = "org.jetbrains.bazel.jvm.jps.JpsBuildWorker", jvm_flags = [ - "-Xms1000m", - "-Xmx6000m", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "-Xms1024m", + "-Xmx6144m", "-Djava.awt.headless=true", "-Dapple.awt.UIElement=true", "-Dkotlin.environment.keepalive=true", "-Djps.use.experimental.storage=true", - # Kotlin Compiler sets it - "-Didea.io.use.nio2=true", - "-Dzip.handler.uses.crc.instead.of.timestamp=true", + "-Djps.kotlin.home=$(rlocationpath @kotlinc//:kotlinc_dist)", + "-Dorg.jetbrains.kotlin.kotlin-serialization-compiler-plugin.path=$(rlocationpath @kotlin-serialization-compiler-plugin//file)", ], visibility = ["//visibility:public"], ) @@ -67,9 +75,12 @@ java_binary( ], main_class = "org.jetbrains.bazel.jvm.jps.TestJpsBuildWorker", jvm_flags = [ - "-Xms1000m", - "-Xmx6000m", - "-Djps.service.manager.impl=org.jetbrains.bazel.jvm.jps.BazelJpsServiceManager", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "-Xms1024m", + "-Xmx6144m", "-Djava.awt.headless=true", "-Dapple.awt.UIElement=true", "-Dkotlin.environment.keepalive=true", diff --git a/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt b/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt index a642b13d71a7..b58e3783b70a 100644 --- a/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt +++ b/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt @@ -1,25 +1,33 @@ -@file:Suppress("UnstableApiUsage") +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("UnstableApiUsage", "ReplaceJavaStaticMethodWithKotlinAnalog", "ReplaceGetOrSet") package org.jetbrains.bazel.jvm.jps +import com.dynatrace.hash4j.hashing.HashFunnel +import com.dynatrace.hash4j.hashing.HashStream64 +import com.dynatrace.hash4j.hashing.Hashing import com.google.devtools.build.runfiles.Runfiles +import org.jetbrains.annotations.Unmodifiable +import org.jetbrains.bazel.jvm.jps.state.TargetConfigurationDigestContainer +import org.jetbrains.bazel.jvm.jps.state.TargetConfigurationDigestProperty import org.jetbrains.bazel.jvm.kotlin.ArgMap import org.jetbrains.bazel.jvm.kotlin.JvmBuilderFlags import org.jetbrains.bazel.jvm.kotlin.configureCommonCompilerArgs -import org.jetbrains.jps.model.JpsDummyElement -import org.jetbrains.jps.model.JpsElementFactory -import org.jetbrains.jps.model.JpsModel -import org.jetbrains.jps.model.JpsProject +import org.jetbrains.jps.model.* +import org.jetbrains.jps.model.ex.JpsNamedCompositeElementBase +import org.jetbrains.jps.model.impl.JpsElementCollectionImpl import org.jetbrains.jps.model.java.* import org.jetbrains.jps.model.java.compiler.JpsJavaCompilerOptions -import org.jetbrains.jps.model.library.JpsOrderRootType -import org.jetbrains.jps.model.library.impl.JpsLibraryImpl +import org.jetbrains.jps.model.library.* +import org.jetbrains.jps.model.library.JpsLibraryRoot.InclusionOptions +import org.jetbrains.jps.model.library.impl.JpsLibraryReferenceImpl import org.jetbrains.jps.model.module.JpsDependenciesList import org.jetbrains.jps.model.module.impl.JpsModuleImpl import org.jetbrains.jps.model.module.impl.JpsModuleSourceRootImpl import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments import org.jetbrains.kotlin.config.KotlinFacetSettings import org.jetbrains.kotlin.jps.model.JpsKotlinFacetModuleExtension +import java.io.File import java.nio.file.Files import java.nio.file.Path import java.util.* @@ -31,21 +39,23 @@ internal val runFiles by lazy { Runfiles.preload().unmapped() } -private fun createClasspath(args: ArgMap, baseDir: Path): Sequence { - // REDUCED_CLASSPATH_MODE is not supported for JPS - return args.mandatory(JvmBuilderFlags.CLASSPATH).asSequence().map { baseDir.resolve(it).normalize() } -} - private val javaHome = Path.of(System.getProperty("java.home")).normalize() ?: error("No java.home system property") internal fun loadJpsModel( sources: List, args: ArgMap, classPathRootDir: Path, - classOutDir: Path -): JpsModel { + classOutDir: Path, + dependencyFileToDigest: Map, +): Pair { val model = jpsElementFactory.createModel() + val digests = TargetConfigurationDigestContainer() + + val configHash = Hashing.xxh3_64().hashStream() + // version + configHash.putInt(1) + // properties not needed for us (not implemented for java) // extension.loadModuleOptions not needed for us (not implemented for java) val module = JpsModuleImpl( @@ -57,7 +67,10 @@ internal fun loadJpsModel( jpsJavaModuleExtension.outputUrl = classOutDir.toUri().toString() val languageLevelEnumName = "JDK_" + args.mandatorySingle(JvmBuilderFlags.JVM_TARGET).let { if (it == "8") "1_8" else it } - jpsJavaModuleExtension.languageLevel = LanguageLevel.valueOf(languageLevelEnumName) + val langLevel = LanguageLevel.valueOf(languageLevelEnumName) + jpsJavaModuleExtension.languageLevel = langLevel + + configHash.putInt(langLevel.ordinal) for (source in sources) { // used as a key - immutable instance cannot be used @@ -65,20 +78,30 @@ internal fun loadJpsModel( module.addSourceRoot(JpsModuleSourceRootImpl(source.toUri().toString(), JavaSourceRootType.SOURCE, properties)) } - configureKotlinCompiler(module = module, args = args, classPathRootDir = classPathRootDir) + configureKotlinCompiler(module = module, args = args, classPathRootDir = classPathRootDir, configHash = configHash) val dependencyList = module.dependenciesList dependencyList.clear() configureJdk(model = model, module = module, dependencyList = dependencyList) dependencyList.addModuleSourceDependency() - configureClasspath(module = module, dependencyList = dependencyList, args = args, baseDir = classPathRootDir) + configureClasspath( + module = module, + dependencyList = dependencyList, + args = args, + baseDir = classPathRootDir, + dependencyFileToDigest = dependencyFileToDigest, + digests = digests, + ) val project = model.project project.addModule(module) + configHash.putUnorderedIterable(args.optionalList(JvmBuilderFlags.ADD_EXPORT), HashFunnel.forString(), Hashing.xxh3_64()) configureJavac(project, args) - return model + digests.set(TargetConfigurationDigestProperty.COMPILER, configHash.asLong) + + return model to digests } private fun configureJavac(project: JpsProject, args: ArgMap) { @@ -94,14 +117,28 @@ private fun configureJavac(project: JpsProject, args: ArgMap) { configuration.setCompilerOptions("Javac", compilerOptions) } -private fun configureKotlinCompiler(module: JpsModuleImpl, args: ArgMap, classPathRootDir: Path) { +private fun configureKotlinCompiler( + module: JpsModuleImpl, + args: ArgMap, + classPathRootDir: Path, + configHash: HashStream64, +) { val kotlinFacetSettings = KotlinFacetSettings() kotlinFacetSettings.useProjectSettings = false val kotlinArgs = K2JVMCompilerArguments() kotlinFacetSettings.compilerArguments = kotlinArgs configureCommonCompilerArgs(kotlinArgs = kotlinArgs, args = args, workingDir = classPathRootDir) + configHash.putString(kotlinArgs.apiVersion ?: "") + configHash.putString(kotlinArgs.languageVersion ?: "") + configHash.putString(kotlinArgs.jvmTarget ?: "") + configHash.putString(kotlinArgs.lambdas ?: "") + configHash.putString(kotlinArgs.jvmDefault) + configHash.putBoolean(kotlinArgs.inlineClasses) + configHash.putBoolean(kotlinArgs.allowKotlinPackage) + val plugins = args.optionalList(JvmBuilderFlags.PLUGIN_ID).zip(args.optionalList(JvmBuilderFlags.PLUGIN_CLASSPATH)) + configHash.putInt(plugins.size) if (plugins.isNotEmpty()) { val pluginClassPaths = mutableListOf() @Suppress("UnusedVariable") @@ -112,6 +149,7 @@ private fun configureKotlinCompiler(module: JpsModuleImpl, args throw IllegalArgumentException("Missing system property $propertyName") } + configHash.putString(id) pluginClassPaths.add(runFiles.rlocation(relativePath)) } kotlinArgs.pluginClasspaths = pluginClassPaths.toTypedArray() @@ -143,17 +181,34 @@ private fun configureClasspath( dependencyList: JpsDependenciesList, args: ArgMap, baseDir: Path, + dependencyFileToDigest: Map, + digests: TargetConfigurationDigestContainer, ) { - val lib = JpsLibraryImpl( - "class-path-lib", - JpsJavaLibraryType.INSTANCE, - jpsElementFactory.createDummyElement() - ) - for (file in createClasspath(args, baseDir)) { - lib.addRoot(file.toUri().toString(), JpsOrderRootType.COMPILED) + // REDUCED_CLASSPATH_MODE is not supported for JPS + val classPathRaw = args.mandatory(JvmBuilderFlags.CLASSPATH) + val files = Array(classPathRaw.size) { + baseDir.resolve(classPathRaw[it]).normalize() } + val lib = BazelJpsLibrary("class-path-lib", files.asList()) module.addModuleLibrary(lib) dependencyList.addLibraryDependency(lib) + + val hash = Hashing.xxh3_64().hashStream() + hash.putOrderedIterable(classPathRaw, HashFunnel.forString()) + digests.set(TargetConfigurationDigestProperty.DEPENDENCY_PATH_LIST, hash.asLong) + hash.reset() + + // todo JPS should support dependency as JARs, but for now we do include digest into hash + for (file in files) { + val digest = requireNotNull(dependencyFileToDigest.get(file)) { + "Missing digest for $file.\nAvailable digests: ${dependencyFileToDigest.keys.joinToString(separator = ",\n") { it.invariantSeparatorsPathString }}" + } + hash.putBytes(digest) + } + hash.putInt(files.size) + + digests.set(TargetConfigurationDigestProperty.DEPENDENCY_DIGEST_LIST, hash.asLong) + hash.reset() } @Suppress("SameParameterValue") @@ -165,3 +220,79 @@ private fun readModulesFromJdkReleaseFile(javaHome: Path): Sequence { val modules = p.getProperty("MODULES") ?: return emptySequence() return modules.removeSurrounding("\"").removeSurrounding("'").splitToSequence(' ').map { jbrBaseUrl + it } } + +private class BazelJpsLibrary( + name: String, + private val files: List, +) : JpsNamedCompositeElementBase(name), JpsTypedLibrary { + private val properties = JpsElementFactory.getInstance().createDummyElement() + //private val roots = files.map { + // JpsLibraryRootImpl("jar://" + it.invariantSeparatorsPathString + "!/", JpsOrderRootType.COMPILED, InclusionOptions.ROOT_ITSELF) + //} + + override fun getType(): JpsJavaLibraryType = JpsJavaLibraryType.INSTANCE + + override fun asTyped(type: JpsLibraryType): JpsTypedLibrary? { + @Suppress("UNCHECKED_CAST") + return if (getType() == type) this as JpsTypedLibrary else null + } + + override fun getProperties(): JpsDummyElement = properties + + override fun getRoots(rootType: JpsOrderRootType): List { + throw UnsupportedOperationException() + } + + override fun addRoot(url: String, rootType: JpsOrderRootType) { + throw UnsupportedOperationException() + } + + override fun addRoot(file: File, rootType: JpsOrderRootType) { + throw UnsupportedOperationException() + } + + override fun addRoot(url: String, rootType: JpsOrderRootType, options: InclusionOptions) { + throw UnsupportedOperationException() + } + + override fun removeUrl(url: String, rootType: JpsOrderRootType) { + throw UnsupportedOperationException() + } + + override fun delete() { + throw UnsupportedOperationException() + } + + override fun getParent(): JpsElementCollectionImpl? { + @Suppress("UNCHECKED_CAST") + return myParent as JpsElementCollectionImpl? + } + + @Suppress("OVERRIDE_DEPRECATION", "removal") + override fun createCopy(): BazelJpsLibrary { + throw UnsupportedOperationException() + } + + override fun createReference(): JpsLibraryReference { + return JpsLibraryReferenceImpl(name, createParentReference()) + } + + private fun createParentReference(): JpsElementReference { + @Suppress("UNCHECKED_CAST") + return (parent!!.parent as JpsReferenceableElement).createReference() + } + + override fun getFiles(rootType: JpsOrderRootType): @Unmodifiable List { + return files.map { it.toFile() } + } + + override fun getPaths(rootType: JpsOrderRootType): List = files + + override fun getRootUrls(rootType: JpsOrderRootType): List { + // kotlin uses this API + //return files.map { "jar://" + it.invariantSeparatorsPathString + "!/" } + throw UnsupportedOperationException() + } + + override fun toString(): String = "BazelJpsLibrary(files=$files)" +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt b/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt index 70b7e26463cd..0d29452e0ea1 100644 --- a/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt +++ b/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt @@ -1,3 +1,4 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("UnstableApiUsage", "ReplaceGetOrSet", "ReplaceJavaStaticMethodWithKotlinAnalog") package org.jetbrains.bazel.jvm.jps @@ -15,7 +16,8 @@ import org.jetbrains.jps.incremental.java.JavaBuilder import org.jetbrains.jps.model.* import org.jetbrains.jps.service.JpsServiceManager import org.jetbrains.jps.service.SharedThreadPool -import org.jetbrains.kotlin.jps.build.KotlinBuilderService +import org.jetbrains.kotlin.jps.build.KotlinBuilder +import org.jetbrains.kotlin.jps.incremental.KotlinCompilerReferenceIndexBuilder import java.util.* import java.util.concurrent.ConcurrentHashMap @@ -31,7 +33,7 @@ internal class BazelJpsServiceManager : JpsServiceManager() { init { extensions.put(JavaCompilingTool::class.java, listOf(JavacCompilerTool())) - extensions.put(BuilderService::class.java, listOf(BazelJavaBuilderService, KotlinBuilderService())) + extensions.put(BuilderService::class.java, listOf(BazelJavaBuilderService)) // org.jetbrains.kotlin.jps.build.KotlinResourcesRootProvider and KotlinSourceRootProvider are not needed extensions.put(AdditionalRootsProviderService::class.java, listOf()) extensions.put(ExcludedJavaSourceRootProvider::class.java, listOf()) @@ -120,7 +122,9 @@ private object BazelJavaBuilderService : BuilderService() { return listOf( JavaBuilder(SharedThreadPool.getInstance()), //NotNullInstrumentingBuilder(), - JavaBackwardReferenceIndexBuilder() + JavaBackwardReferenceIndexBuilder(), + KotlinBuilder(), + KotlinCompilerReferenceIndexBuilder(), ) } } \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/BazelSharedThreadPool.kt b/build/jvm-rules/src/jps-builder/BazelSharedThreadPool.kt index b199b8e516fc..9bd9ea65b79e 100644 --- a/build/jvm-rules/src/jps-builder/BazelSharedThreadPool.kt +++ b/build/jvm-rules/src/jps-builder/BazelSharedThreadPool.kt @@ -1,3 +1,4 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("UnstableApiUsage") package org.jetbrains.bazel.jvm.jps @@ -7,16 +8,15 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.asExecutor import org.jetbrains.jps.service.SharedThreadPool import java.util.concurrent.* -import java.util.concurrent.atomic.AtomicBoolean -object BazelSharedThreadPool : SharedThreadPool() { +internal object BazelSharedThreadPool : SharedThreadPool() { private val executor = Dispatchers.Default.asExecutor() // JPS uses MAX_BUILDER_THREADS as maxThreads - doesn't make sense for standalone, use coroutine dispatcher pool override fun createBoundedExecutor(name: String, maxThreads: Int) = this override fun createCustomPriorityQueueBoundedExecutor(name: String, maxThreads: Int, comparator: Comparator): Executor { - return PriorityQueueExecutor(this, comparator) + throw UnsupportedOperationException() } override fun execute(command: Runnable) { @@ -68,40 +68,4 @@ object BazelSharedThreadPool : SharedThreadPool() { override fun invokeAny(tasks: Collection>, timeout: Long, unit: TimeUnit): T? { throw UnsupportedOperationException() } -} - -private class PriorityQueueExecutor( - private val delegateExecutor: Executor, - comparator: Comparator -) : Executor { - private val taskQueue = PriorityBlockingQueue(11, comparator) - private val isProcessing = AtomicBoolean(false) - - override fun execute(command: Runnable) { - taskQueue.put(command) - processQueue() - } - - private fun processQueue() { - if (!isProcessing.compareAndSet(false, true)) { - return - } - - delegateExecutor.execute { - try { - while (true) { - val task = taskQueue.poll() ?: break - task.run() - } - } - finally { - if (taskQueue.isEmpty()) { - isProcessing.set(false) - } - else { - processQueue() - } - } - } - } -} +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/JpsBuilder.kt b/build/jvm-rules/src/jps-builder/JpsBuilder.kt index 14fafc66ac12..576d13c3a4c0 100644 --- a/build/jvm-rules/src/jps-builder/JpsBuilder.kt +++ b/build/jvm-rules/src/jps-builder/JpsBuilder.kt @@ -1,43 +1,42 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("HardCodedStringLiteral", "UnstableApiUsage", "ReplaceJavaStaticMethodWithKotlinAnalog") package org.jetbrains.bazel.jvm.jps -import com.google.devtools.build.lib.worker.WorkerProtocol -import com.intellij.openapi.util.io.FileUtilRt +import com.intellij.openapi.diagnostic.IdeaLogRecordFormatter +import com.intellij.openapi.diagnostic.Logger +import com.intellij.tracing.Tracer +import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import org.jetbrains.bazel.jvm.WorkRequest import org.jetbrains.bazel.jvm.WorkRequestExecutor -import org.jetbrains.bazel.jvm.jps.impl.BazelModuleExcludeIndex +import org.jetbrains.bazel.jvm.jps.impl.* +import org.jetbrains.bazel.jvm.jps.state.TargetConfigurationDigestContainer +import org.jetbrains.bazel.jvm.jps.state.saveTargetState import org.jetbrains.bazel.jvm.kotlin.ArgMap import org.jetbrains.bazel.jvm.kotlin.JvmBuilderFlags import org.jetbrains.bazel.jvm.kotlin.parseArgs import org.jetbrains.bazel.jvm.processRequests +import org.jetbrains.jps.api.CanceledStatus import org.jetbrains.jps.api.GlobalOptions -import org.jetbrains.jps.backwardRefs.JavaBackwardReferenceIndexWriter -import org.jetbrains.jps.builders.impl.BuildDataPathsImpl -import org.jetbrains.jps.builders.impl.BuildRootIndexImpl -import org.jetbrains.jps.builders.impl.BuildTargetIndexImpl -import org.jetbrains.jps.builders.impl.BuildTargetRegistryImpl import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType -import org.jetbrains.jps.builders.logging.BuildLoggingManager -import org.jetbrains.jps.cmdline.ProjectDescriptor -import org.jetbrains.jps.incremental.BuilderRegistry -import org.jetbrains.jps.incremental.CompileScopeImpl -import org.jetbrains.jps.incremental.MessageHandler -import org.jetbrains.jps.incremental.RebuildRequestedException -import org.jetbrains.jps.incremental.fs.BuildFSState -import org.jetbrains.jps.incremental.messages.BuildMessage -import org.jetbrains.jps.incremental.messages.CompilerMessage +import org.jetbrains.jps.incremental.* +import org.jetbrains.jps.incremental.relativizer.PathRelativizer import org.jetbrains.jps.incremental.relativizer.PathRelativizerService -import org.jetbrains.jps.incremental.storage.BuildDataManager -import org.jetbrains.jps.incremental.storage.BuildTargetsState +import org.jetbrains.jps.incremental.storage.ExperimentalSourceToOutputMapping +import org.jetbrains.jps.incremental.storage.PathTypeAwareRelativizer +import org.jetbrains.jps.incremental.storage.RelativePathType import org.jetbrains.jps.incremental.storage.StorageManager -import org.jetbrains.jps.indices.impl.IgnoredFileIndexImpl -import org.jetbrains.jps.model.JpsModel import org.jetbrains.kotlin.config.IncrementalCompilation +import java.io.File import java.io.Writer +import java.nio.file.Files import java.nio.file.Path -import java.util.Map import java.util.Set +import kotlin.coroutines.coroutineContext +import kotlin.io.path.invariantSeparatorsPathString // Please note: for performance reasons, we do not set `jps.new.storage.compact.on.close` to true. // As a result, the database file on disk may grow to some extent. @@ -52,6 +51,9 @@ private fun configureKotlincHome() { } internal fun configureGlobalJps() { + Logger.setFactory { BazelLogger(IdeaLogRecordFormatter.smartAbbreviate(it)) } + System.setProperty("jps.service.manager.impl", BazelJpsServiceManager::class.java.name) + System.setProperty("jps.backward.ref.index.builder.fs.case.sensitive", "true") System.setProperty(GlobalOptions.COMPILE_PARALLEL_MAX_THREADS_OPTION, Runtime.getRuntime().availableProcessors().toString()) System.setProperty(GlobalOptions.COMPILE_PARALLEL_OPTION, "true") System.setProperty(GlobalOptions.DEPENDENCY_GRAPH_ENABLED, "true") @@ -68,195 +70,186 @@ object JpsBuildWorker : WorkRequestExecutor { processRequests(startupArgs, this) } - override suspend fun execute(request: WorkerProtocol.WorkRequest, writer: Writer, baseDir: Path): Int { - val sources = request.inputsList.asSequence() - .filter { it.path.endsWith(".kt") || it.path.endsWith(".java") } - .map { baseDir.resolve(it.path) } - .toList() - return buildUsingJps(workingDir = baseDir, args = parseArgs(request.argumentsList), out = writer, sources = sources) + override suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path): Int { + val dependencyFileToDigest = HashMap() + val sources = ArrayList() + for (input in request.inputs) { + if (input.path.endsWith(".kt") || input.path.endsWith(".java")) { + sources.add(baseDir.resolve(input.path).normalize()) + } + else if (input.path.endsWith(".jar")) { + dependencyFileToDigest.put(baseDir.resolve(input.path).normalize(), input.digest) + } + } + return buildUsingJps( + baseDir = baseDir, + args = parseArgs(request.arguments), + out = writer, + sources = sources, + dependencyFileToDigest = dependencyFileToDigest, + isDebugEnabled = request.verbosity > 0, + ) } } internal suspend fun buildUsingJps( - workingDir: Path, + baseDir: Path, args: ArgMap, out: Writer, sources: List, - classPathRootDir: Path = workingDir, + dependencyFileToDigest: Map, + isDebugEnabled: Boolean, ): Int { - val messageHandler = ConsoleMessageHandler(out) - val outJar = workingDir.resolve(args.mandatorySingle(JvmBuilderFlags.OUTPUT)).normalize() + val abiJar = args.optionalSingle(JvmBuilderFlags.ABI_OUT)?.let { baseDir.resolve(it).normalize() } + val outJar = baseDir.resolve(args.mandatorySingle(JvmBuilderFlags.OUT)).normalize() val prefix = outJar.fileName.toString().removeSuffix(".jar") val bazelOutDir = outJar.parent - val dataStorageRoot = bazelOutDir.resolve("$prefix-jps-data") + val dataDir = bazelOutDir.resolve("$prefix-jps-data") // incremental compilation - we do not clear dir val classOutDir = bazelOutDir.resolve("$prefix-classes") - val buildRunner = BuildRunner() - val jpsModel = loadJpsModel(sources = sources, args = args, classPathRootDir = classPathRootDir, classOutDir = classOutDir) + val (jpsModel, targetDigests) = loadJpsModel( + sources = sources, + args = args, + classPathRootDir = baseDir, + classOutDir = classOutDir, + dependencyFileToDigest = dependencyFileToDigest, + ) + val moduleTarget = BazelModuleBuildTarget( + outDir = classOutDir, + module = jpsModel.project.modules.single(), + sources = sources, + ) - fun createProjectDescriptor(): ProjectDescriptor? { - return buildRunner.load( - messageHandler = messageHandler, - dataStorageRoot = dataStorageRoot, - fsState = BuildFSState(/* alwaysScanFS = */ true), - jpsModel = jpsModel, - ) - } + val relativizer = createPathRelativizer(baseDir = baseDir, classOutDir = classOutDir) - fun clearStorage() { - // todo rename and store - FileUtilRt.deleteRecursively(dataStorageRoot) - FileUtilRt.deleteRecursively(classOutDir) - } + val compileScope = CompileScopeImpl( + /* types = */ Set.of(JavaModuleBuildTargetType.PRODUCTION), + /* typesToForceBuild = */ Set.of(), + /* targets = */ Set.of(), + /* files = */ java.util.Map.of() + ) + + suspend fun initAndBuild(isRebuild: Boolean): Boolean { + val storageInitializer = StorageInitializer(dataDir = dataDir, classOutDir = classOutDir) + val storageManager = if (isRebuild) { + storageInitializer.clearAndInit(messageHandler) + } + else { + storageInitializer.init(messageHandler, targetDigests) + } - try { - var projectDescriptor: ProjectDescriptor? = null try { - projectDescriptor = createProjectDescriptor() - var checkRebuildRequired = true - if (projectDescriptor == null) { - clearStorage() - checkRebuildRequired = false + val projectDescriptor = storageInitializer.createProjectDescriptor(messageHandler, jpsModel, moduleTarget, relativizer) + try { + if (storageInitializer.isCheckRebuildRequired) { + val rebuildRequiredSpan = Tracer.start("JpsProjectBuilder.checkRebuildRequired") + val isRebuildRequired = checkRebuildRequired( + scope = compileScope, + projectDescriptor = projectDescriptor, + moduleTarget = moduleTarget, + isDebugEnabled = isDebugEnabled, + messageHandler = messageHandler, + ) + rebuildRequiredSpan.complete() - projectDescriptor = requireNotNull(createProjectDescriptor()) { - "The storage has been corrupted twice in a row, resulting in an unrecoverable error" + if (isRebuildRequired) { + return false + } } - } - doBuild(checkRebuildRequired = checkRebuildRequired, projectDescriptor = projectDescriptor, messageHandler = messageHandler) + val coroutineContext = coroutineContext + val context = CompileContextImpl( + compileScope, + projectDescriptor, + messageHandler, + emptyMap(), + object : CanceledStatus { + override fun isCanceled(): Boolean = !coroutineContext.isActive + }, + ) + JpsProjectBuilder( + builderRegistry = BuilderRegistry.getInstance(), + messageHandler = messageHandler, + isCleanBuild = storageInitializer.isCleanBuild, + ).build(context) + postBuild( + messageHandler = messageHandler, + moduleTarget = moduleTarget, + outJar = outJar, + abiJar = abiJar, + classOutDir = classOutDir, + context = context, + targetDigests = targetDigests, + storageManager = storageManager, + ) + return true + } + finally { + projectDescriptor.release() + } + } + catch (_: RebuildRequestedException) { + return false } finally { - projectDescriptor?.release() + storageManager.forceClose() } } - catch (_: RebuildRequestedException) { - clearStorage() - val projectDescriptor = requireNotNull(createProjectDescriptor()) { - "Unrecoverable error" - } - - try { - doBuild(checkRebuildRequired = false, projectDescriptor = projectDescriptor, messageHandler = messageHandler) - } - finally { - projectDescriptor.release() - } + // if class output dir doesn't exist, make sure that we do not to use existing cache - pass `isRebuild` as true in this case + if (!initAndBuild(isRebuild = Files.notExists(classOutDir))) { + initAndBuild(isRebuild = true) } return if (messageHandler.hasErrors()) 1 else 0 } -private suspend fun doBuild( - checkRebuildRequired: Boolean, - projectDescriptor: ProjectDescriptor, - messageHandler: MessageHandler, + +private suspend fun postBuild( + messageHandler: ConsoleMessageHandler, + moduleTarget: ModuleBuildTarget, + outJar: Path, + abiJar: Path?, + classOutDir: Path, + context: CompileContextImpl, + targetDigests: TargetConfigurationDigestContainer, + storageManager: StorageManager, ) { - val compileScope = CompileScopeImpl( - /* types = */ Set.of(JavaModuleBuildTargetType.PRODUCTION), - /* typesToForceBuild = */ Set.of(), - /* targets = */ Set.of(), - /* files = */ Map.of() - ) - - val builder = JpsProjectBuilder( - projectDescriptor = projectDescriptor, - builderRegistry = BuilderRegistry.getInstance(), - builderParams = emptyMap(), - isTestMode = false, - messageHandler = messageHandler, - ) - if (checkRebuildRequired) { - builder.checkRebuildRequired(compileScope) - } - builder.build(compileScope) -} - -private class ConsoleMessageHandler(private val out: Writer) : MessageHandler { - private var hasErrors = false - - override fun processMessage(message: BuildMessage) { - val messageText = when (message) { - is CompilerMessage -> { - when { - message.sourcePath == null -> message.messageText - message.line < 0 -> message.sourcePath + ": " + message.messageText - else -> message.sourcePath + "(" + message.line + ":" + message.column + "): " + message.messageText - } - } - - else -> message.messageText - } - - if (messageText.isEmpty()) { - return - } - - if (message.kind == BuildMessage.Kind.ERROR) { - out.appendLine("Error: $messageText") - hasErrors = true - } - else if (message.kind !== BuildMessage.Kind.PROGRESS || !messageText.startsWith("Compiled") && !messageText.startsWith("Copying")) { - out.appendLine(messageText) - } - } - - fun hasErrors(): Boolean = hasErrors -} - -private fun createStorageManager(dataStorageRoot: Path): StorageManager { - val manager = StorageManager(dataStorageRoot.resolve("jps-portable-cache.db")) - manager.open() - return manager -} - -private class BuildRunner() { - fun load(messageHandler: MessageHandler, dataStorageRoot: Path, fsState: BuildFSState, jpsModel: JpsModel): ProjectDescriptor? { - val dataPaths = BuildDataPathsImpl(dataStorageRoot.toFile()) - val targetRegistry = BuildTargetRegistryImpl(jpsModel) - val moduleExcludeIndex = BazelModuleExcludeIndex - val ignoredFileIndex = IgnoredFileIndexImpl(jpsModel) - val buildRootIndex = BuildRootIndexImpl(targetRegistry, jpsModel, moduleExcludeIndex, dataPaths, ignoredFileIndex) - val targetIndex = BuildTargetIndexImpl(targetRegistry, buildRootIndex) - - val relativizer = PathRelativizerService(jpsModel.project, JavaBackwardReferenceIndexWriter.isCompilerReferenceFSCaseSensitive()) - - val storageManager = createStorageManager(dataStorageRoot) - var dataManager: BuildDataManager? = null - try { - @Suppress("DEPRECATION") - dataManager = BuildDataManager( - dataPaths, - BuildTargetsState(dataPaths, jpsModel, buildRootIndex), - relativizer, - storageManager, + coroutineScope { + val dataManager = context.projectDescriptor.dataManager + launch(CoroutineName("save caches")) { + // save config state only in the end + saveTargetState( + targetDigests = targetDigests, + manager = context.projectDescriptor.dataManager.targetStateManager as BazelBuildTargetStateManager, + storageManager = storageManager, ) - if (dataManager.versionDiffers()) { - storageManager.forceClose() - messageHandler.processMessage( - CompilerMessage("", BuildMessage.Kind.INFO, "Dependency data format has changed, project rebuild required"), + dataManager.flush(/* memoryCachesOnly = */ false) + } + + launch(CoroutineName("create output JAR and ABI JAR")) { + // pack to jar + messageHandler.measureTime("pack and abi") { + val sourceToOutputMap = dataManager.getSourceToOutputMap(moduleTarget) as ExperimentalSourceToOutputMapping + packageToJar( + outJar = outJar, + abiJar = abiJar, + sourceToOutputMap = sourceToOutputMap, + classOutDir = classOutDir, + messageHandler = messageHandler, ) - return null } } - catch (e: Exception) { - messageHandler.processMessage( - CompilerMessage("", BuildMessage.Kind.INTERNAL_BUILDER_ERROR, "Cannot open cache storage: ${e.stackTraceToString()}"), - ) - storageManager.forceClose() - dataManager?.close() - - return null + launch(CoroutineName("report build state")) { + dataManager.reportUnhandledRelativizerPaths() + reportRebuiltModules(context) + reportUnprocessedChanges(context, moduleTarget) } - - return ProjectDescriptor( - jpsModel, fsState, dataManager, BuildLoggingManager.DEFAULT, moduleExcludeIndex, targetIndex, buildRootIndex, ignoredFileIndex - ) } } \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/JpsProjectBuilder.kt b/build/jvm-rules/src/jps-builder/JpsProjectBuilder.kt deleted file mode 100644 index cbf23bd79c3c..000000000000 --- a/build/jvm-rules/src/jps-builder/JpsProjectBuilder.kt +++ /dev/null @@ -1,1240 +0,0 @@ -// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. -@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "UnstableApiUsage", "ReplaceGetOrSet", "HardCodedStringLiteral") - -package org.jetbrains.bazel.jvm.jps - -import com.intellij.openapi.diagnostic.Logger -import com.intellij.openapi.util.NlsSafe -import com.intellij.openapi.util.io.FileUtilRt -import com.intellij.openapi.util.text.Formats -import com.intellij.tracing.Tracer -import com.intellij.util.ReflectionUtil -import com.intellij.util.containers.CollectionFactory -import com.intellij.util.containers.FileCollectionFactory -import com.intellij.util.containers.FileHashStrategy -import it.unimi.dsi.fastutil.objects.Object2ObjectMaps -import it.unimi.dsi.fastutil.objects.Object2ObjectOpenCustomHashMap -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch -import org.jetbrains.bazel.jvm.jps.impl.BuildTaskLauncher -import org.jetbrains.bazel.jvm.jps.impl.isBuildChunkAffected -import org.jetbrains.jps.ModuleChunk -import org.jetbrains.jps.api.CanceledStatus -import org.jetbrains.jps.builders.* -import org.jetbrains.jps.builders.impl.BuildOutputConsumerImpl -import org.jetbrains.jps.builders.impl.BuildTargetChunk -import org.jetbrains.jps.builders.impl.DirtyFilesHolderBase -import org.jetbrains.jps.builders.java.JavaBuilderUtil -import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType -import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor -import org.jetbrains.jps.builders.storage.BuildDataCorruptedException -import org.jetbrains.jps.builders.storage.SourceToOutputMapping -import org.jetbrains.jps.cmdline.ProjectDescriptor -import org.jetbrains.jps.incremental.* -import org.jetbrains.jps.incremental.ModuleLevelBuilder.OutputConsumer -import org.jetbrains.jps.incremental.fs.CompilationRound -import org.jetbrains.jps.incremental.messages.* -import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration -import org.jetbrains.jps.incremental.storage.BuildTargetSourcesState -import org.jetbrains.jps.incremental.storage.SourceToOutputMappingImpl -import org.jetbrains.jps.model.serialization.impl.TimingLog -import org.jetbrains.jps.util.JpsPathUtil -import java.io.File -import java.io.IOException -import java.lang.invoke.MethodHandles -import java.lang.reflect.InvocationHandler -import java.lang.reflect.Method -import java.nio.file.FileSystem -import java.nio.file.FileSystems -import java.nio.file.Files -import java.nio.file.Path -import java.util.* -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicLong -import java.util.function.Function -import java.util.function.Predicate -import java.util.function.Supplier -import java.util.stream.Collectors -import kotlin.concurrent.Volatile -import kotlin.math.min - -private val TARGET_WITH_CLEARED_OUTPUT = GlobalContextKey.create>>("_targets_with_cleared_output_") -private val LOG = Logger.getInstance(JpsProjectBuilder::class.java) -private val lookup = MethodHandles.lookup() - -private const val CLASSPATH_INDEX_FILE_NAME = "classpath.index" - -// CLASSPATH_INDEX_FILE_NAME cannot be used because IDEA on run creates CLASSPATH_INDEX_FILE_NAME only if some module class is loaded, -// so, not possible to distinguish case -// "classpath.index doesn't exist because deleted on module file change" vs. "classpath.index doesn't exist because was not created" -private const val UNMODIFIED_MARK_FILE_NAME = ".unmodified" - -internal class JpsProjectBuilder( - private val projectDescriptor: ProjectDescriptor, - private val builderRegistry: BuilderRegistry, - private val builderParams: Map, - private val isTestMode: Boolean, - private val messageHandler: MessageHandler, -) { - - private val totalModuleLevelBuilderCount = builderRegistry.moduleLevelBuilderCount - private val elapsedTimeNanosByBuilder = ConcurrentHashMap() - private val numberOfSourcesProcessedByBuilder = ConcurrentHashMap() - - fun checkRebuildRequired(scope: CompileScope) { - val rebuildRequiredSpan = Tracer.start("IncProjectBuilder.checkRebuildRequired") - doCheckRebuildRequired(scope) - rebuildRequiredSpan.complete() - } - - suspend fun build(scope: CompileScope) { - var context: CompileContextImpl? = null - var sourceState: BuildTargetSourcesState? = null - try { - context = createContext(scope) - sourceState = BuildTargetSourcesState(context) - val buildSpan = Tracer.start("IncProjectBuilder.runBuild") - runBuild(context = context) - buildSpan.complete() - val dataManager = projectDescriptor.dataManager - dataManager.saveVersion() - dataManager.reportUnhandledRelativizerPaths() - sourceState.reportSourcesState() - reportRebuiltModules(context) - reportUnprocessedChanges(context) - } - catch (e: StopBuildException) { - if (context != null) { - reportRebuiltModules(context) - reportUnprocessedChanges(context) - } - // If build was canceled for some reason, e.g., compilation error, we should report built modules - sourceState?.reportSourcesState() - // some builder decided to stop the build - // report optional progress message if any - e.message?.takeIf { it.isNotEmpty() }?.let { - messageHandler.processMessage(ProgressMessage(it)) - } - } - catch (e: BuildDataCorruptedException) { - requestRebuild(e = e, cause = null) - } - catch (e: ProjectBuildException) { - val cause = e.cause - if (cause is IOException || cause is BuildDataCorruptedException || (cause is RuntimeException && cause.cause is IOException)) { - requestRebuild(e, cause) - } - else { - // should stop the build with error - throw e - } - } - finally { - val finishingCompilationSpan = Tracer.start("finishing compilation") - context?.projectDescriptor?.dataManager?.flush(false) - finishingCompilationSpan.complete() - } - } - - private fun doCheckRebuildRequired(scope: CompileScope) { - val isDebugEnabled = LOG.isDebugEnabled - if (isTestMode) { - // do not use the heuristic in tests to properly test all cases - // automatic builds should not cause to start full project rebuilds to avoid situations when user does not expect rebuild - if (isDebugEnabled) { - LOG.debug("Rebuild heuristic: skipping the check; isTestMode = true") - } - return - } - - val targetsState = projectDescriptor.targetsState - val timeThreshold = targetsState.lastSuccessfulRebuildDuration * 95 / 100 // 95% of last registered clean rebuild time - if (timeThreshold <= 0) { - if (isDebugEnabled) { - LOG.debug("Rebuild heuristic: no stats available") - } - return - } - - // check that this is a whole-project incremental build - // checking only JavaModuleBuildTargetType because these target types directly correspond to project modules - for (type in JavaModuleBuildTargetType.ALL_TYPES) { - if (!scope.isBuildIncrementally(type)) { - if (isDebugEnabled) { - LOG.debug("Rebuild heuristic: skipping the check because rebuild is forced for targets of type ${type.typeId}") - } - return - } - if (!scope.isAllTargetsOfTypeAffected(type)) { - if (isDebugEnabled) { - LOG.debug("Rebuild heuristic: skipping the check because some targets are excluded from compilation scope, e.g. targets of type ${type.typeId}") - } - return - } - } - - // compute estimated times for dirty targets - val allTargetsAffected = HashSet(JavaModuleBuildTargetType.ALL_TYPES) - val estimatedWorkTime = calculateEstimatedBuildTime(projectDescriptor) { target -> - // optimization, since we know here that all targets of types JavaModuleBuildTargetType are affected - allTargetsAffected.contains(target.getTargetType()) || scope.isAffected(target) - } - if (isDebugEnabled) { - LOG.debug("Rebuild heuristic: estimated build time / timeThreshold : $estimatedWorkTime / $timeThreshold") - } - - if (estimatedWorkTime >= timeThreshold) { - val message = "Too many modules require recompilation, forcing full project rebuild" - LOG.info(message) - LOG.info("Estimated build duration (linear): " + Formats.formatDuration(estimatedWorkTime)) - LOG.info("Last successful rebuild duration (linear): " + Formats.formatDuration(targetsState.lastSuccessfulRebuildDuration)) - LOG.info("Rebuild heuristic time threshold: " + Formats.formatDuration(timeThreshold)) - messageHandler.processMessage(CompilerMessage("", BuildMessage.Kind.INFO, message)) - throw RebuildRequestedException(null) - } - } - - private fun requestRebuild(e: Exception, cause: Throwable?) { - messageHandler.processMessage(CompilerMessage( - "", BuildMessage.Kind.INFO, "Internal caches are corrupted or have outdated format, forcing project rebuild: $e") - ) - throw RebuildRequestedException(cause ?: e) - } - - private suspend fun runBuild(context: CompileContextImpl) { - context.setDone(0.0f) - - context.addBuildListener(ChainedTargetsBuildListener(context)) - - // deletes class loader classpath index files for changed output roots - context.addBuildListener(object : BuildListener { - override fun filesGenerated(event: FileGeneratedEvent) { - val paths = event.paths - val fs = FileSystems.getDefault() - if (paths.size == 1) { - deleteFiles(paths.iterator().next().first, fs) - return - } - - val outputs = HashSet() - for (pair in paths) { - val root = pair.getFirst() - if (outputs.add(root)) { - deleteFiles(root, fs) - } - } - } - - private fun deleteFiles(rootPath: String, fs: FileSystem) { - val root = fs.getPath(rootPath) - try { - Files.deleteIfExists(root.resolve(CLASSPATH_INDEX_FILE_NAME)) - Files.deleteIfExists(root.resolve(UNMODIFIED_MARK_FILE_NAME)) - } - catch (_: IOException) { - } - } - }) - val allTargetBuilderBuildStartedSpan = Tracer.start("All TargetBuilder.buildStarted") - for (builder in builderRegistry.targetBuilders) { - builder.buildStarted(context) - } - allTargetBuilderBuildStartedSpan.complete() - val allModuleLevelBuildersBuildStartedSpan = Tracer.start("All ModuleLevelBuilder.buildStarted") - for (builder in builderRegistry.moduleLevelBuilders) { - builder.buildStarted(context) - } - allModuleLevelBuildersBuildStartedSpan.complete() - - var buildProgress: BuildProgress? = null - try { - buildProgress = BuildProgress( - projectDescriptor.dataManager, - projectDescriptor.buildTargetIndex, - projectDescriptor.buildTargetIndex.getSortedTargetChunks(context), - Predicate { isBuildChunkAffected(scope = context.scope, chunk = it) } - ) - - // clean roots for targets for which rebuild is forced - val cleanOutputSourcesSpan = Tracer.start("Clean output sources") - cleanOutputRoots(context = context) - cleanOutputSourcesSpan.complete() - - val beforeTasksSpan = Tracer.start("'before' tasks") - for (task in builderRegistry.beforeTasks) { - task.build(context) - } - TimingLog.LOG.debug("'before' tasks finished") - beforeTasksSpan.complete() - - val checkingSourcesSpan = Tracer.start("Building targets") - val buildSpan = Tracer.start("Parallel build") - BuildTaskLauncher(context, buildProgress, this).buildInParallel() - buildSpan.complete() - TimingLog.LOG.debug("Building targets finished") - checkingSourcesSpan.complete() - - val afterTasksSpan = Tracer.start("'after' span") - for (task in builderRegistry.afterTasks) { - task.build(context) - } - TimingLog.LOG.debug("'after' tasks finished") - sendElapsedTimeMessages(context) - afterTasksSpan.complete() - } - finally { - if (buildProgress != null) { - buildProgress.updateExpectedAverageTime() - if (context.isProjectRebuild && !Utils.errorsDetected(context) && !context.cancelStatus.isCanceled) { - projectDescriptor.targetsState.lastSuccessfulRebuildDuration = buildProgress.absoluteBuildTime - } - } - for (builder in builderRegistry.targetBuilders) { - builder.buildFinished(context) - } - for (builder in builderRegistry.moduleLevelBuilders) { - builder.buildFinished(context) - } - } - } - - private fun sendElapsedTimeMessages(context: CompileContext) { - elapsedTimeNanosByBuilder.entries - .stream() - .map { entry -> - val processedSourcesRef = numberOfSourcesProcessedByBuilder.get(entry!!.key) - val processedSources = processedSourcesRef?.get() ?: 0 - BuilderStatisticsMessage(entry.key.presentableName, processedSources, entry.value.get() / 1000000) - } - .sorted(Comparator.comparing(Function { obj: BuilderStatisticsMessage? -> obj!!.builderName })) - .forEach { buildMessage: BuilderStatisticsMessage? -> context.processMessage(buildMessage) } - } - - private fun runBuildersForChunk(context: CompileContext, chunk: BuildTargetChunk, buildProgress: BuildProgress): Boolean { - val targets: Set> = chunk.targets - if (targets.size > 1) { - val moduleTargets = LinkedHashSet() - for (target in targets) { - if (target is ModuleBuildTarget) { - moduleTargets.add(target) - } - else { - val targetsString = targets.joinToString(separator = ", ") { it.presentableName } - val message = "Cannot build \"${target.presentableName}\" because it is included into a circular dependency ($targetsString))" - context.processMessage(CompilerMessage("", BuildMessage.Kind.ERROR, message)) - return false - } - } - - return runModuleLevelBuilders(wrapWithModuleInfoAppender(context, moduleTargets), ModuleChunk(moduleTargets), buildProgress) - } - - val target = targets.iterator().next() - if (target is ModuleBuildTarget) { - @Suppress("RemoveRedundantQualifierName") - val mbt = java.util.Set.of(target) - return runModuleLevelBuilders(wrapWithModuleInfoAppender(context, mbt), ModuleChunk(mbt), buildProgress) - } - - @Suppress("UNCHECKED_CAST") - completeRecompiledSourcesSet(context, targets as Collection>) - - /* - In general, the set of files corresponding to changed source file may be different - Need this, for example, to keep up with case changes in file names for case-insensitive OSes: - deleting the output before copying is the only way to ensure the case of the output file's name is exactly the same as source file's case - */ - cleanOldOutputs(context, target) - - val builders = BuilderRegistry.getInstance().targetBuilders - var builderCount = 0 - for (builder in builders) { - buildTarget(target, context, builder) - builderCount++ - buildProgress.updateProgress(target, (builderCount.toDouble()) / builders.size, context) - } - return true - } - - private fun createContext(scope: CompileScope): CompileContextImpl { - return CompileContextImpl(scope, projectDescriptor, object : MessageHandler { - override fun processMessage(msg: BuildMessage?) { - messageHandler.processMessage(msg) - } - }, builderParams, CanceledStatus.NULL) - } - - private suspend fun cleanOutputRoots(context: CompileContext) { - val cleanStart = System.nanoTime() - try { - coroutineScope { - clearOutputs(context) - } - - for (type in TargetTypeRegistry.getInstance().targetTypes) { - if (context.scope.isAllTargetsOfTypeAffected(type)) { - cleanOutputOfStaleTargets(type, context) - } - } - } - finally { - LOG.info("Cleaned output directories in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - cleanStart) + " ms") - } - } - - private fun cleanOutputOfStaleTargets(targetType: BuildTargetType<*>, context: CompileContext) { - val dataManager = projectDescriptor.dataManager - val targetIds = dataManager.targetsState.getStaleTargetIds(targetType) - if (targetIds.isEmpty()) { - return - } - - context.processMessage(ProgressMessage("Cleaning old output directories\u2026")) - for (ids in targetIds) { - val targetId = ids.first!! - try { - var mapping: SourceToOutputMappingImpl? = null - try { - mapping = dataManager.createSourceToOutputMapForStaleTarget(targetType, targetId) - clearOutputFiles(context = context, mapping = mapping, targetType = targetType, targetId = ids.second) - } - finally { - mapping?.close() - } - dataManager.cleanStaleTarget(targetType, targetId) - } - catch (e: IOException) { - LOG.warn(e) - messageHandler.processMessage(CompilerMessage("", BuildMessage.Kind.WARNING, - "Failed to delete output files from obsolete \"$targetId\" target: $e")) - } - } - } - - @Throws(ProjectBuildException::class) - private fun processDeletedPaths(context: CompileContext, targets: MutableSet>): Boolean { - var doneSomething = false - try { - // cleanup outputs - val targetToRemovedSources: MutableMap?, MutableCollection?> = HashMap?, MutableCollection?>() - - val dirsToDelete = FileCollectionFactory.createCanonicalFileSet() - for (target in targets) { - val deletedPaths = projectDescriptor.fsState.getAndClearDeletedPaths(target) - if (deletedPaths.isEmpty()) { - continue - } - - targetToRemovedSources.put(target, deletedPaths) - if (isTargetOutputCleared(context, target)) { - continue - } - val buildTargetId = context.projectDescriptor.targetsState.getBuildTargetId(target) - val shouldPruneEmptyDirs = target is ModuleBasedTarget<*> - val dataManager = context.projectDescriptor.dataManager - val sourceToOutputStorage = dataManager.getSourceToOutputMap(target) - val logger = context.loggingManager.projectBuilderLogger - // actually delete outputs associated with removed paths - val pathsForIteration: Collection - if (isTestMode) { - // ensure predictable order in test logs - pathsForIteration = deletedPaths.sorted() - } - else { - pathsForIteration = deletedPaths - } - for (deletedSource in pathsForIteration) { - // deleting outputs corresponding to a non-existing source - val outputs = sourceToOutputStorage.getOutputs(deletedSource) - if (outputs != null && !outputs.isEmpty()) { - val deletedOutputPaths: MutableList = ArrayList() - val outputToSourceRegistry = dataManager.outputToTargetMapping - for (output in outputToSourceRegistry.removeTargetAndGetSafeToDeleteOutputs(outputs, buildTargetId, sourceToOutputStorage)) { - val deleted = BuildOperations.deleteRecursively(output, deletedOutputPaths, if (shouldPruneEmptyDirs) dirsToDelete else null) - if (deleted) { - doneSomething = true - } - } - if (!deletedOutputPaths.isEmpty()) { - if (logger.isEnabled) { - logger.logDeletedFiles(deletedOutputPaths) - } - context.processMessage(FileDeletedEvent(deletedOutputPaths)) - } - } - - if (target is ModuleBuildTarget) { - // check if the deleted source was associated with a form - val sourceToFormMap = dataManager.getSourceToFormMap(target) - val boundForms = sourceToFormMap.getOutputs(deletedSource) - if (boundForms != null) { - for (formPath in boundForms) { - val formFile = File(formPath) - if (formFile.exists()) { - FSOperations.markDirty(context, CompilationRound.CURRENT, formFile) - } - } - sourceToFormMap.remove(deletedSource) - } - } - } - } - if (!targetToRemovedSources.isEmpty()) { - val existing = Utils.REMOVED_SOURCES_KEY.get(context) - if (existing != null) { - for (entry in existing.entries) { - val paths = targetToRemovedSources.get(entry.key) - if (paths != null) { - paths.addAll(entry.value!!) - } - else { - targetToRemovedSources.put(entry.key, entry.value) - } - } - } - Utils.REMOVED_SOURCES_KEY.set(context, targetToRemovedSources) - } - - FSOperations.pruneEmptyDirs(context, dirsToDelete) - } - catch (e: IOException) { - throw ProjectBuildException(e) - } - return doneSomething - } - - private fun CoroutineScope.clearOutputs(context: CompileContext) { - val rootsToDelete = HashMap>>() - val allSourceRoots = FileCollectionFactory.createCanonicalFileSet() - - val projectDescriptor = context.projectDescriptor - val allTargets = projectDescriptor.buildTargetIndex.allTargets - for (target in allTargets) { - if (target is ModuleBasedTarget<*>) { - for (file in target.getOutputRoots(context)) { - rootsToDelete.computeIfAbsent(file) { ArrayList() }.add(target) - } - } - else if (context.scope.isBuildForced(target)) { - launch { - doClearOutputFiles(context, target) - } - } - } - - val moduleIndex = projectDescriptor.moduleExcludeIndex - for (target in allTargets) { - for (descriptor in projectDescriptor.buildRootIndex.getTargetRoots(target, context)) { - // excluding from checks roots with generated sources; because it is safe to delete generated stuff - if (!descriptor.isGenerated) { - val rootFile = descriptor.rootFile - // Some roots aren't marked by as generated, but in fact they are produced by some builder, and it's safe to remove them. - // However, if a root isn't excluded, it means that its content will be shown in 'Project View' - // and a user can create new files under it, so it would be dangerous to clean such roots - if (moduleIndex.isInContent(rootFile)) { - allSourceRoots.add(rootFile) - } - } - } - } - - // check that output and source roots are not overlapping - val compileScope = context.scope - val filesToDelete = ArrayList() - for (entry in rootsToDelete) { - context.checkCanceled() - val outputRoot = entry.key - val rootTargets = entry.value - val applicability = Applicability.calculate(rootTargets) { compileScope.isBuildForced(it) } - if (applicability == Applicability.NONE) { - continue - } - - // It makes no sense to delete already empty root, but instead it makes sense to clean up the target, because there may exist - // a directory that has been previously the output root for the target - var okToDelete = applicability == Applicability.ALL && !isDirEmpty(outputRoot.toPath()) - if (okToDelete && !moduleIndex.isExcluded(outputRoot)) { - // If the output root itself is directly or indirectly excluded, - // there cannot be any manageable sources under it. - // This holds true even if the output root is located within a source root. - // Therefore, in such cases, it is safe to delete the output root. - if (JpsPathUtil.isUnder(allSourceRoots, outputRoot)) { - okToDelete = false - } - else { - val outRootCanonical = FileCollectionFactory.createCanonicalFileSet(listOf(outputRoot)) - for (srcRoot in allSourceRoots) { - if (JpsPathUtil.isUnder(outRootCanonical, srcRoot)) { - okToDelete = false - break - } - } - } - if (!okToDelete) { - context.processMessage(CompilerMessage( - "", BuildMessage.Kind.WARNING, - "Output path $outputRoot intersects with a source root. Only files that were created by build will be cleaned.") - ) - } - } - - if (okToDelete) { - filesToDelete.add(outputRoot.toPath()) - registerTargetsWithClearedOutput(context, rootTargets) - } - else { - // clean only those files we are aware of - for (target in rootTargets) { - if (compileScope.isBuildForced(target)) { - launch { - doClearOutputFiles(context, target) - } - } - } - } - } - - if (!filesToDelete.isEmpty()) { - launch { - var error: Throwable? = null - for (file in filesToDelete) { - try { - FileUtilRt.deleteRecursively(file) - } - catch (e: Throwable) { - if (error == null) { - error = e - } - else { - error.addSuppressed(e) - } - } - } - - error?.let { - throw it - } - } - } - } - - private fun > buildTarget(target: T, context: CompileContext, builder: TargetBuilder<*, *>) { - if (builder.getTargetTypes().contains(target.getTargetType())) { - val holder = object : DirtyFilesHolderBase(context) { - override fun processDirtyFiles(processor: FileProcessor) { - context.projectDescriptor.fsState.processFilesToRecompile(context, target, processor) - } - } - val outputConsumer = BuildOutputConsumerImpl(target, context) - val start = System.nanoTime() - @Suppress("UNCHECKED_CAST") - (builder as TargetBuilder).build(target, holder, outputConsumer, context) - storeBuilderStatistics(builder, System.nanoTime() - start, outputConsumer.numberOfProcessedSources) - outputConsumer.fireFileGeneratedEvent() - context.checkCanceled() - } - } - - // return true if changed something, false otherwise - private fun runModuleLevelBuilders(context: CompileContext, chunk: ModuleChunk, buildProgress: BuildProgress): Boolean { - for (category in BuilderCategory.entries) { - for (builder in builderRegistry.getBuilders(category)) { - builder.chunkBuildStarted(context, chunk) - } - } - - completeRecompiledSourcesSet(context, chunk.targets) - - var doneSomething = false - var rebuildFromScratchRequested = false - var nextPassRequired: Boolean - val outputConsumer = ChunkBuildOutputConsumerImpl(context) - try { - do { - nextPassRequired = false - projectDescriptor.fsState.beforeNextRoundStart(context, chunk) - - val dirtyFilesHolder = object : DirtyFilesHolderBase(context) { - override fun processDirtyFiles(processor: FileProcessor) { - FSOperations.processFilesToRecompile(context, chunk, processor) - } - } - if (!JavaBuilderUtil.isForcedRecompilationAllJavaModules(context.scope)) { - val cleanedSources = BuildOperations.cleanOutputsCorrespondingToChangedFiles(context, dirtyFilesHolder) - for (entry in cleanedSources.entries) { - val target: ModuleBuildTarget = entry.key - val files = entry.value.keys - if (!files.isEmpty()) { - val mapping = context.projectDescriptor.dataManager.getSourceToOutputMap(target) - for (srcFile in files) { - val outputs = entry.value.get(srcFile)!! - mapping.setOutputs(srcFile.path, outputs) - if (!outputs.isEmpty()) { - LOG.info("Some outputs were not removed for ${srcFile.path} source file: $outputs") - } - } - } - } - } - - try { - var buildersPassed = 0 - BUILDER_CATEGORY_LOOP@ for (category in BuilderCategory.entries) { - val builders = builderRegistry.getBuilders(category) - if (category == BuilderCategory.CLASS_POST_PROCESSOR) { - // ensure changes from instruments are visible to class post-processors - saveInstrumentedClasses(outputConsumer) - } - if (builders.isEmpty()) { - continue - } - - try { - for (builder in builders) { - outputConsumer.setCurrentBuilderName(builder.presentableName) - processDeletedPaths(context, chunk.targets) - val start = System.nanoTime() - val processedSourcesBefore = outputConsumer.getNumberOfProcessedSources() - val buildResult = builder.build(context, chunk, dirtyFilesHolder, outputConsumer) - storeBuilderStatistics(builder, System.nanoTime() - start, outputConsumer.getNumberOfProcessedSources() - processedSourcesBefore) - - doneSomething = doneSomething or (buildResult != ModuleLevelBuilder.ExitCode.NOTHING_DONE) - - if (buildResult == ModuleLevelBuilder.ExitCode.ABORT) { - throw StopBuildException("Builder ${builder.presentableName} requested build stop") - } - context.checkCanceled() - if (buildResult == ModuleLevelBuilder.ExitCode.ADDITIONAL_PASS_REQUIRED) { - nextPassRequired = true - } - else if (buildResult == ModuleLevelBuilder.ExitCode.CHUNK_REBUILD_REQUIRED) { - if (!rebuildFromScratchRequested && !JavaBuilderUtil.isForcedRecompilationAllJavaModules(context.scope)) { - notifyChunkRebuildRequested(context, chunk, builder) - // allow rebuild from scratch only once per chunk - rebuildFromScratchRequested = true - try { - // forcibly mark all files in the chunk dirty - context.projectDescriptor.fsState.clearContextRoundData(context) - FSOperations.markDirty(context, CompilationRound.NEXT, chunk, null) - // reverting to the beginning - nextPassRequired = true - outputConsumer.clear() - break@BUILDER_CATEGORY_LOOP - } - catch (e: Exception) { - throw ProjectBuildException(e) - } - } - else { - LOG.debug("Builder " + builder.presentableName + " requested second chunk rebuild") - } - } - - buildersPassed++ - for (target in chunk.targets) { - buildProgress.updateProgress(target, (buildersPassed.toDouble()) / totalModuleLevelBuilderCount, context) - } - } - } - finally { - outputConsumer.setCurrentBuilderName(null) - } - } - } - finally { - val moreToCompile = JavaBuilderUtil.updateMappingsOnRoundCompletion(context, dirtyFilesHolder, chunk) - if (moreToCompile) { - nextPassRequired = true - } - JavaBuilderUtil.clearDataOnRoundCompletion(context) - } - } while (nextPassRequired) - } - finally { - saveInstrumentedClasses(outputConsumer) - outputConsumer.fireFileGeneratedEvents() - outputConsumer.clear() - for (category in BuilderCategory.entries) { - for (builder in builderRegistry.getBuilders(category)) { - builder.chunkBuildFinished(context, chunk) - } - } - if (Utils.errorsDetected(context)) { - context.processMessage(CompilerMessage("", BuildMessage.Kind.JPS_INFO, "Errors occurred while compiling module ${chunk.presentableShortName}")) - } - } - - return doneSomething - } - - internal fun buildTargetsChunk(context: CompileContext, chunk: BuildTargetChunk, buildProgress: BuildProgress) { - val buildSpan = Tracer.start(Supplier { "Building ${chunk.presentableName}" }) - val fsState = projectDescriptor.fsState - var doneSomething: Boolean - try { - context.setCompilationStartStamp(chunk.targets, System.currentTimeMillis()) - - sendBuildingTargetMessages(chunk.targets, BuildingTargetProgressMessage.Event.STARTED) - Utils.ERRORS_DETECTED_KEY.set(context, java.lang.Boolean.FALSE) - - for (target in chunk.targets) { - BuildOperations.ensureFSStateInitialized(context, target, false) - } - - doneSomething = processDeletedPaths(context, chunk.targets) - - fsState.beforeChunkBuildStart(context, chunk) - - val runBuildersSpan = Tracer.start(Supplier { "runBuilders " + chunk.presentableName }) - doneSomething = doneSomething or runBuildersForChunk(context, chunk, buildProgress) - runBuildersSpan.complete() - - fsState.clearContextRoundData(context) - fsState.clearContextChunk(context) - - if (doneSomething) { - BuildOperations.markTargetsUpToDate(context, chunk) - } - } - catch (e: BuildDataCorruptedException) { - throw e - } - catch (e: ProjectBuildException) { - throw e - } - catch (e: Throwable) { - val message: @NlsSafe StringBuilder = StringBuilder() - message.append(chunk.presentableName).append(": ").append(e.javaClass.getName()) - val exceptionMessage = e.message - if (exceptionMessage != null) { - message.append(": ").append(exceptionMessage) - } - throw ProjectBuildException(message.toString(), e) - } - finally { - buildProgress.onTargetChunkFinished(chunk, context) - for (rd in context.projectDescriptor.buildRootIndex.clearTempRoots(context)) { - context.projectDescriptor.fsState.clearRecompile(rd) - } - try { - // restore deleted paths that were not processed by 'integrate' - val map = Utils.REMOVED_SOURCES_KEY.get(context) - if (map != null) { - for (entry in map.entries) { - val target = entry.key - val paths = entry.value - if (paths != null) { - for (path in paths) { - fsState.registerDeleted(context, target, File(path), null) - } - } - } - } - } - catch (e: IOException) { - throw ProjectBuildException(e) - } - finally { - Utils.REMOVED_SOURCES_KEY.set(context, null) - sendBuildingTargetMessages(chunk.targets, BuildingTargetProgressMessage.Event.FINISHED) - buildSpan.complete() - } - } - } - - private fun sendBuildingTargetMessages(targets: Set>, event: BuildingTargetProgressMessage.Event) { - messageHandler.processMessage(BuildingTargetProgressMessage(targets, event)) - } - - private fun storeBuilderStatistics(builder: Builder, elapsedTime: Long, processedFiles: Int) { - elapsedTimeNanosByBuilder.computeIfAbsent(builder) { AtomicLong() }.addAndGet(elapsedTime) - numberOfSourcesProcessedByBuilder.computeIfAbsent(builder) { AtomicInteger() }.addAndGet(processedFiles) - } -} - -private class ChunkBuildOutputConsumerImpl(private val context: CompileContext) : OutputConsumer { - private val target2Consumer = HashMap, BuildOutputConsumerImpl>() - private val classes = HashMap() - private val targetToClassesMap = HashMap, MutableCollection>() - private val outputToBuilderNameMap = Object2ObjectMaps.synchronize(Object2ObjectOpenCustomHashMap(FileHashStrategy)) - - @Volatile - private var currentBuilderName: String? = null - - fun setCurrentBuilderName(builderName: String?) { - currentBuilderName = builderName - } - - override fun getTargetCompiledClasses(target: BuildTarget<*>): Collection { - return Collections.unmodifiableCollection(targetToClassesMap.get(target) ?: return emptyList()) - } - - override fun getCompiledClasses(): MutableMap = Collections.unmodifiableMap(classes) - - override fun lookupClassBytes(className: String?): BinaryContent? = classes.get(className)?.content - - override fun registerCompiledClass(target: BuildTarget<*>?, compiled: CompiledClass) { - val className = compiled.className - if (className != null) { - classes.put(className, compiled) - if (target != null) { - var classes = targetToClassesMap.get(target) - if (classes == null) { - classes = ArrayList() - targetToClassesMap.put(target, classes) - } - classes.add(compiled) - } - } - if (target != null) { - registerOutputFile(target = target, outputFile = compiled.outputFile, sourcePaths = compiled.sourceFilesPaths) - } - } - - override fun registerOutputFile(target: BuildTarget<*>, outputFile: File, sourcePaths: MutableCollection) { - val currentBuilder = currentBuilderName - if (currentBuilder != null) { - val previousBuilder = outputToBuilderNameMap.put(outputFile, currentBuilder) - if (previousBuilder != null && previousBuilder != currentBuilder) { - val source = if (sourcePaths.isEmpty()) null else sourcePaths.iterator().next() - context.processMessage(CompilerMessage( - currentBuilder, BuildMessage.Kind.ERROR, "Output file \"${outputFile}\" has already been registered by \"$previousBuilder\"", source - )) - } - } - var consumer = target2Consumer.get(target) - if (consumer == null) { - consumer = BuildOutputConsumerImpl(target, context) - target2Consumer.put(target, consumer) - } - consumer.registerOutputFile(outputFile, sourcePaths) - } - - fun fireFileGeneratedEvents() { - for (consumer in target2Consumer.values) { - consumer.fireFileGeneratedEvent() - } - } - - fun getNumberOfProcessedSources(): Int { - var total = 0 - for (consumer in target2Consumer.values) { - total += consumer.numberOfProcessedSources - } - return total - } - - fun clear() { - target2Consumer.clear() - classes.clear() - targetToClassesMap.clear() - outputToBuilderNameMap.clear() - } -} - -private class ChainedTargetsBuildListener(private val context: CompileContextImpl) : BuildListener { - override fun filesGenerated(event: FileGeneratedEvent) { - val projectDescriptor = this.context.projectDescriptor - val fsState = projectDescriptor.fsState - for (pair in event.paths) { - val relativePath = pair.getSecond() as String - val file = if (relativePath == ".") File(pair.getFirst()) else File(pair.getFirst(), relativePath) - for (buildRootDescriptor in projectDescriptor.buildRootIndex.findAllParentDescriptors(file, this.context)) { - val target = buildRootDescriptor.target - if (event.sourceTarget != target) { - try { - fsState.markDirty(context, file, buildRootDescriptor, projectDescriptor.dataManager.getFileStampStorage(target), false) - } - catch (_: IOException) { - } - } - } - } - } - - override fun filesDeleted(event: FileDeletedEvent) { - val state = context.projectDescriptor.fsState - val rootIndex = context.projectDescriptor.buildRootIndex - for (path in event.filePaths) { - val file = File(FileUtilRt.toSystemDependentName(path)) - for (desc in rootIndex.findAllParentDescriptors(file, context)) { - state.registerDeleted(context, desc.target, file) - } - } - } -} - -private enum class Applicability { - NONE, PARTIAL, ALL; - - companion object { - inline fun calculate(collection: Collection, p: (T) -> Boolean): Applicability { - var count = 0 - var item = 0 - for (elem in collection) { - item++ - if (p(elem)) { - count++ - if (item > count) { - return PARTIAL - } - } - else if (count > 0) { - return PARTIAL - } - } - return if (count == 0) NONE else ALL - } - } -} - -private fun calculateEstimatedBuildTime(projectDescriptor: ProjectDescriptor, isAffected: (BuildTarget<*>) -> Boolean): Long { - val targetsState = projectDescriptor.targetsState - // compute estimated times for dirty targets - var estimatedBuildTime = 0L - - val targetIndex = projectDescriptor.buildTargetIndex - var affectedTargets = 0 - for (target in targetIndex.allTargets) { - if (!targetIndex.isDummy(target)) { - val avgTimeToBuild = targetsState.getAverageBuildTime(target.getTargetType()) - if (avgTimeToBuild > 0) { - // 1. in general case, this time should include dependency analysis and cache update times - // 2. need to check isAffected() since some targets (like artifacts) may be unaffected even for rebuild - if (targetsState.getTargetConfiguration(target).isTargetDirty(projectDescriptor) && isAffected(target)) { - estimatedBuildTime += avgTimeToBuild - affectedTargets++ - } - } - } - } - LOG.info("Affected build targets count: $affectedTargets") - return estimatedBuildTime -} - -private fun reportRebuiltModules(context: CompileContextImpl) { - val modules = BuildTargetConfiguration.MODULES_WITH_TARGET_CONFIG_CHANGED_KEY.get(context) - if (modules.isNullOrEmpty()) { - return - } - - val shown = if (modules.size == 6) 6 else min(5, modules.size) - val modulesText = modules.stream().limit(shown.toLong()).map { m -> "'" + m.name + "'" }.collect(Collectors.joining(", ")) - val text = JpsBuildBundle.message("build.messages.modules.were.fully.rebuilt", modulesText, modules.size, - modules.size - shown, if (ModuleBuildTarget.REBUILD_ON_DEPENDENCY_CHANGE) 1 else 0) - context.processMessage(CompilerMessage("", BuildMessage.Kind.INFO, text)) -} - -private fun reportUnprocessedChanges(context: CompileContextImpl) { - val pd = context.projectDescriptor - val fsState = pd.fsState - for (target in pd.buildTargetIndex.allTargets) { - if (fsState.hasUnprocessedChanges(context, target)) { - context.processMessage(UnprocessedFSChangesNotification()) - break - } - } -} - -internal fun clearOutputFiles(context: CompileContext, target: BuildTarget<*>) { - val map = context.projectDescriptor.dataManager.getSourceToOutputMap(target) - val targetType = target.getTargetType() - clearOutputFiles( - context = context, - mapping = map, - targetType = targetType, - targetId = context.projectDescriptor.dataManager.targetsState.getBuildTargetId(target) - ) - registerTargetsWithClearedOutput(context, listOf(target)) -} - -private fun registerTargetsWithClearedOutput(context: CompileContext, targets: Collection>) { - synchronized(TARGET_WITH_CLEARED_OUTPUT) { - var data = context.getUserData(TARGET_WITH_CLEARED_OUTPUT) - if (data == null) { - data = HashSet>() - context.putUserData(TARGET_WITH_CLEARED_OUTPUT, data) - } - data.addAll(targets) - } -} - -private fun isTargetOutputCleared(context: CompileContext, target: BuildTarget<*>?): Boolean { - synchronized(TARGET_WITH_CLEARED_OUTPUT) { - val data = context.getUserData(TARGET_WITH_CLEARED_OUTPUT) - return data != null && data.contains(target) - } -} - -private fun isDirEmpty(outputRoot: Path): Boolean { - try { - Files.newDirectoryStream(outputRoot).use { dirStream -> return !dirStream.iterator().hasNext() } - } - catch (_: IOException) { - return true - } - return true -} - -private fun doClearOutputFiles(context: CompileContext, target: BuildTarget<*>) { - try { - clearOutputFiles(context, target) - } - catch (e: Throwable) { - LOG.info(e) - val reason = e.message ?: e.javaClass.getName() - context.processMessage(CompilerMessage("", BuildMessage.Kind.WARNING, "Problems clearing output files for target \"${target.presentableName}\": $reason")) - } -} - -private fun wrapWithModuleInfoAppender(context: CompileContext, moduleTargets: MutableCollection): CompileContext { - val messageHandlerInterface = MessageHandler::class.java - return ReflectionUtil.proxy(context.javaClass.getClassLoader(), CompileContext::class.java, object : InvocationHandler { - override fun invoke(proxy: Any?, method: Method, args: Array?): Any? { - if (args != null && args.isNotEmpty() && messageHandlerInterface == method.declaringClass) { - for (arg in args) { - if (arg is CompilerMessage) { - val compilerMessage = arg - for (target in moduleTargets) { - compilerMessage.addModuleName(target.module.name) - } - break - } - } - } - val mh = lookup.unreflect(method) - return if (args == null) mh.invoke(context) else mh.bindTo(context).asSpreader(Array::class.java, args.size).invoke(args) - } - }) -} - -/** - * if an output file is generated from multiple sources, make sure all of them are added for recompilation - */ -@Suppress("SpellCheckingInspection") -private fun , R : BuildRootDescriptor> completeRecompiledSourcesSet(context: CompileContext, targets: Collection) { - val scope = context.scope - for (target in targets) { - if (scope.isBuildForced(target)) { - // assuming build is either forced for all targets in a chunk or for none of them - return - } - } - - val projectDescriptor = context.projectDescriptor - val affectedOutputs = CollectionFactory.createFilePathSet() - val affectedSources = CollectionFactory.createFilePathSet() - - val mappings = ArrayList() - for (target in targets) { - projectDescriptor.fsState.processFilesToRecompile(context, target, object : FileProcessor { - private var srcToOut: SourceToOutputMapping? = null - - override fun apply(target: T, file: File, root: R): Boolean { - val src = file.invariantSeparatorsPath - if (!affectedSources.add(src)) { - return true - } - - // lazy init - var srcToOut = this.srcToOut - if (srcToOut == null) { - srcToOut = projectDescriptor.dataManager.getSourceToOutputMap(target) - mappings.add(srcToOut) - this.srcToOut = srcToOut - } - - val outs = srcToOut.getOutputs(src) ?: return true - // Temporary hack for KTIJ-197 - // Change of only one input of *.kotlin_module files didn't trigger recompilation of all inputs in old behavior. - // Now it does. It isn't yet obvious whether it is right or wrong behavior. Let's leave old behavior for a - // while for safety and keeping kotlin incremental JPS tests green - outs.filterTo(affectedOutputs) { "kotlin_module" != it.substringAfterLast('.') } - return true - } - }) - } - - if (affectedOutputs.isEmpty()) { - return - } - - for (srcToOut in mappings) { - val cursor = srcToOut.cursor() - while (cursor.hasNext()) { - val src = cursor.next() - if (!affectedSources.contains(src)) { - for (out in cursor.outputPaths) { - if (affectedOutputs.contains(out)) { - FSOperations.markDirtyIfNotDeleted(context, CompilationRound.CURRENT, File(src)) - break - } - } - } - } - } -} - -private fun cleanOldOutputs(context: CompileContext, target: BuildTarget) { - if (!context.scope.isBuildForced(target)) { - BuildOperations.cleanOutputsCorrespondingToChangedFiles(context, object : DirtyFilesHolderBase>(context) { - override fun processDirtyFiles(processor: FileProcessor>) { - context.projectDescriptor.fsState.processFilesToRecompile(context, target, processor) - } - }) - } -} - -private fun clearOutputFiles( - context: CompileContext, - mapping: SourceToOutputMapping, - targetType: BuildTargetType<*>, - targetId: Int, -) { - val dirsToDelete = if (targetType is ModuleBasedBuildTargetType<*>) FileCollectionFactory.createCanonicalFileSet() else null - val outputToTargetRegistry = context.projectDescriptor.dataManager.outputToTargetMapping - val cursor = mapping.cursor() - while (cursor.hasNext()) { - cursor.next() - val outs = cursor.outputPaths - if (outs.isNotEmpty()) { - val deletedPaths = ArrayList() - for (out in outs) { - BuildOperations.deleteRecursively(out, deletedPaths, dirsToDelete) - } - outputToTargetRegistry.removeMappings(outs.asList(), targetId, mapping) - if (!deletedPaths.isEmpty()) { - context.processMessage(FileDeletedEvent(deletedPaths)) - } - } - } - if (dirsToDelete != null) { - FSOperations.pruneEmptyDirs(context, dirsToDelete) - } -} - -private fun notifyChunkRebuildRequested(context: CompileContext, chunk: ModuleChunk, builder: ModuleLevelBuilder) { - var infoMessage = "Builder \"${builder.presentableName}\" requested rebuild of module chunk \"${chunk.name}\"" - var kind = BuildMessage.Kind.JPS_INFO - val scope = context.scope - for (target in chunk.targets) { - if (!scope.isWholeTargetAffected(target)) { - infoMessage += ".\n" - infoMessage += "Consider building whole project or rebuilding the module." - kind = BuildMessage.Kind.INFO - break - } - } - context.processMessage(CompilerMessage("", kind, infoMessage)) -} - -private fun saveInstrumentedClasses(outputConsumer: ChunkBuildOutputConsumerImpl) { - for (compiledClass in outputConsumer.compiledClasses.values) { - if (compiledClass.isDirty) { - compiledClass.save() - } - } -} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/StorageInitializer.kt b/build/jvm-rules/src/jps-builder/StorageInitializer.kt new file mode 100644 index 000000000000..28826a0bbd1a --- /dev/null +++ b/build/jvm-rules/src/jps-builder/StorageInitializer.kt @@ -0,0 +1,162 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("UnstableApiUsage", "ReplaceGetOrSet") + +package org.jetbrains.bazel.jvm.jps + +import com.intellij.openapi.util.io.FileUtilRt +import kotlinx.coroutines.ensureActive +import org.h2.mvstore.MVStore +import org.jetbrains.bazel.jvm.jps.impl.BazelModuleBuildTarget +import org.jetbrains.bazel.jvm.jps.impl.loadJpsProject +import org.jetbrains.bazel.jvm.jps.state.TargetConfigurationDigestContainer +import org.jetbrains.bazel.jvm.jps.state.checkConfiguration +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.incremental.fs.BuildFSState +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import org.jetbrains.jps.incremental.storage.StorageManager +import org.jetbrains.jps.incremental.storage.StoreLogger +import org.jetbrains.jps.incremental.storage.tryOpenMvStore +import org.jetbrains.jps.model.JpsModel +import java.nio.file.Files +import java.nio.file.Path +import kotlin.coroutines.coroutineContext + +internal class StorageInitializer(private val dataDir: Path, private val classOutDir: Path) { + private var storageManager: StorageManager? = null + private val cacheDbFile = dataDir.resolve("jps-portable-cache.db") + + var isCheckRebuildRequired: Boolean = true + private set + + private var wasCleared = false + + var isCleanBuild: Boolean = false + private set + + fun clearAndInit(messageHandler: ConsoleMessageHandler): StorageManager { + isCheckRebuildRequired = false + wasCleared = true + isCleanBuild = true + + clearStorage() + + Files.createDirectories(dataDir) + val logger = createLogger(messageHandler) + val store = tryOpenMvStore(file = cacheDbFile, readOnly = false, autoCommitDelay = 0, logger = logger) + return StorageManager(cacheDbFile, store) + .also { storageManager = it } + } + + suspend fun init(messageHandler: ConsoleMessageHandler, targetDigests: TargetConfigurationDigestContainer): StorageManager { + val logger = createLogger(messageHandler) + coroutineContext.ensureActive() + + isCleanBuild = Files.notExists(cacheDbFile) + + if (isCleanBuild && Files.isDirectory(dataDir)) { + messageHandler.info("remove $dataDir and $classOutDir because no cache db file found: $cacheDbFile") + // if no db file, make sure that data dir is also not reused + deleteDirs() + } + Files.createDirectories(dataDir) + + var store = try { + tryOpenMvStore(file = cacheDbFile, readOnly = false, autoCommitDelay = 0, logger = logger) + } + catch (e: Throwable) { + messageHandler.info("rebuild due to internal error: ${e.stackTraceToString()}") + clearStorage() + + return StorageManager(cacheDbFile, createStoreAfterClear(logger)) + .also { storageManager = it } + } + + val reasonToRebuild = try { + checkConfiguration(store, targetDigests) + } + catch (e: Throwable) { + store.closeImmediately() + "Cannot open cache storage: ${e.stackTraceToString()}" + } + + if (reasonToRebuild != null) { + store.closeImmediately() + + messageHandler.info("rebuild: $reasonToRebuild") + clearStorage() + store = createStoreAfterClear(logger) + } + + return StorageManager(cacheDbFile, store) + .also { storageManager = it } + } + + private suspend fun createStoreAfterClear(logger: StoreLogger): MVStore { + require(wasCleared) + + coroutineContext.ensureActive() + + Files.createDirectories(dataDir) + return tryOpenMvStore(file = cacheDbFile, readOnly = false, autoCommitDelay = 0, logger = logger) + } + + private fun createLogger(messageHandler: ConsoleMessageHandler): StoreLogger { + return { m: String, e: Throwable, isWarn: Boolean -> + val message = "$m: ${e.stackTraceToString()}" + if (isWarn) { + messageHandler.warn(message) + } + else { + messageHandler.error(message) + } + } + } + + fun createProjectDescriptor( + messageHandler: ConsoleMessageHandler, + jpsModel: JpsModel, + moduleTarget: BazelModuleBuildTarget, + relativizer: PathRelativizerService, + ): ProjectDescriptor { + try { + return loadJpsProject( + dataStorageRoot = dataDir, + // alwaysScanFS doesn't matter, we use our own version of `BuildOperations.ensureFSStateInitialized`, + // see `JpsProjectBuilder.ensureFsStateInitialized` + fsState = BuildFSState(/* alwaysScanFS = */ true), + jpsModel = jpsModel, + moduleTarget = moduleTarget, + relativizer = relativizer, + storageManager = storageManager!!, + ) + } + catch (e: Throwable) { + storageManager!!.forceClose() + if (wasCleared) { + throw e + } + + messageHandler.warn("Cannot open cache storage: ${e.stackTraceToString()}") + } + + return createProjectDescriptor( + messageHandler = messageHandler, + jpsModel = jpsModel, + moduleTarget = moduleTarget, + relativizer = relativizer, + ) + } + + private fun clearStorage() { + isCleanBuild = true + wasCleared = true + isCheckRebuildRequired = false + // todo rename and store + deleteDirs() + } + + private fun deleteDirs() { + FileUtilRt.deleteRecursively(dataDir) + FileUtilRt.deleteRecursively(classOutDir) + } +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/TestJpsBuildWorker.kt b/build/jvm-rules/src/jps-builder/TestJpsBuildWorker.kt index 827f13545fde..18e21f7db03e 100644 --- a/build/jvm-rules/src/jps-builder/TestJpsBuildWorker.kt +++ b/build/jvm-rules/src/jps-builder/TestJpsBuildWorker.kt @@ -3,10 +3,12 @@ package org.jetbrains.bazel.jvm.jps import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking +import org.jetbrains.bazel.jvm.kotlin.JvmBuilderFlags import org.jetbrains.bazel.jvm.kotlin.parseArgs -import java.io.PrintWriter +import java.io.StringWriter import java.nio.file.Files import java.nio.file.Path +import java.security.MessageDigest import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.invariantSeparatorsPathString import kotlin.io.path.walk @@ -18,41 +20,52 @@ object TestJpsBuildWorker { configureGlobalJps() val userHomeDir = Path.of(System.getProperty("user.home")) - val workingDir = userHomeDir.resolve("bazel-jps-test") - Files.createDirectories(workingDir) + val out = StringWriter() try { - val out = PrintWriter(System.out) - try { - //val sources = Files.newDirectoryStream(userHomeDir.resolve("projects/idea/community/platform/util/xmlDom/src")).use { it.toList() } - val sourceDir = userHomeDir.resolve("projects/idea/community/platform/platform-impl/src") - val sources = sourceDir - .walk() - .filter { - val p = it.toString() - p.endsWith(".kt") || p.endsWith(".java") - } - .sortedBy { sourceDir.relativize(it).invariantSeparatorsPathString } - .toList() - runBlocking(Dispatchers.Default) { - buildUsingJps( - workingDir = workingDir, - args = parseArgs(testParams.trimStart().lines()), - out = out, - sources = sources, - classPathRootDir = Path.of("/private/var/tmp/_bazel_develar/1693e3b60d193556354eca5e9446027e/execroot/_main"), - ) + @Suppress("SpellCheckingInspection") + val baseDir = Path.of("/private/var/tmp/_bazel_develar/c002af20f6ada3e2667e9e2ceaf2ceca/execroot/_main") + + //val sources = Files.newDirectoryStream(userHomeDir.resolve("projects/idea/community/platform/util/xmlDom/src")).use { it.toList() } + val ideaProjectDirName = if (Runtime.getRuntime().availableProcessors() >= 20) "idea-push" else "idea-second" + val communityDir = userHomeDir.resolve("projects/$ideaProjectDirName/community") + val sources = communityDir.resolve("platform/platform-impl/src") + .walk() + .filter { + val p = it.toString() + p.endsWith(".kt") || p.endsWith(".java") } - } - finally { - out.flush() + .map { "../community+/" + communityDir.relativize(it).invariantSeparatorsPathString } + .sorted() + .map { baseDir.resolve(it).normalize() } + .toList() + + require(sources.isNotEmpty()) + + runBlocking(Dispatchers.Default) { + val args = parseArgs(testParams.trimStart().lines().toTypedArray()) + val messageDigest = MessageDigest.getInstance("SHA-256") + buildUsingJps( + baseDir = baseDir, + args = args, + out = out, + sources = sources, + dependencyFileToDigest = args.optionalList(JvmBuilderFlags.CLASSPATH).associate { + val file = baseDir.resolve(it).normalize() + val digest = messageDigest.digest(Files.readAllBytes(file)) + messageDigest.reset() + file to digest + }, + isDebugEnabled = true, + ) } } finally { - //workingDir.deleteRecursively() + System.out.append(out.toString()) } } } +@Suppress("SpellCheckingInspection") private const val testParams = """ --target_label @community//platform/platform-impl:ide-impl @@ -69,8 +82,10 @@ java.desktop/sun.awt.image=ALL-UNNAMED java.desktop/sun.awt.datatransfer=ALL-UNNAMED java.desktop/sun.swing=ALL-UNNAMED java.base/sun.nio.fs=ALL-UNNAMED ---output +--out bazel-out/community+/darwin_arm64-fastbuild/bin/platform/platform-impl/ide-impl.jar +--abi-out +bazel-out/community+/darwin_arm64-fastbuild/bin/platform/platform-impl/ide-impl.abi.jar --warn off --jvm-default diff --git a/build/jvm-rules/src/jps-builder/impl/BazelBuildRootIndex.kt b/build/jvm-rules/src/jps-builder/impl/BazelBuildRootIndex.kt new file mode 100644 index 000000000000..533952193172 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/impl/BazelBuildRootIndex.kt @@ -0,0 +1,92 @@ +@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "ReplaceGetOrSet") + +package org.jetbrains.bazel.jvm.jps.impl + +import com.intellij.openapi.util.io.FileFilters +import org.jetbrains.jps.builders.BuildRootDescriptor +import org.jetbrains.jps.builders.BuildRootIndex +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.BuildTargetType +import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor +import org.jetbrains.jps.incremental.CompileContext +import java.io.File +import java.io.FileFilter +import java.nio.file.Path + +internal class BazelBuildRootIndex(target: BazelModuleBuildTarget) : BuildRootIndex { + @JvmField val fileToDescriptors = HashMap() + @JvmField val descriptors = target.computeRootDescriptors() + + init { + val descriptors = descriptors + for (descriptor in descriptors) { + val old = fileToDescriptors.put(descriptor.rootFile, descriptor) + require(old == null) { + "Duplicated root (old=$old, new=$descriptor)" + } + } + } + + override fun getRootDescriptors( + root: File, + types: Collection>>?, + context: CompileContext?, + ): List { + val descriptor = fileToDescriptors.get(root.toPath()) ?: return java.util.List.of() + if (types == null || types.any { it == JavaSourceRootDescriptor::class.java }) { + @Suppress("UNCHECKED_CAST") + return java.util.List.of(descriptor as R) + } + return java.util.List.of() + } + + @Suppress("UNCHECKED_CAST") + override fun getTargetRoots(target: BuildTarget, context: CompileContext?): List = descriptors as List + + override fun getTempTargetRoots(target: BuildTarget, context: CompileContext): List = java.util.List.of() + + override fun associateTempRoot(context: CompileContext, target: BuildTarget, root: R) { + throw IllegalStateException("Should not be called") + } + + // in Bazel, each root it is a file, so, no parent + override fun findParentDescriptor( + file: File, + types: Collection>>, + context: CompileContext?, + ): R? { + if (types.any { it == JavaSourceRootDescriptor::class.java }) { + @Suppress("UNCHECKED_CAST") + return fileToDescriptors.get(file.toPath()) as R + } + return null + } + + override fun findAllParentDescriptors( + file: File, + types: Collection>>?, + context: CompileContext?, + ): Collection { + if (types == null || types.any { it == JavaSourceRootDescriptor::class.java }) { + return findAllParentDescriptors(file, context) + } + return java.util.List.of() + } + + override fun findAllParentDescriptors(file: File, context: CompileContext?): Collection { + @Suppress("UNCHECKED_CAST") + return fileToDescriptors.get(file.toPath())?.let { java.util.List.of(it as R) } ?: java.util.List.of() + } + + override fun clearTempRoots(context: CompileContext): Collection = java.util.List.of() + + override fun findJavaRootDescriptor(context: CompileContext?, file: File): JavaSourceRootDescriptor? { + return fileToDescriptors.get(file.toPath()) + } + + override fun getRootFilter(descriptor: BuildRootDescriptor): FileFilter = FileFilters.EVERYTHING + + override fun isFileAccepted(file: Path, descriptor: BuildRootDescriptor) = true + + override fun isDirectoryAccepted(dir: Path, descriptor: BuildRootDescriptor) = true +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/BazelBuildTargetStateManager.kt b/build/jvm-rules/src/jps-builder/impl/BazelBuildTargetStateManager.kt new file mode 100644 index 000000000000..bf5e2172135b --- /dev/null +++ b/build/jvm-rules/src/jps-builder/impl/BazelBuildTargetStateManager.kt @@ -0,0 +1,62 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("NonExtendableApiUsage", "UnstableApiUsage", "ReplaceJavaStaticMethodWithKotlinAnalog") + +package org.jetbrains.bazel.jvm.jps.impl + +import com.intellij.openapi.util.Pair +import org.jetbrains.bazel.jvm.jps.state.TargetStateContainer +import org.jetbrains.bazel.jvm.jps.state.TargetStateProperty +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.BuildTargetType +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration +import org.jetbrains.jps.incremental.storage.BuildTargetStateManager + +internal class BazelBuildTargetStateManager( + val state: TargetStateContainer, +) : BuildTargetStateManager { + override fun getBuildTargetId(target: BuildTarget<*>): Int = 1 + + override fun getTargetConfiguration(target: BuildTarget<*>): BuildTargetConfiguration { + throw UnsupportedOperationException() + } + + override fun getStaleTargetIds(type: BuildTargetType<*>): List> = java.util.List.of() + + override fun cleanStaleTarget(type: BuildTargetType<*>, targetId: String) { + } + + override fun getLastSuccessfulRebuildDuration(): Long { + return state.get(TargetStateProperty.LastSuccessfulRebuildDuration) + } + + override fun setLastSuccessfulRebuildDuration(duration: Long) { + state.set(TargetStateProperty.LastSuccessfulRebuildDuration, duration) + } + + override fun getAverageBuildTime(type: BuildTargetType<*>): Long { + return state.get(TargetStateProperty.AverageBuildTime) + } + + override fun setAverageBuildTime(type: BuildTargetType<*>, time: Long) { + state.set(TargetStateProperty.AverageBuildTime, time) + } + + override fun save() { + } + + override fun clean() { + throw UnsupportedOperationException() + } + + override fun storeNonExistentOutputRoots(target: BuildTarget<*>, context: CompileContext) { + } + + override fun isTargetDirty(target: BuildTarget<*>, projectDescriptor: ProjectDescriptor): Boolean = false + + override fun invalidate(target: BuildTarget<*>) { + // we do not invalidate as JPS does - if target configuration is changed, we delete output before build + throw UnsupportedOperationException() + } +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/BazelModuleBuildTarget.kt b/build/jvm-rules/src/jps-builder/impl/BazelModuleBuildTarget.kt new file mode 100644 index 000000000000..170f2217e036 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/impl/BazelModuleBuildTarget.kt @@ -0,0 +1,63 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "NonExtendableApiUsage", "RemoveRedundantQualifierName", "ReplaceGetOrSet", "UnstableApiUsage") + +package org.jetbrains.bazel.jvm.jps.impl + +import com.dynatrace.hash4j.hashing.HashSink +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.BuildTargetRegistry +import org.jetbrains.jps.builders.TargetOutputIndex +import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType +import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor +import org.jetbrains.jps.builders.storage.BuildDataPaths +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.ModuleBuildTarget +import org.jetbrains.jps.indices.IgnoredFileIndex +import org.jetbrains.jps.indices.ModuleExcludeIndex +import org.jetbrains.jps.model.JpsModel +import org.jetbrains.jps.model.module.JpsModule +import java.io.File +import java.nio.file.Path + +internal class BazelModuleBuildTarget( + outDir: Path, + module: JpsModule, + private val sources: List, +) : ModuleBuildTarget(module, JavaModuleBuildTargetType.PRODUCTION) { + private val outDirFile = outDir.toFile() + private val outRoots = java.util.List.of(outDirFile) + + override fun getOutputDir(): File = outDirFile + + override fun getOutputRoots(context: CompileContext): Collection = outRoots + + override fun isTests() = false + + override fun computeDependencies(targetRegistry: BuildTargetRegistry, outputIndex: TargetOutputIndex): Collection> { + return java.util.List.of() + } + + override fun computeRootDescriptors( + model: JpsModel, + index: ModuleExcludeIndex, + ignoredFileIndex: IgnoredFileIndex, + dataPaths: BuildDataPaths, + ): List { + return computeRootDescriptors() + } + + fun computeRootDescriptors(): List { + val roots = ArrayList(sources.size) + for (sourceRoot in sources) { + roots.add(JavaSourceRootDescriptor.createJavaSourceRootDescriptor(sourceRoot, this)) + } + return roots + } + + @ApiStatus.Internal + override fun computeConfigurationDigest(projectDescriptor: ProjectDescriptor, hash: HashSink) { + // handled by StorageInitializer and TargetDigest + } +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt b/build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt index 1a502d385edb..950b57a570f6 100644 --- a/build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt +++ b/build/jvm-rules/src/jps-builder/impl/BuildTaskLauncher.kt @@ -1,3 +1,4 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("UnstableApiUsage") package org.jetbrains.bazel.jvm.jps.impl @@ -9,19 +10,18 @@ import com.intellij.tracing.Tracer import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch -import org.jetbrains.bazel.jvm.jps.JpsProjectBuilder import org.jetbrains.jps.builders.BuildTarget import org.jetbrains.jps.builders.impl.BuildTargetChunk -import org.jetbrains.jps.incremental.* +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.CompileScope +import org.jetbrains.jps.incremental.GlobalContextKey +import org.jetbrains.jps.incremental.Utils import org.jetbrains.jps.incremental.messages.BuildMessage import org.jetbrains.jps.incremental.messages.BuildProgress import java.util.* -import java.util.concurrent.CancellationException import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicReference -private const val FLUSH_INVOCATIONS_TO_SKIP = 10 private val LOG = Logger.getInstance(JpsProjectBuilder::class.java) internal class BuildTaskLauncher( @@ -29,16 +29,12 @@ internal class BuildTaskLauncher( private val buildProgress: BuildProgress, private val builder: JpsProjectBuilder, ) { - private val exception = AtomicReference() private val tasks: MutableList - private val flushCommand: Runnable init { val span = Tracer.start("BuildTaskLauncher constructor") - val projectDescriptor = context.projectDescriptor - val targetIndex = projectDescriptor.buildTargetIndex - flushCommand = Utils.asCountedRunnable(FLUSH_INVOCATIONS_TO_SKIP, Runnable { projectDescriptor.dataManager.flush(true) }) + val targetIndex = context.projectDescriptor.buildTargetIndex val chunks = targetIndex.getSortedTargetChunks(context) tasks = ArrayList(chunks.size) val targetToTask = HashMap, BuildChunkTask>(chunks.size) @@ -88,17 +84,11 @@ internal class BuildTaskLauncher( } suspend fun buildInParallel() { + val buildSpan = Tracer.start("Parallel build") coroutineScope { queueTasks(tasks = tasks.filter { it.isReady }, isDebugLogEnabled = LOG.isDebugEnabled, coroutineScope = this) } - - val throwable = exception.get() - if (throwable is ProjectBuildException) { - throw throwable - } - else if (throwable != null) { - throw ProjectBuildException(throwable) - } + buildSpan.complete() } private fun queueTasks(tasks: List, isDebugLogEnabled: Boolean, coroutineScope: CoroutineScope) { @@ -122,26 +112,17 @@ internal class BuildTaskLauncher( ) { try { try { - if (exception.get() == null) { - val isAffectedSpan = Tracer.start("isAffected") - val affected = isBuildChunkAffected(scope = context.scope, chunk = task.chunk) - isAffectedSpan.complete() - if (affected) { - builder.buildTargetsChunk(context = chunkLocalContext, chunk = task.chunk, buildProgress = buildProgress) - } + val isAffectedSpan = Tracer.start("isAffected") + val affected = isBuildChunkAffected(scope = context.scope, chunk = task.chunk) + isAffectedSpan.complete() + if (affected) { + builder.buildTargetChunk(context = chunkLocalContext, chunk = task.chunk, buildProgress = buildProgress) } } finally { context.projectDescriptor.dataManager.closeSourceToOutputStorages(task.chunk) - flushCommand.run() } } - catch (e: CancellationException) { - throw e - } - catch (e: Throwable) { - exception.compareAndSet(null, e) - } finally { if (isDebugLogEnabled) { LOG.debug("Finished compilation of ${task.chunk}") @@ -158,8 +139,10 @@ internal class BuildTaskLauncher( internal class BuildChunkTask(@JvmField val chunk: BuildTargetChunk) { private val notBuildDependenciesCount = AtomicInteger(0) private val notBuiltDependencies = HashSet() + @JvmField val tasksDependsOnThis = ArrayList() + @JvmField var index = 0 diff --git a/build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt b/build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt new file mode 100644 index 000000000000..9e4cb7e511c4 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/impl/JpsModuleBuilder.kt @@ -0,0 +1,882 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceJavaStaticMethodWithKotlinAnalog", "UnstableApiUsage", "ReplaceGetOrSet", "HardCodedStringLiteral") + +package org.jetbrains.bazel.jvm.jps.impl + +import com.intellij.openapi.util.text.Formats +import com.intellij.tracing.Tracer +import com.intellij.util.containers.CollectionFactory +import com.intellij.util.containers.FileHashStrategy +import it.unimi.dsi.fastutil.objects.Object2ObjectMaps +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenCustomHashMap +import org.jetbrains.bazel.jvm.jps.ConsoleMessageHandler +import org.jetbrains.jps.ModuleChunk +import org.jetbrains.jps.builders.BuildRootDescriptor +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.FileProcessor +import org.jetbrains.jps.builders.ModuleBasedTarget +import org.jetbrains.jps.builders.impl.BuildOutputConsumerImpl +import org.jetbrains.jps.builders.impl.BuildTargetChunk +import org.jetbrains.jps.builders.impl.DirtyFilesHolderBase +import org.jetbrains.jps.builders.java.JavaBuilderUtil +import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType +import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor +import org.jetbrains.jps.builders.storage.BuildDataCorruptedException +import org.jetbrains.jps.builders.storage.SourceToOutputMapping +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.incremental.* +import org.jetbrains.jps.incremental.ModuleLevelBuilder.OutputConsumer +import org.jetbrains.jps.incremental.fs.CompilationRound +import org.jetbrains.jps.incremental.messages.* +import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration +import java.io.File +import java.io.IOException +import java.nio.file.FileSystem +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Path +import java.util.* +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong +import java.util.function.Function +import java.util.function.Predicate +import kotlin.concurrent.Volatile + +private val TARGET_WITH_CLEARED_OUTPUT = GlobalContextKey.create>>("_targets_with_cleared_output_") + +private const val CLASSPATH_INDEX_FILE_NAME = "classpath.index" + +// CLASSPATH_INDEX_FILE_NAME cannot be used because IDEA on run creates CLASSPATH_INDEX_FILE_NAME only if some module class is loaded, +// so, not possible to distinguish case +// "classpath.index doesn't exist because deleted on module file change" vs. "classpath.index doesn't exist because was not created" +private const val UNMODIFIED_MARK_FILE_NAME = ".unmodified" + +internal class JpsProjectBuilder( + private val builderRegistry: BuilderRegistry, + private val messageHandler: ConsoleMessageHandler, + private val isCleanBuild: Boolean, +) { + private val totalModuleLevelBuilderCount = builderRegistry.moduleLevelBuilderCount + private val elapsedTimeNanosByBuilder = ConcurrentHashMap() + private val numberOfSourcesProcessedByBuilder = ConcurrentHashMap() + + suspend fun build(context: CompileContextImpl) { + try { + val buildSpan = Tracer.start("IncProjectBuilder.runBuild") + runBuild(context) + buildSpan.complete() + } + catch (e: StopBuildException) { + // some builder decided to stop the build - report optional progress message if any + e.message?.takeIf { it.isNotEmpty() }?.let { + messageHandler.processMessage(ProgressMessage(it)) + } + } + catch (e: BuildDataCorruptedException) { + messageHandler.warn("Internal caches are corrupted or have outdated format, forcing project rebuild: $e") + throw RebuildRequestedException(e) + } + catch (e: ProjectBuildException) { + val cause = e.cause + if (cause is IOException || cause is BuildDataCorruptedException || (cause is RuntimeException && cause.cause is IOException)) { + messageHandler.warn("Internal caches are corrupted or have outdated format, forcing project rebuild: $e") + throw RebuildRequestedException(cause) + } + else { + // should stop the build with error + throw e + } + } + } + + private suspend fun runBuild(context: CompileContextImpl) { + context.setDone(0.0f) + + context.addBuildListener(ChainedTargetsBuildListener(context)) + + // deletes class loader classpath index files for changed output roots + context.addBuildListener(object : BuildListener { + override fun filesGenerated(event: FileGeneratedEvent) { + val paths = event.paths + val fs = FileSystems.getDefault() + if (paths.size == 1) { + deleteFiles(paths.iterator().next().first, fs) + return + } + + val outputs = HashSet() + for (pair in paths) { + val root = pair.getFirst() + if (outputs.add(root)) { + deleteFiles(root, fs) + } + } + } + + private fun deleteFiles(rootPath: String, fs: FileSystem) { + val root = fs.getPath(rootPath) + try { + Files.deleteIfExists(root.resolve(CLASSPATH_INDEX_FILE_NAME)) + Files.deleteIfExists(root.resolve(UNMODIFIED_MARK_FILE_NAME)) + } + catch (_: IOException) { + } + } + }) + require(builderRegistry.targetBuilders.isEmpty()) + val allModuleLevelBuildersBuildStartedSpan = Tracer.start("All ModuleLevelBuilder.buildStarted") + for (builder in builderRegistry.moduleLevelBuilders) { + builder.buildStarted(context) + } + allModuleLevelBuildersBuildStartedSpan.complete() + + var buildProgress: BuildProgress? = null + try { + val projectDescriptor = context.projectDescriptor + val sortedTargetChunks = projectDescriptor.buildTargetIndex.getSortedTargetChunks(context) + buildProgress = BuildProgress( + projectDescriptor.dataManager, + projectDescriptor.buildTargetIndex, + sortedTargetChunks, + Predicate { isBuildChunkAffected(scope = context.scope, chunk = it) } + ) + + require(builderRegistry.beforeTasks.isEmpty()) + + val checkingSourcesSpan = Tracer.start("Building targets") + BuildTaskLauncher(context = context, buildProgress = buildProgress, builder = this).buildInParallel() + checkingSourcesSpan.complete() + + require(builderRegistry.afterTasks.isEmpty()) + sendElapsedTimeMessages(context) + } + finally { + if (buildProgress != null) { + buildProgress.updateExpectedAverageTime() + if (isCleanBuild && !Utils.errorsDetected(context) && !context.cancelStatus.isCanceled) { + context.projectDescriptor.dataManager.targetStateManager.setLastSuccessfulRebuildDuration(buildProgress.absoluteBuildTime) + } + } + + for (builder in builderRegistry.moduleLevelBuilders) { + builder.buildFinished(context) + } + } + } + + private fun sendElapsedTimeMessages(context: CompileContext) { + elapsedTimeNanosByBuilder.entries + .stream() + .map { entry -> + val processedSourcesRef = numberOfSourcesProcessedByBuilder.get(entry!!.key) + val processedSources = processedSourcesRef?.get() ?: 0 + BuilderStatisticsMessage(entry.key.presentableName, processedSources, entry.value.get() / 1000000) + } + .sorted(Comparator.comparing(Function { obj: BuilderStatisticsMessage? -> obj!!.builderName })) + .forEach { buildMessage: BuilderStatisticsMessage? -> context.processMessage(buildMessage) } + } + + private fun runBuildersForChunk(context: CompileContext, chunk: BuildTargetChunk, buildProgress: BuildProgress): Boolean { + val targets: Set> = chunk.targets + if (targets.size > 1) { + val moduleTargets = LinkedHashSet() + for (target in targets) { + if (target is ModuleBuildTarget) { + moduleTargets.add(target) + } + else { + val targetsString = targets.joinToString(separator = ", ") { it.presentableName } + val message = "Cannot build \"${target.presentableName}\" because it is included into a circular dependency ($targetsString))" + context.processMessage(CompilerMessage("", BuildMessage.Kind.ERROR, message)) + return false + } + } + + return runModuleLevelBuilders(ModuleInfoAwareCompileContextCopy(context, moduleTargets), ModuleChunk(moduleTargets), buildProgress) + } + + val target = targets.iterator().next() + if (target is ModuleBuildTarget) { + @Suppress("RemoveRedundantQualifierName") + val mbt = java.util.Set.of(target) + return runModuleLevelBuilders(ModuleInfoAwareCompileContextCopy(context, mbt), ModuleChunk(mbt), buildProgress) + } + + @Suppress("UNCHECKED_CAST") + completeRecompiledSourcesSet(context, targets as Collection>) + + /* + In general, the set of files corresponding to changed source file may be different + Need this, for example, to keep up with case changes in file names for case-insensitive OSes: + deleting the output before copying is the only way to ensure the case of the output file's name is exactly the same as source file's case + */ + cleanOldOutputs(context, target) + + val builders = BuilderRegistry.getInstance().targetBuilders + var builderCount = 0 + for (builder in builders) { + buildTarget(target, context, builder) + builderCount++ + buildProgress.updateProgress(target, (builderCount.toDouble()) / builders.size, context) + } + return true + } + + private fun processDeletedPaths(context: CompileContext, targets: Set>): Boolean { + var doneSomething = false + // cleanup outputs + val targetToRemovedSources = HashMap, MutableCollection>() + + val dirsToDelete = HashSet() + for (target in targets) { + val deletedPaths = context.projectDescriptor.fsState.getAndClearDeletedPaths(target) + if (deletedPaths.isEmpty()) { + continue + } + + targetToRemovedSources.put(target, deletedPaths) + if (isTargetOutputCleared(context, target)) { + continue + } + + val dataManager = context.projectDescriptor.dataManager + val buildTargetId = dataManager.targetStateManager.getBuildTargetId(target) + val shouldPruneEmptyDirs = target is ModuleBasedTarget<*> + val sourceToOutputStorage = dataManager.getSourceToOutputMap(target) + val logger = context.loggingManager.projectBuilderLogger + // actually delete outputs associated with removed paths + for (deletedSource in deletedPaths.sorted()) { + // deleting outputs corresponding to a non-existing source + val outputs = sourceToOutputStorage.getOutputs(deletedSource) + if (outputs != null && !outputs.isEmpty()) { + val deletedOutputPaths = ArrayList() + val outputToSourceRegistry = dataManager.outputToTargetMapping + for (output in outputToSourceRegistry.removeTargetAndGetSafeToDeleteOutputs(outputs, buildTargetId, sourceToOutputStorage)) { + val deleted = BuildOperations.deleteRecursivelyAndCollectDeleted(Path.of(output), deletedOutputPaths, if (shouldPruneEmptyDirs) dirsToDelete else null) + if (deleted) { + doneSomething = true + } + } + if (!deletedOutputPaths.isEmpty()) { + if (logger.isEnabled) { + logger.logDeletedFiles(deletedOutputPaths) + } + context.processMessage(FileDeletedEvent(deletedOutputPaths)) + } + } + + if (target is ModuleBuildTarget) { + // check if the deleted source was associated with a form + val sourceToFormMap = dataManager.getSourceToFormMap(target) + val boundForms = sourceToFormMap.getOutputs(deletedSource) + if (boundForms != null) { + for (formPath in boundForms) { + val formFile = Path.of(formPath) + if (Files.exists(formFile)) { + FSOperations.markDirty(context, CompilationRound.CURRENT, formFile.toFile()) + } + } + sourceToFormMap.remove(deletedSource) + } + } + } + } + + if (!targetToRemovedSources.isEmpty()) { + val existing = Utils.REMOVED_SOURCES_KEY.get(context) + if (existing != null) { + for (entry in existing.entries) { + val paths = targetToRemovedSources.get(entry.key) + if (paths == null) { + targetToRemovedSources.put(entry.key, entry.value) + } + else { + paths.addAll(entry.value) + } + } + } + Utils.REMOVED_SOURCES_KEY.set(context, targetToRemovedSources) + } + + FSOperations.pruneEmptyDirs(context, dirsToDelete) + return doneSomething + } + + private fun > buildTarget(target: T, context: CompileContext, builder: TargetBuilder<*, *>) { + if (builder.getTargetTypes().contains(target.getTargetType())) { + val holder = object : DirtyFilesHolderBase(context) { + override fun processDirtyFiles(processor: FileProcessor) { + context.projectDescriptor.fsState.processFilesToRecompile(context, target, processor) + } + } + val outputConsumer = BuildOutputConsumerImpl(target, context) + val start = System.nanoTime() + @Suppress("UNCHECKED_CAST") + (builder as TargetBuilder).build(target, holder, outputConsumer, context) + storeBuilderStatistics(builder, System.nanoTime() - start, outputConsumer.numberOfProcessedSources) + outputConsumer.fireFileGeneratedEvent() + context.checkCanceled() + } + } + + // return true if changed something, false otherwise + private fun runModuleLevelBuilders(context: CompileContext, chunk: ModuleChunk, buildProgress: BuildProgress): Boolean { + for (category in BuilderCategory.entries) { + for (builder in builderRegistry.getBuilders(category)) { + builder.chunkBuildStarted(context, chunk) + } + } + + completeRecompiledSourcesSet(context, chunk.targets) + + var doneSomething = false + var rebuildFromScratchRequested = false + var nextPassRequired: Boolean + val outputConsumer = ChunkBuildOutputConsumerImpl(context) + try { + val fsState = context.projectDescriptor.fsState + val dirtyFilesHolder = object : DirtyFilesHolderBase(context) { + override fun processDirtyFiles(processor: FileProcessor) { + for (target in chunk.targets) { + fsState.processFilesToRecompile(context, target, processor) + } + } + } + + do { + nextPassRequired = false + fsState.beforeNextRoundStart(context, chunk) + + if (!JavaBuilderUtil.isForcedRecompilationAllJavaModules(context.scope)) { + val cleanedSources = BuildOperations.cleanOutputsCorrespondingToChangedFiles(context, dirtyFilesHolder) + for (entry in cleanedSources.entries) { + val files = entry.value.keys + if (files.isEmpty()) { + continue + } + + val mapping = context.projectDescriptor.dataManager.getSourceToOutputMap(entry.key) + for (srcFile in files) { + val outputs = entry.value.get(srcFile)!! + mapping.setOutputs(srcFile.path, outputs) + if (!outputs.isEmpty()) { + messageHandler.info("Some outputs were not removed for ${srcFile.path} source file: $outputs") + } + } + } + } + + try { + var buildersPassed = 0 + BUILDER_CATEGORY_LOOP@ for (category in BuilderCategory.entries) { + val builders = builderRegistry.getBuilders(category) + if (category == BuilderCategory.CLASS_POST_PROCESSOR) { + // ensure changes from instruments are visible to class post-processors + saveInstrumentedClasses(outputConsumer) + } + if (builders.isEmpty()) { + continue + } + + try { + for (builder in builders) { + outputConsumer.setCurrentBuilderName(builder.presentableName) + processDeletedPaths(context, chunk.targets) + val start = System.nanoTime() + val processedSourcesBefore = outputConsumer.getNumberOfProcessedSources() + val buildResult = builder.build(context, chunk, dirtyFilesHolder, outputConsumer) + storeBuilderStatistics( + builder = builder, + elapsedTime = System.nanoTime() - start, + processedFiles = outputConsumer.getNumberOfProcessedSources() - processedSourcesBefore, + ) + + doneSomething = doneSomething or (buildResult != ModuleLevelBuilder.ExitCode.NOTHING_DONE) + + if (buildResult == ModuleLevelBuilder.ExitCode.ABORT) { + throw StopBuildException("Builder ${builder.presentableName} requested build stop") + } + context.checkCanceled() + if (buildResult == ModuleLevelBuilder.ExitCode.ADDITIONAL_PASS_REQUIRED) { + nextPassRequired = true + } + else if (buildResult == ModuleLevelBuilder.ExitCode.CHUNK_REBUILD_REQUIRED) { + if (!rebuildFromScratchRequested && !JavaBuilderUtil.isForcedRecompilationAllJavaModules(context.scope)) { + notifyChunkRebuildRequested(context, chunk, builder) + // allow rebuild from scratch only once per chunk + rebuildFromScratchRequested = true + try { + // forcibly mark all files in the chunk dirty + fsState.clearContextRoundData(context) + FSOperations.markDirty(context, CompilationRound.NEXT, chunk, null) + // reverting to the beginning + nextPassRequired = true + outputConsumer.clear() + break@BUILDER_CATEGORY_LOOP + } + catch (e: Exception) { + throw ProjectBuildException(e) + } + } + else { + messageHandler.debug("Builder ${builder.presentableName} requested second chunk rebuild") + } + } + + buildersPassed++ + for (target in chunk.targets) { + buildProgress.updateProgress(target, (buildersPassed.toDouble()) / totalModuleLevelBuilderCount, context) + } + } + } + finally { + outputConsumer.setCurrentBuilderName(null) + } + } + } + finally { + val moreToCompile = JavaBuilderUtil.updateMappingsOnRoundCompletion(context, dirtyFilesHolder, chunk) + if (moreToCompile) { + nextPassRequired = true + } + JavaBuilderUtil.clearDataOnRoundCompletion(context) + } + } while (nextPassRequired) + } + finally { + saveInstrumentedClasses(outputConsumer) + outputConsumer.fireFileGeneratedEvents() + outputConsumer.clear() + for (category in BuilderCategory.entries) { + for (builder in builderRegistry.getBuilders(category)) { + builder.chunkBuildFinished(context, chunk) + } + } + if (Utils.errorsDetected(context)) { + context.processMessage(CompilerMessage("", BuildMessage.Kind.JPS_INFO, "Errors occurred while compiling module ${chunk.presentableShortName}")) + } + } + + return doneSomething + } + + private fun ensureFsStateInitialized(context: CompileContext, target: BuildTarget<*>) { + val fsState = context.projectDescriptor.fsState + if (isCleanBuild) { + val targetRoots = (context.projectDescriptor.buildRootIndex as BazelBuildRootIndex).descriptors + fsState.getDelta(target).clearRecompile(targetRoots) + for (rootDescriptor in targetRoots) { + // if it is a full project rebuild, all storages are already completely cleared; + // so passing null as stampStorage because there is no need to access the storage to clear non-existing data + fsState.markDirty( + /* context = */ context, + /* round = */ CompilationRound.CURRENT, + /* file = */ rootDescriptor.file, + /* buildRootDescriptor = */ rootDescriptor, + /* stampStorage = */ null, + /* saveEventStamp = */ false, + ) + } + FSOperations.addCompletelyMarkedDirtyTarget(context, target) + fsState.markInitialScanPerformed(target) + } + else if (!fsState.isInitialScanPerformed(target)) { + BuildOperations.initTargetFSState(context, target, false) + } + } + + internal fun buildTargetChunk(context: CompileContext, chunk: BuildTargetChunk, buildProgress: BuildProgress) { + val buildSpan = Tracer.start { "Building ${chunk.presentableName}" } + val fsState = context.projectDescriptor.fsState + var doneSomething: Boolean + try { + context.setCompilationStartStamp(chunk.targets, System.currentTimeMillis()) + + sendBuildingTargetMessages(chunk.targets, BuildingTargetProgressMessage.Event.STARTED) + Utils.ERRORS_DETECTED_KEY.set(context, false) + + for (target in chunk.targets) { + ensureFsStateInitialized(context = context, target = target) + } + + doneSomething = processDeletedPaths(context, chunk.targets) + + fsState.beforeChunkBuildStart(context, chunk) + + val runBuildersSpan = Tracer.start { "runBuilders " + chunk.presentableName } + doneSomething = doneSomething or runBuildersForChunk(context, chunk, buildProgress) + runBuildersSpan.complete() + + fsState.clearContextRoundData(context) + fsState.clearContextChunk(context) + + if (doneSomething) { + BuildOperations.markTargetsUpToDate(context, chunk) + } + } + catch (e: BuildDataCorruptedException) { + throw e + } + catch (e: ProjectBuildException) { + throw e + } + catch (e: Throwable) { + val message = StringBuilder() + message.append(chunk.presentableName).append(": ").append(e.javaClass.getName()) + e.message?.let { + message.append(": ").append(it) + } + throw ProjectBuildException(message.toString(), e) + } + finally { + buildProgress.onTargetChunkFinished(chunk, context) + try { + // restore deleted paths that were not processed by 'integrate' + val map = Utils.REMOVED_SOURCES_KEY.get(context) + if (map != null) { + for (entry in map.entries) { + val paths = entry.value + if (paths != null) { + for (path in paths) { + fsState.registerDeleted(context, entry.key, Path.of(path), null) + } + } + } + } + } + finally { + Utils.REMOVED_SOURCES_KEY.set(context, null) + sendBuildingTargetMessages(chunk.targets, BuildingTargetProgressMessage.Event.FINISHED) + buildSpan.complete() + } + } + } + + private fun sendBuildingTargetMessages(targets: Set>, event: BuildingTargetProgressMessage.Event) { + messageHandler.processMessage(BuildingTargetProgressMessage(targets, event)) + } + + private fun storeBuilderStatistics(builder: Builder, elapsedTime: Long, processedFiles: Int) { + elapsedTimeNanosByBuilder.computeIfAbsent(builder) { AtomicLong() }.addAndGet(elapsedTime) + numberOfSourcesProcessedByBuilder.computeIfAbsent(builder) { AtomicInteger() }.addAndGet(processedFiles) + } +} + +private class ChunkBuildOutputConsumerImpl(private val context: CompileContext) : OutputConsumer { + private val target2Consumer = HashMap, BuildOutputConsumerImpl>() + private val classes = HashMap() + private val targetToClassesMap = HashMap, MutableCollection>() + private val outputToBuilderNameMap = Object2ObjectMaps.synchronize(Object2ObjectOpenCustomHashMap(FileHashStrategy)) + + @Volatile + private var currentBuilderName: String? = null + + fun setCurrentBuilderName(builderName: String?) { + currentBuilderName = builderName + } + + override fun getTargetCompiledClasses(target: BuildTarget<*>): Collection { + return Collections.unmodifiableCollection(targetToClassesMap.get(target) ?: return emptyList()) + } + + override fun getCompiledClasses(): MutableMap = Collections.unmodifiableMap(classes) + + override fun lookupClassBytes(className: String?): BinaryContent? = classes.get(className)?.content + + override fun registerCompiledClass(target: BuildTarget<*>?, compiled: CompiledClass) { + val className = compiled.className + if (className != null) { + classes.put(className, compiled) + if (target != null) { + var classes = targetToClassesMap.get(target) + if (classes == null) { + classes = ArrayList() + targetToClassesMap.put(target, classes) + } + classes.add(compiled) + } + } + if (target != null) { + registerOutputFile(target = target, outputFile = compiled.outputFile, sourcePaths = compiled.sourceFilesPaths) + } + } + + override fun registerOutputFile(target: BuildTarget<*>, outputFile: File, sourcePaths: MutableCollection) { + val currentBuilder = currentBuilderName + if (currentBuilder != null) { + val previousBuilder = outputToBuilderNameMap.put(outputFile, currentBuilder) + if (previousBuilder != null && previousBuilder != currentBuilder) { + val source = if (sourcePaths.isEmpty()) null else sourcePaths.iterator().next() + context.processMessage(CompilerMessage( + currentBuilder, BuildMessage.Kind.ERROR, "Output file \"${outputFile}\" has already been registered by \"$previousBuilder\"", source + )) + } + } + var consumer = target2Consumer.get(target) + if (consumer == null) { + consumer = BuildOutputConsumerImpl(target, context) + target2Consumer.put(target, consumer) + } + consumer.registerOutputFile(outputFile, sourcePaths) + } + + fun fireFileGeneratedEvents() { + for (consumer in target2Consumer.values) { + consumer.fireFileGeneratedEvent() + } + } + + fun getNumberOfProcessedSources(): Int { + var total = 0 + for (consumer in target2Consumer.values) { + total += consumer.numberOfProcessedSources + } + return total + } + + fun clear() { + target2Consumer.clear() + classes.clear() + targetToClassesMap.clear() + outputToBuilderNameMap.clear() + } +} + +private class ChainedTargetsBuildListener(private val context: CompileContextImpl) : BuildListener { + override fun filesGenerated(event: FileGeneratedEvent) { + val projectDescriptor = context.projectDescriptor + val fsState = projectDescriptor.fsState + for (pair in event.paths) { + val relativePath = pair.getSecond() + val file = if (relativePath == ".") File(pair.getFirst()) else File(pair.getFirst(), relativePath) + for (buildRootDescriptor in projectDescriptor.buildRootIndex.findAllParentDescriptors(file, context)) { + val target = buildRootDescriptor.target + if (event.sourceTarget != target) { + fsState.markDirty(context, file, buildRootDescriptor, projectDescriptor.dataManager.getFileStampStorage(target), false) + } + } + } + } + + override fun filesDeleted(event: FileDeletedEvent) { + val fsState = context.projectDescriptor.fsState + val rootIndex = context.projectDescriptor.buildRootIndex as BazelBuildRootIndex + for (path in event.filePaths) { + val file = Path.of(path) + val rootDescriptor = rootIndex.fileToDescriptors.get(file) ?: continue + fsState.registerDeleted(context, rootDescriptor.target, file) + } + } +} + +private fun calculateEstimatedBuildTime( + projectDescriptor: ProjectDescriptor, + target: ModuleBuildTarget, + messageHandler: ConsoleMessageHandler, +): Long { + // compute estimated times for dirty targets + var affectedTargets = 0 + val avgTimeToBuild = projectDescriptor.dataManager.targetStateManager.getAverageBuildTime(target.targetType) + val estimatedBuildTime = if (avgTimeToBuild > 0) { + affectedTargets = 1 + // 1. in general case, this time should include dependency analysis and cache update times + // 2. need to check isAffected() since some targets (like artifacts) may be unaffected even for rebuild + avgTimeToBuild + } + else { + 0L + } + messageHandler.info("Affected build targets count: $affectedTargets") + return estimatedBuildTime +} + +internal fun reportRebuiltModules(context: CompileContextImpl) { + val modules = BuildTargetConfiguration.MODULES_WITH_TARGET_CONFIG_CHANGED_KEY.get(context) + if (modules.isNullOrEmpty()) { + return + } + + val text = "${modules.joinToString { m -> "'" + m.name + "'" }} was fully rebuilt due to project configuration changes" + context.processMessage(CompilerMessage("", BuildMessage.Kind.INFO, text)) +} + +internal fun reportUnprocessedChanges(context: CompileContextImpl, moduleTarget: ModuleBuildTarget) { + if (context.projectDescriptor.fsState.hasUnprocessedChanges(context, moduleTarget)) { + context.processMessage(UnprocessedFSChangesNotification()) + } +} + +private fun isTargetOutputCleared(context: CompileContext, target: BuildTarget<*>?): Boolean { + synchronized(TARGET_WITH_CLEARED_OUTPUT) { + val data = context.getUserData(TARGET_WITH_CLEARED_OUTPUT) + return data != null && data.contains(target) + } +} + +private class ModuleInfoAwareCompileContextCopy( + private val context: CompileContext, + private val moduleTargets: Collection, +) : CompileContext by context { + override fun processMessage(message: BuildMessage) { + if (message is CompilerMessage) { + for (target in moduleTargets) { + message.addModuleName(target.module.name) + } + } + context.processMessage(message) + } + + @Suppress("OVERRIDE_DEPRECATION") + override fun isProjectRebuild() = false + + override fun isCanceled(): Boolean = context.isCanceled +} + +/** + * if an output file is generated from multiple sources, make sure all of them are added for recompilation + */ +@Suppress("SpellCheckingInspection") +private fun , R : BuildRootDescriptor> completeRecompiledSourcesSet(context: CompileContext, targets: Collection) { + val scope = context.scope + for (target in targets) { + if (scope.isBuildForced(target)) { + // assuming build is either forced for all targets in a chunk or for none of them + return + } + } + + val projectDescriptor = context.projectDescriptor + val affectedOutputs = CollectionFactory.createFilePathSet() + val affectedSources = CollectionFactory.createFilePathSet() + + val mappings = ArrayList() + for (target in targets) { + projectDescriptor.fsState.processFilesToRecompile(context, target, object : FileProcessor { + private var srcToOut: SourceToOutputMapping? = null + + override fun apply(target: T, file: File, root: R): Boolean { + val src = file.invariantSeparatorsPath + if (!affectedSources.add(src)) { + return true + } + + // lazy init + var srcToOut = this.srcToOut + if (srcToOut == null) { + srcToOut = projectDescriptor.dataManager.getSourceToOutputMap(target) + mappings.add(srcToOut) + this.srcToOut = srcToOut + } + + val outs = srcToOut.getOutputs(src) ?: return true + // Temporary hack for KTIJ-197 + // Change of only one input of *.kotlin_module files didn't trigger recompilation of all inputs in old behavior. + // Now it does. It isn't yet obvious whether it is right or wrong behavior. Let's leave old behavior for a + // while for safety and keeping kotlin incremental JPS tests green + outs.filterTo(affectedOutputs) { "kotlin_module" != it.substringAfterLast('.') } + return true + } + }) + } + + if (affectedOutputs.isEmpty()) { + return + } + + for (srcToOut in mappings) { + val cursor = srcToOut.cursor() + while (cursor.hasNext()) { + val src = cursor.next() + if (!affectedSources.contains(src)) { + for (out in cursor.outputPaths) { + if (affectedOutputs.contains(out)) { + FSOperations.markDirtyIfNotDeleted(context, CompilationRound.CURRENT, Path.of(src)) + break + } + } + } + } + } +} + +private fun cleanOldOutputs(context: CompileContext, target: BuildTarget) { + if (!context.scope.isBuildForced(target)) { + BuildOperations.cleanOutputsCorrespondingToChangedFiles(context, object : DirtyFilesHolderBase>(context) { + override fun processDirtyFiles(processor: FileProcessor>) { + context.projectDescriptor.fsState.processFilesToRecompile(context, target, processor) + } + }) + } +} + +private fun notifyChunkRebuildRequested(context: CompileContext, chunk: ModuleChunk, builder: ModuleLevelBuilder) { + var infoMessage = "Builder \"${builder.presentableName}\" requested rebuild of module chunk \"${chunk.name}\"" + var kind = BuildMessage.Kind.JPS_INFO + val scope = context.scope + for (target in chunk.targets) { + if (!scope.isWholeTargetAffected(target)) { + infoMessage += ".\n" + infoMessage += "Consider building whole project or rebuilding the module." + kind = BuildMessage.Kind.INFO + break + } + } + context.processMessage(CompilerMessage("", kind, infoMessage)) +} + +private fun saveInstrumentedClasses(outputConsumer: ChunkBuildOutputConsumerImpl) { + for (compiledClass in outputConsumer.compiledClasses.values) { + if (compiledClass.isDirty) { + compiledClass.save() + } + } +} + +internal fun checkRebuildRequired( + scope: CompileScope, + projectDescriptor: ProjectDescriptor, + moduleTarget: ModuleBuildTarget, + isDebugEnabled: Boolean, + messageHandler: ConsoleMessageHandler, +): Boolean { + val targetStateManager = projectDescriptor.dataManager.targetStateManager + val timeThreshold = targetStateManager.getLastSuccessfulRebuildDuration() * 95 / 100 // 95% of last registered clean rebuild time + if (timeThreshold <= 0) { + if (isDebugEnabled) { + messageHandler.debug("Rebuild heuristic: no stats available") + } + return false + } + + // check that this is a whole-project incremental build + // checking only JavaModuleBuildTargetType because these target types directly correspond to project modules + val type = JavaModuleBuildTargetType.PRODUCTION + if (!scope.isAllTargetsOfTypeAffected(type)) { + if (isDebugEnabled) { + messageHandler.debug("Rebuild heuristic: skipping the check because some targets are excluded from compilation scope," + + " e.g. targets of type ${type.typeId}") + } + return false + } + + // compute estimated times for dirty targets + val estimatedWorkTime = calculateEstimatedBuildTime(projectDescriptor, moduleTarget, messageHandler) + if (isDebugEnabled) { + messageHandler.debug("Rebuild heuristic: estimated build time / timeThreshold : $estimatedWorkTime / $timeThreshold") + } + + if (estimatedWorkTime < timeThreshold) { + return false + } + + val message = """ + Too many files require recompilation, forcing full rebuild. + * Estimated build duration (linear): ${Formats.formatDuration(estimatedWorkTime)} + * Last successful rebuild duration (linear): ${Formats.formatDuration(targetStateManager.getLastSuccessfulRebuildDuration())} + * Rebuild heuristic time threshold: ${Formats.formatDuration(timeThreshold)} + """.trimIndent() + messageHandler.info(message) + return true +} + diff --git a/build/jvm-rules/src/jps-builder/impl/BazelModuleExcludeIndex.kt b/build/jvm-rules/src/jps-builder/impl/NoopModuleExcludeIndex.kt similarity index 77% rename from build/jvm-rules/src/jps-builder/impl/BazelModuleExcludeIndex.kt rename to build/jvm-rules/src/jps-builder/impl/NoopModuleExcludeIndex.kt index df9637e1cf90..fca2865f65d0 100644 --- a/build/jvm-rules/src/jps-builder/impl/BazelModuleExcludeIndex.kt +++ b/build/jvm-rules/src/jps-builder/impl/NoopModuleExcludeIndex.kt @@ -1,3 +1,4 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.bazel.jvm.jps.impl import com.intellij.openapi.util.io.FileFilters @@ -7,7 +8,7 @@ import java.io.File import java.io.FileFilter import java.nio.file.Path -object BazelModuleExcludeIndex : ModuleExcludeIndex { +internal object NoopModuleExcludeIndex : ModuleExcludeIndex { override fun isExcluded(file: File) = false override fun isExcludedFromModule(file: File, module: JpsModule) = false diff --git a/build/jvm-rules/src/jps-builder/impl/load.kt b/build/jvm-rules/src/jps-builder/impl/load.kt new file mode 100644 index 000000000000..75deb7417a6c --- /dev/null +++ b/build/jvm-rules/src/jps-builder/impl/load.kt @@ -0,0 +1,84 @@ +@file:Suppress("UnstableApiUsage", "HardCodedStringLiteral", "ReplaceJavaStaticMethodWithKotlinAnalog", "ReplaceGetOrSet") + +package org.jetbrains.bazel.jvm.jps.impl + +import org.jetbrains.bazel.jvm.jps.state.loadTargetState +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.BuildTargetIndex +import org.jetbrains.jps.builders.BuildTargetRegistry.ModuleTargetSelector +import org.jetbrains.jps.builders.BuildTargetType +import org.jetbrains.jps.builders.impl.BuildDataPathsImpl +import org.jetbrains.jps.builders.impl.BuildTargetChunk +import org.jetbrains.jps.builders.logging.BuildLoggingManager +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.ModuleBuildTarget +import org.jetbrains.jps.incremental.fs.BuildFSState +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import org.jetbrains.jps.incremental.storage.BuildDataManager +import org.jetbrains.jps.incremental.storage.BuildDataVersionManager +import org.jetbrains.jps.incremental.storage.StorageManager +import org.jetbrains.jps.indices.IgnoredFileIndex +import org.jetbrains.jps.model.JpsModel +import org.jetbrains.jps.model.module.JpsModule +import java.nio.file.Path + +internal fun loadJpsProject( + storageManager: StorageManager, + dataStorageRoot: Path, + fsState: BuildFSState, + jpsModel: JpsModel, + moduleTarget: BazelModuleBuildTarget, + relativizer: PathRelativizerService, +): ProjectDescriptor { + val dataPaths = BuildDataPathsImpl(dataStorageRoot) + val dataManager = BuildDataManager.createSingleDb( + /* dataPaths = */ dataPaths, + /* targetStateManager = */ BazelBuildTargetStateManager(loadTargetState(storageManager)), + /* relativizer = */ relativizer, + /* versionManager = */ NoopBuildDataVersionManager, + /* storageManager = */ storageManager, + ) + return ProjectDescriptor( + /* model = */ jpsModel, + /* fsState = */ fsState, + /* dataManager = */ dataManager, + /* loggingManager = */ BuildLoggingManager.DEFAULT, + /* moduleExcludeIndex = */ NoopModuleExcludeIndex, + /* buildTargetIndex = */ BazelBuildTargetIndex(moduleTarget), + /* buildRootIndex = */ BazelBuildRootIndex(moduleTarget), + /* ignoredFileIndex = */ NoopIgnoredFileIndex, + ) +} + +private object NoopIgnoredFileIndex : IgnoredFileIndex { + override fun isIgnored(path: String) = false +} + +private object NoopBuildDataVersionManager : BuildDataVersionManager { + override fun versionDiffers() = false + + override fun saveVersion() { + } +} + +private class BazelBuildTargetIndex(moduleTarget: ModuleBuildTarget) : BuildTargetIndex { + private var targetChunks = java.util.List.of(BuildTargetChunk(java.util.Set.of(moduleTarget))) + private var targets = java.util.List.of(moduleTarget) + + override fun getSortedTargetChunks(context: CompileContext): List = targetChunks + + override fun isDummy(target: BuildTarget<*>) = false + + @Suppress("OVERRIDE_DEPRECATION", "removal") + override fun getDependenciesRecursively(target: BuildTarget<*>, context: CompileContext): Set> = emptySet() + + override fun getDependencies(target: BuildTarget<*>, context: CompileContext): Collection> = emptyList() + + override fun getModuleBasedTargets(module: JpsModule, selector: ModuleTargetSelector): List = targets + + @Suppress("UNCHECKED_CAST") + override fun > getAllTargets(type: BuildTargetType) = targets as List + + override fun getAllTargets(): List = targets +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/pathRelativizer.kt b/build/jvm-rules/src/jps-builder/impl/pathRelativizer.kt new file mode 100644 index 000000000000..0cd12983fa1e --- /dev/null +++ b/build/jvm-rules/src/jps-builder/impl/pathRelativizer.kt @@ -0,0 +1,69 @@ +@file:Suppress("UnstableApiUsage") + +package org.jetbrains.bazel.jvm.jps.impl + +import org.jetbrains.jps.incremental.relativizer.PathRelativizer +import org.jetbrains.jps.incremental.relativizer.PathRelativizerService +import org.jetbrains.jps.incremental.storage.PathTypeAwareRelativizer +import org.jetbrains.jps.incremental.storage.RelativePathType +import java.io.File +import java.nio.file.Path +import kotlin.io.path.invariantSeparatorsPathString + +private const val BASE_ID_PREFIX = "#" +private const val OUT_ID_PREFIX = "@" + +internal fun createPathRelativizer(baseDir: Path, classOutDir: Path): PathRelativizerService { + val baseDirPrefix = "${baseDir.invariantSeparatorsPathString}/" + // Bazel may use paths with `../` + val parentOfBaseDirPrefix = "${baseDir.parent.invariantSeparatorsPathString}/" + val outBaseDirPrefix = "${classOutDir.invariantSeparatorsPathString}/" + + val typeAwareRelativizer = object : PathTypeAwareRelativizer { + override fun toRelative(path: String, type: RelativePathType): String { + val p = path.replace(File.separatorChar, '/') + when (type) { + RelativePathType.SOURCE -> { + return when { + p.startsWith(baseDirPrefix) -> p.substring(baseDirPrefix.length) + p.startsWith(parentOfBaseDirPrefix) -> "../" + p.substring(parentOfBaseDirPrefix.length) + else -> error("Unexpected path: $p") + } + } + RelativePathType.OUTPUT -> { + require(p.startsWith(outBaseDirPrefix)) { "Unexpected path: $p" } + return p.substring(outBaseDirPrefix.length) + } + } + } + + override fun toRelative(path: Path, type: RelativePathType): String { + return toRelative(path.invariantSeparatorsPathString, type) + } + + override fun toAbsolute(path: String, type: RelativePathType): String { + return when (type) { + RelativePathType.SOURCE -> baseDirPrefix + path + RelativePathType.OUTPUT -> outBaseDirPrefix + path + } + } + } + + return PathRelativizerService(arrayOf(object : PathRelativizer { + override fun toRelativePath(path: String): String? { + return when { + path.startsWith(outBaseDirPrefix) -> OUT_ID_PREFIX + path.substring(outBaseDirPrefix.length) + path.startsWith(baseDirPrefix) -> BASE_ID_PREFIX + path.substring(baseDirPrefix.length) + else -> null + } + } + + override fun toAbsolutePath(path: String): String? { + return when { + path.startsWith(OUT_ID_PREFIX) -> outBaseDirPrefix + path.substring(OUT_ID_PREFIX.length) + path.startsWith(BASE_ID_PREFIX) -> baseDirPrefix + path.substring(BASE_ID_PREFIX.length) + else -> null + } + } + }), typeAwareRelativizer) +} diff --git a/build/jvm-rules/src/jps-builder/logger.kt b/build/jvm-rules/src/jps-builder/logger.kt new file mode 100644 index 000000000000..cb22fcf7e3ec --- /dev/null +++ b/build/jvm-rules/src/jps-builder/logger.kt @@ -0,0 +1,66 @@ +@file:Suppress("UnstableApiUsage") + +package org.jetbrains.bazel.jvm.jps + +import com.intellij.openapi.diagnostic.DefaultLogger +import com.intellij.openapi.diagnostic.LogLevel +import com.intellij.openapi.diagnostic.Logger + +// write to System.err as Bazel expects for worker log +internal class BazelLogger(private val category: String) : Logger() { + private var level = LogLevel.DEBUG + + override fun isDebugEnabled(): Boolean = level >= LogLevel.DEBUG + + override fun isTraceEnabled(): Boolean = level >= LogLevel.TRACE + + override fun trace(message: String) { + if (isTraceEnabled()) { + System.err.println("TRACE[$category]: $message") + } + } + + override fun trace(t: Throwable?) { + if (t != null && isTraceEnabled()) { + System.err.println("TRACE[$category]: ") + t.printStackTrace(System.err) + } + } + + override fun debug(message: String, t: Throwable?) { + doPrint(LogLevel.DEBUG, message, t) + } + + private fun doPrint(level: LogLevel, message: String, t: Throwable?) { + if (this.level >= level) { + return + } + + var text = "${level.name}[$category]: $message" + if (t != null) { + ensureNotControlFlow(t) + text += " " + t.stackTraceToString() + } + System.err.println(text) + } + + override fun info(message: String, t: Throwable?) { + doPrint(LogLevel.INFO, message, t) + } + + override fun warn(message: String, t: Throwable?) { + doPrint(LogLevel.WARNING, message, t) + } + + override fun error(message: String, t: Throwable?, vararg details: String) { + var t = t + t = ensureNotControlFlow(t) + System.err.println("ERROR: " + message + DefaultLogger.detailsToString(*details) + DefaultLogger.attachmentsToString(t)) + t?.printStackTrace(System.err) + throw AssertionError(message, t) + } + + override fun setLevel(level: LogLevel) { + this.level = level + } +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/packager/BUILD.bazel b/build/jvm-rules/src/jps-builder/packager/BUILD.bazel new file mode 100644 index 000000000000..739bd476b56a --- /dev/null +++ b/build/jvm-rules/src/jps-builder/packager/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_java//java:defs.bzl", "java_binary") +load("@rules_jvm//:jvm.bzl", "jvm_import") +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") + +# separate to avoid adding dependency on proguard and zip to JPS worker impl + +kt_jvm_library( + name = "packager", + srcs = glob(["*.kt"]), + kotlinc_opts = "//:rules_jvm_bootstrap_kotlinc_options", + deps = [ + "@lib//:kotlin-stdlib", + "//:kotlinx-coroutines-core", + "@lib//:asm", + "//zip:build-zip", + "//src/jps-builder:jps-standalone", + "//:kotlin-metadata", + ], + visibility = ["//visibility:public"], +) diff --git a/build/jvm-rules/src/jps-builder/packager/ConsoleMessageHandler.kt b/build/jvm-rules/src/jps-builder/packager/ConsoleMessageHandler.kt new file mode 100644 index 000000000000..e98f30d52d8f --- /dev/null +++ b/build/jvm-rules/src/jps-builder/packager/ConsoleMessageHandler.kt @@ -0,0 +1,62 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("HardCodedStringLiteral") + +package org.jetbrains.bazel.jvm.jps + +import org.jetbrains.jps.incremental.MessageHandler +import org.jetbrains.jps.incremental.messages.BuildMessage +import org.jetbrains.jps.incremental.messages.CompilerMessage +import java.io.Writer + +class ConsoleMessageHandler(@PublishedApi @JvmField internal val out: Writer) : MessageHandler { + private var hasErrors = false + + fun warn(message: String) { + out.appendLine("WARN: $message") + } + + fun error(message: String) { + out.appendLine("ERROR: $message") + } + + fun info(message: String) { + out.appendLine("INFO: $message") + } + + fun debug(message: String) { + out.appendLine("DEBUG: $message") + } + + inline fun measureTime(label: String, block: () -> Unit) { + val duration = kotlin.time.measureTime(block) + out.appendLine("TIME: $label: $duration") + } + + override fun processMessage(message: BuildMessage) { + val messageText = when (message) { + is CompilerMessage -> { + when { + message.sourcePath == null -> message.messageText + message.line < 0 -> message.sourcePath + ": " + message.messageText + else -> message.sourcePath + "(" + message.line + ":" + message.column + "): " + message.messageText + } + } + + else -> message.messageText + } + + if (messageText.isEmpty()) { + return + } + + if (message.kind == BuildMessage.Kind.ERROR) { + out.appendLine("Error: $messageText") + hasErrors = true + } + else if (message.kind !== BuildMessage.Kind.PROGRESS || !messageText.startsWith("Compiled") && !messageText.startsWith("Copying")) { + out.appendLine(messageText) + } + } + + fun hasErrors(): Boolean = hasErrors +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/packager/KotlinAnnotationVisitor.kt b/build/jvm-rules/src/jps-builder/packager/KotlinAnnotationVisitor.kt new file mode 100644 index 000000000000..15171b8a3e7b --- /dev/null +++ b/build/jvm-rules/src/jps-builder/packager/KotlinAnnotationVisitor.kt @@ -0,0 +1,95 @@ +package org.jetbrains.bazel.jvm.jps + +import org.jetbrains.org.objectweb.asm.AnnotationVisitor +import org.jetbrains.org.objectweb.asm.Opcodes + +private const val KIND_FIELD_NAME = "k" +private const val METADATA_EXTRA_INT_FIELD_NAME = "xi" +private const val METADATA_VERSION_FIELD_NAME = "mv" +private const val METADATA_DATA_FIELD_NAME = "d1" +private const val METADATA_STRINGS_FIELD_NAME = "d2" +private const val METADATA_EXTRA_STRING_FIELD_NAME = "xs" +private const val METADATA_PACKAGE_NAME_FIELD_NAME = "pn" + +internal class KotlinAnnotationVisitor(private val resultConsumer: (Metadata) -> Unit) : AnnotationVisitor(Opcodes.API_VERSION) { + private var kind: Int = 1 + private var metadataVersion: IntArray = intArrayOf() + private var data1: MutableList = mutableListOf() + private var data2: MutableList = mutableListOf() + private var extraString: String = "" + private var packageName: String = "" + private var extraInt: Int = 0 + + override fun visit(name: String, value: Any?) { + when (name) { + KIND_FIELD_NAME -> kind = value as Int + METADATA_EXTRA_INT_FIELD_NAME -> extraInt = value as Int + METADATA_VERSION_FIELD_NAME -> metadataVersion = value as IntArray + METADATA_EXTRA_STRING_FIELD_NAME -> extraString = value as String + METADATA_PACKAGE_NAME_FIELD_NAME -> packageName = value as String + } + } + + override fun visitArray(name: String): AnnotationVisitor? { + val destination = when (name) { + METADATA_DATA_FIELD_NAME -> data1 + METADATA_STRINGS_FIELD_NAME -> data2 + else -> return null + } + return object : AnnotationVisitor(Opcodes.API_VERSION) { + override fun visit(name: String?, value: Any?) { + destination.add(value as String) + } + } + } + + override fun visitEnd() { + resultConsumer(Metadata( + kind = kind, + metadataVersion = metadataVersion, + data1 = data1.toTypedArray(), + data2 = data2.toTypedArray(), + extraString = extraString, + packageName = packageName, + extraInt = extraInt, + )) + } +} + +/** + * Serialize a KotlinClassHeader to an existing Kotlin Metadata annotation visitor. + */ +internal fun visitKotlinMetadata(annotationVisitor: AnnotationVisitor, header: Metadata) { + annotationVisitor.visit(KIND_FIELD_NAME, header.kind) + annotationVisitor.visit(METADATA_VERSION_FIELD_NAME, header.metadataVersion) + + if (header.data1.isNotEmpty()) { + val arrayVisitor = annotationVisitor.visitArray(METADATA_DATA_FIELD_NAME) + for (v in header.data1) { + arrayVisitor.visit(null, v) + } + arrayVisitor.visitEnd() + } + + if (header.data2.isNotEmpty()) { + val arrayVisitor = annotationVisitor.visitArray(METADATA_STRINGS_FIELD_NAME) + for (v in header.data2) { + arrayVisitor.visit(null, v) + } + arrayVisitor.visitEnd() + } + + if (header.extraString.isNotEmpty()) { + annotationVisitor.visit(METADATA_EXTRA_STRING_FIELD_NAME, header.extraString) + } + + if (header.packageName.isNotEmpty()) { + annotationVisitor.visit(METADATA_PACKAGE_NAME_FIELD_NAME, header.packageName) + } + + if (header.extraInt != 0) { + annotationVisitor.visit(METADATA_EXTRA_INT_FIELD_NAME, header.extraInt) + } + + annotationVisitor.visitEnd() +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/packager/abi.kt b/build/jvm-rules/src/jps-builder/packager/abi.kt new file mode 100644 index 000000000000..7ae94a685d65 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/packager/abi.kt @@ -0,0 +1,268 @@ +package org.jetbrains.bazel.jvm.jps + +import org.jetbrains.org.objectweb.asm.* +import org.jetbrains.org.objectweb.asm.tree.AnnotationNode +import org.jetbrains.org.objectweb.asm.tree.FieldNode +import org.jetbrains.org.objectweb.asm.tree.MethodNode +import kotlin.metadata.* +import kotlin.metadata.jvm.KotlinClassMetadata +import kotlin.metadata.jvm.getterSignature +import kotlin.metadata.jvm.localDelegatedProperties +import kotlin.metadata.jvm.signature + +/** + * ClassVisitor that strips non-public methods/fields and Kotlin `internal` methods. + */ +internal class AbiClassVisitor( + classVisitor: ClassVisitor, + private val classesToBeDeleted: MutableSet, +) : ClassVisitor(Opcodes.API_VERSION, classVisitor) { + // tracks if this class has any public API members + var isApiClass: Boolean = false + private set + + private val classAnnotations = mutableListOf>() + private val fields = mutableListOf() + private val methods = mutableListOf() + private var kotlinMetadata: Pair? = null + + override fun visitAnnotation(descriptor: String, visible: Boolean): AnnotationVisitor? { + // parse @Metadata for Kotlin-specific visibility information + @Suppress("SpellCheckingInspection") + if (descriptor == "Lkotlin/Metadata;") { + return KotlinAnnotationVisitor { + kotlinMetadata = descriptor to KotlinClassMetadata.readStrict(it) + } + } + else { + val annotationNode = AnnotationNode(api, descriptor) + classAnnotations.add(annotationNode to visible) + return annotationNode + } + } + + override fun visit(version: Int, access: Int, name: String, signature: String?, superName: String?, interfaces: Array?) { + if (access and Opcodes.ACC_PUBLIC == 0) { + classesToBeDeleted.add(name) + } + else { + isApiClass = true + super.visit(version, access, name, signature, superName, interfaces) + } + } + + override fun visitField( + access: Int, + name: String?, + descriptor: String?, + signature: String?, + value: Any?, + ): FieldVisitor? { + if (access and Opcodes.ACC_PUBLIC == 0) { + return null + } + + val fieldNode = FieldNode(api, access, name, descriptor, signature, value) + fields.add(fieldNode) + return fieldNode + } + + override fun visitMethod( + access: Int, + name: String, + descriptor: String?, + signature: String?, + exceptions: Array?, + ): MethodVisitor? { + // retain public methods and exclude Kotlin `internal` methods + val isPublic = access and Opcodes.ACC_PUBLIC != 0 + val isInternal = isKotlinInternal(name) + if (!isPublic || isInternal) { + return null + } + + val methodNode = MethodNode(api, access, name, descriptor, signature, exceptions) + methods.add(methodNode) + return methodNode + } + + override fun visitEnd() { + classAnnotations.sortBy { it.first.desc } + fields.sortBy { it.name } + methods.sortBy { it.name } + + kotlinMetadata?.let { (descriptor, header) -> + transformAndWriteKotlinMetadata( + metadata = header, + descriptor = descriptor, + classVisitor = cv, + classesToBeDeleted = classesToBeDeleted, + ) + } + + for ((annotation, visible) in classAnnotations) { + val annotationVisitor = cv.visitAnnotation(annotation.desc, visible) + annotation.accept(annotationVisitor) + } + + for (field in fields) { + field.accept(cv) + } + + for (method in methods) { + //val exceptionsArray = if (method.exceptions == null) null else method.exceptions.toTypedArray() + //val methodVisitor = cv.visitMethod(method.access, method.name, method.desc, method.signature, exceptionsArray) + //method.accept(methodVisitor) + + val mv = cv.visitMethod(method.access, method.name, method.desc, method.signature, method.exceptions?.toTypedArray()) + val stripper = MethodBodyStripper(mv) + stripper.visitCode() + stripper.visitMaxs(0, 0) + stripper.visitEnd() + } + + super.visitEnd() + } + + private fun isKotlinInternal(methodName: String?): Boolean { + val kClass = (kotlinMetadata?.second as? KotlinClassMetadata.Class)?.kmClass ?: return false + for (function in kClass.functions) { + if (function.visibility == Visibility.INTERNAL && function.name == methodName) { + return true + } + } + return false + } +} + +private class MethodBodyStripper(mv: MethodVisitor) : MethodVisitor(Opcodes.API_VERSION, mv) { + override fun visitCode() { + super.visitCode() + mv.visitInsn(Opcodes.RETURN) + } + + override fun visitMaxs(maxStack: Int, maxLocals: Int) { + // indicate no stack and locals required for stripped body + super.visitMaxs(0, 0) + } +} + +private fun transformAndWriteKotlinMetadata( + metadata: KotlinClassMetadata, + descriptor: String, + classVisitor: ClassVisitor, + classesToBeDeleted: Set, +) { + val treatInternalAsPrivate = false + when (metadata) { + is KotlinClassMetadata.Class -> { + removePrivateDeclarationsForClass( + klass = metadata.kmClass, + removeCopyAlongWithConstructor = false, + preserveDeclarationOrder = false, + classesToBeDeleted = classesToBeDeleted, + // todo try to set to true + pruneClass = false, + treatInternalAsPrivate = treatInternalAsPrivate, + ) + } + + is KotlinClassMetadata.FileFacade -> { + removePrivateDeclarationsForPackage( + kmPackage = metadata.kmPackage, + preserveDeclarationOrder = false, + pruneClass = false, + treatInternalAsPrivate = treatInternalAsPrivate, + ) + } + + is KotlinClassMetadata.MultiFileClassPart -> { + removePrivateDeclarationsForPackage( + kmPackage = metadata.kmPackage, + preserveDeclarationOrder = false, + pruneClass = false, + treatInternalAsPrivate = treatInternalAsPrivate, + ) + } + + else -> { + } + } + + val annotationVisitor = classVisitor.visitAnnotation(descriptor, true) + visitKotlinMetadata(annotationVisitor, metadata.write()) +} + +@Suppress("SameParameterValue") +private fun removePrivateDeclarationsForPackage( + kmPackage: KmPackage, + preserveDeclarationOrder: Boolean, + pruneClass: Boolean, + treatInternalAsPrivate: Boolean, +) { + removePrivateDeclarationsForDeclarationContainer( + container = kmPackage as KmDeclarationContainer, + copyFunShouldBeDeleted = false, + preserveDeclarationOrder = preserveDeclarationOrder, + pruneClass = pruneClass, + treatInternalAsPrivate = treatInternalAsPrivate, + ) + kmPackage.localDelegatedProperties.clear() +} + +@Suppress("SameParameterValue") +private fun removePrivateDeclarationsForClass( + klass: KmClass, + removeCopyAlongWithConstructor: Boolean, + preserveDeclarationOrder: Boolean, + classesToBeDeleted: Set, + pruneClass: Boolean, + treatInternalAsPrivate: Boolean, +) { + klass.constructors.removeIf { pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) } + removePrivateDeclarationsForDeclarationContainer( + container = klass as KmDeclarationContainer, + copyFunShouldBeDeleted = klass.copyFunShouldBeDeleted(removeDataClassCopy = removeCopyAlongWithConstructor), + preserveDeclarationOrder = preserveDeclarationOrder, + pruneClass = pruneClass, + treatInternalAsPrivate = treatInternalAsPrivate, + ) + klass.nestedClasses.removeIf { "${klass.name}$$it" in classesToBeDeleted } + klass.companionObject = klass.companionObject.takeUnless { "${klass.name}$$it" in classesToBeDeleted } + klass.localDelegatedProperties.clear() +} + +private fun KmClass.copyFunShouldBeDeleted(removeDataClassCopy: Boolean): Boolean { + return removeDataClassCopy && isData && constructors.none { !it.isSecondary } +} + +private fun Visibility.shouldRemove(treatInternalAsPrivate: Boolean): Boolean { + return this == Visibility.PRIVATE || this == Visibility.PRIVATE_TO_THIS || this == Visibility.LOCAL || (treatInternalAsPrivate && this == Visibility.INTERNAL) +} + +private fun removePrivateDeclarationsForDeclarationContainer( + container: KmDeclarationContainer, + copyFunShouldBeDeleted: Boolean, + preserveDeclarationOrder: Boolean, + pruneClass: Boolean, + treatInternalAsPrivate: Boolean, +) { + container.functions.removeIf { + pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) || (copyFunShouldBeDeleted && it.name == "copy") + } + container.properties.removeIf { + pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) + } + + if (!preserveDeclarationOrder) { + container.functions.sortWith(compareBy(KmFunction::name, { it.signature.toString() })) + container.properties.sortWith(compareBy(KmProperty::name, { it.getterSignature.toString() })) + } + + for (property in container.properties) { + // whether the *non-const* property is initialized by a compile-time constant is not a part of the ABI. + if (!property.isConst) { + property.hasConstant = false + } + } +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/packager/packager.kt b/build/jvm-rules/src/jps-builder/packager/packager.kt new file mode 100644 index 000000000000..f34263fe8d3e --- /dev/null +++ b/build/jvm-rules/src/jps-builder/packager/packager.kt @@ -0,0 +1,175 @@ +@file:Suppress("UnstableApiUsage") + +package org.jetbrains.bazel.jvm.jps + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.jetbrains.intellij.build.io.* +import org.jetbrains.jps.incremental.storage.ExperimentalSourceToOutputMapping +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassWriter +import java.nio.ByteBuffer +import java.nio.channels.FileChannel +import java.nio.file.* +import java.nio.file.attribute.DosFileAttributeView +import java.nio.file.attribute.PosixFileAttributeView +import java.nio.file.attribute.PosixFilePermission +import java.util.zip.ZipEntry + + +suspend fun packageToJar( + outJar: Path, + abiJar: Path?, + sourceToOutputMap: ExperimentalSourceToOutputMapping, + classOutDir: Path, + messageHandler: ConsoleMessageHandler +) { + if (abiJar == null) { + withContext(Dispatchers.IO) { + createJar( + outJar = outJar, + sourceToOutputMap = sourceToOutputMap, + classOutDir = classOutDir, + abiChannel = null, + messageHandler = messageHandler, + ) + } + return + } + + val classChannel = Channel>(capacity = 8) + withContext(Dispatchers.IO) { + launch { + createJar( + outJar = outJar, + sourceToOutputMap = sourceToOutputMap, + classOutDir = classOutDir, + abiChannel = classChannel, + messageHandler = messageHandler, + ) + classChannel.close() + } + + writeZipUsingTempFile(abiJar, indexWriter = null) { stream -> + val classesToBeDeleted = HashSet() + for ((name, classData) in classChannel) { + val classWriter = ClassWriter(0) + val abiClassVisitor = AbiClassVisitor(classVisitor = classWriter, classesToBeDeleted = classesToBeDeleted) + ClassReader(classData).accept(abiClassVisitor, 0) + if (!abiClassVisitor.isApiClass) { + continue + } + + val abiData = classWriter.toByteArray() + stream.writeDataRawEntry(ByteBuffer.wrap(abiData), name, abiData.size, abiData.size, ZipEntry.STORED, 0) + } + + if (classesToBeDeleted.isNotEmpty()) { + messageHandler.out.appendLine("Classes to be deleted: ${classesToBeDeleted.size}") + } + } + } +} + +private suspend fun createJar( + outJar: Path, + sourceToOutputMap: ExperimentalSourceToOutputMapping, + classOutDir: Path, + abiChannel: Channel>?, + messageHandler: ConsoleMessageHandler, +) { + val packageIndexBuilder = PackageIndexBuilder() + writeZipUsingTempFile(outJar, packageIndexBuilder.indexWriter) { stream -> + // MVStore like a TreeMap, keys already sorted + //val all = sourceToOutputMap.outputs().toList() + //val unique = LinkedHashSet(all) + //if (unique.size != all.size) { + // messageHandler.out.appendLine("Duplicated outputs: ${all.groupingBy { it } + // .eachCount() + // .filter { it.value > 1 } + // .keys + // }") + //} + + for (path in sourceToOutputMap.outputs().toList()) { + // duplicated - ignore it + if (path.endsWith(".kotlin_module")) { + continue + } + + packageIndexBuilder.addFile(name = path, addClassDir = false) + try { + val file = classOutDir.resolve(path) + if (abiChannel != null && path.endsWith(".class")) { + val name = path.toByteArray() + val classData = stream.fileAndGetData(name, file) + abiChannel.send(name to classData) + } + else { + stream.file(nameString = path, file = file) + } + } + catch (_: NoSuchFileException) { + messageHandler.warn("output file exists in src-to-output mapping, but not found on disk: $path") + } + } + packageIndexBuilder.writePackageIndex(stream = stream, addDirEntriesMode = AddDirEntriesMode.RESOURCE_ONLY) + } +} + +private inline fun writeZipUsingTempFile(file: Path, indexWriter: IkvIndexBuilder?, task: (ZipArchiveOutputStream) -> Unit) { + val tempFile = Files.createTempFile(file.parent, file.fileName.toString(), ".tmp") + var moved = false + try { + ZipArchiveOutputStream( + channel = FileChannel.open(tempFile, WRITE), + zipIndexWriter = ZipIndexWriter(indexWriter), + ).use { + task(it) + } + + try { + moveAtomic(tempFile, file) + } + catch (e: AccessDeniedException) { + makeFileWritable(file, e) + moveAtomic(tempFile, file) + } + moved = true + } + finally { + if (!moved) { + Files.deleteIfExists(tempFile) + } + } +} + +private fun moveAtomic(from: Path, to: Path) { + try { + Files.move(from, to, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE) + } + catch (_: AtomicMoveNotSupportedException) { + Files.move(from, to, StandardCopyOption.REPLACE_EXISTING) + } +} + +private fun makeFileWritable(file: Path, cause: Throwable) { + val posixView = Files.getFileAttributeView(file, PosixFileAttributeView::class.java) + if (posixView != null) { + val permissions = posixView.readAttributes().permissions() + permissions.add(PosixFilePermission.OWNER_WRITE) + posixView.setPermissions(permissions) + } + + val dosView = Files.getFileAttributeView(file, DosFileAttributeView::class.java) + @Suppress("IfThenToSafeAccess") + if (dosView != null) { + dosView.setReadOnly(false) + } + + throw UnsupportedOperationException("Unable to modify file attributes. Unsupported platform.", cause) +} + + diff --git a/build/jvm-rules/src/jps-builder/state/configuration.kt b/build/jvm-rules/src/jps-builder/state/configuration.kt new file mode 100644 index 000000000000..a294ed4644a2 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/state/configuration.kt @@ -0,0 +1,36 @@ +package org.jetbrains.bazel.jvm.jps.state + +internal enum class TargetConfigurationDigestProperty(@JvmField val description: String) { + COMPILER("kotlinc/javac configuration"), + DEPENDENCY_PATH_LIST("dependency path list"), + DEPENDENCY_DIGEST_LIST("dependency digest list"); + + companion object { + val VERSION = versionDigest() + } +} + +private fun emptyContainer(): LongArray { + val list = LongArray(TargetConfigurationDigestProperty.entries.size + 1) + list[list.lastIndex] = TargetConfigurationDigestProperty.VERSION + return list +} + +@JvmInline +internal value class TargetConfigurationDigestContainer( + private val list: LongArray = emptyContainer(), +) { + fun get(kind: TargetConfigurationDigestProperty): Long = list[kind.ordinal] + + fun set(kind: TargetConfigurationDigestProperty, hash: Long) { + list[kind.ordinal] = hash + } + + fun asArray(): LongArray = list.copyOf() + + val version: Long + get() = list.last() + + val rawSize: Int + get() = list.size +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/state/hash.kt b/build/jvm-rules/src/jps-builder/state/hash.kt new file mode 100644 index 000000000000..6aec507452c0 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/state/hash.kt @@ -0,0 +1,16 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.bazel.jvm.jps.state + +import com.dynatrace.hash4j.hashing.HashFunnel +import com.dynatrace.hash4j.hashing.Hashing + +internal inline fun > hashFunnel(): HashFunnel { + return HashFunnel { obj, sink -> + sink.putInt(obj.ordinal) + sink.putString(obj.name) + } +} + +internal inline fun > versionDigest(): Long { + return Hashing.xxh3_64().hashStream().putOrderedIterable(enumValues().asIterable(), hashFunnel()).asLong +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/state/state.kt b/build/jvm-rules/src/jps-builder/state/state.kt new file mode 100644 index 000000000000..6a6b9a44b1d5 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/state/state.kt @@ -0,0 +1,31 @@ +package org.jetbrains.bazel.jvm.jps.state + +internal enum class TargetStateProperty { + AverageBuildTime, + LastSuccessfulRebuildDuration, + ; + + companion object { + val VERSION = versionDigest() + } +} + +private fun emptyContainer(): LongArray { + val list = LongArray(TargetStateProperty.entries.size + 1) { -1 } + list[list.lastIndex] = TargetStateProperty.VERSION + return list +} + +@JvmInline +internal value class TargetStateContainer(private val list: LongArray = emptyContainer()) { + fun get(kind: TargetStateProperty): Long = list[kind.ordinal] + + fun set(kind: TargetStateProperty, value: Long) { + list[kind.ordinal] = value + } + + fun asArray(): LongArray = list.copyOf() + + val isCorrect: Boolean + get() = list.size == (TargetStateProperty.entries.size + 1) && list.last() == TargetStateProperty.VERSION +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/state/targetState.kt b/build/jvm-rules/src/jps-builder/state/targetState.kt new file mode 100644 index 000000000000..04a409a93d4e --- /dev/null +++ b/build/jvm-rules/src/jps-builder/state/targetState.kt @@ -0,0 +1,80 @@ +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("UnstableApiUsage", "ReplaceGetOrSet") + +package org.jetbrains.bazel.jvm.jps.state + +import org.h2.mvstore.MVMap +import org.h2.mvstore.MVStore +import org.h2.mvstore.type.LongDataType +import org.jetbrains.bazel.jvm.jps.impl.BazelBuildTargetStateManager +import org.jetbrains.jps.incremental.storage.StorageManager +import org.jetbrains.jps.incremental.storage.dataTypes.LongListKeyDataType + +private const val targetStateMapName = "target-state" + +private const val DIGEST_LIST_KEY = 1L +private const val DURATION_KEY = 2L + +private val targetStateMapBuilder = run { + val mapBuilder = MVMap.Builder() + mapBuilder.setKeyType(LongDataType.INSTANCE) + mapBuilder.setValueType(LongListKeyDataType) + mapBuilder +} + +// returns a reason to force rebuild +internal fun checkConfiguration( + store: MVStore, + targetDigests: TargetConfigurationDigestContainer, +): String? { + val map = try { + store.openMap(targetStateMapName, targetStateMapBuilder) + } + catch (e: Throwable) { + return "Cannot open map $targetStateMapName: ${e.stackTraceToString()}" + } + + val storedDigest = map.get(DIGEST_LIST_KEY)?.let { TargetConfigurationDigestContainer(it) } + when { + storedDigest == null -> { + return "Configuration digest not found" + } + storedDigest.rawSize != targetDigests.rawSize -> { + return "Configuration digest size mismatch: expected ${targetDigests.rawSize}, got ${storedDigest.rawSize}" + } + storedDigest.version != TargetConfigurationDigestProperty.VERSION -> { + return "Configuration digest format version mismatch: expected ${TargetConfigurationDigestProperty.VERSION}, got ${storedDigest.version}" + } + } + + for (kind in TargetConfigurationDigestProperty.entries) { + val storedHash = storedDigest.get(kind) + val hash = targetDigests.get(kind) + if (hash != storedHash) { + return "Configuration digest mismatch (${kind.description}): expected $hash, got $storedHash" + } + } + return null +} + +internal fun saveTargetState( + targetDigests: TargetConfigurationDigestContainer, + manager: BazelBuildTargetStateManager, + storageManager: StorageManager +) { + val map = storageManager.openMap(targetStateMapName, targetStateMapBuilder).map + map.operate(DIGEST_LIST_KEY, targetDigests.asArray(), PutIfChanged) + map.operate(DURATION_KEY, manager.state.asArray(), PutIfChanged) +} + +private object PutIfChanged : MVMap.DecisionMaker() { + override fun decide(existingValue: LongArray?, providedValue: LongArray): MVMap.Decision { + return if (providedValue.contentEquals(existingValue)) MVMap.Decision.ABORT else MVMap.Decision.PUT + } +} + +internal fun loadTargetState(storageManager: StorageManager): TargetStateContainer { + val map = storageManager.openMap(targetStateMapName, targetStateMapBuilder).map + // copyOf - do not mutate data in store directly + return map.get(DURATION_KEY)?.let { TargetStateContainer(it.copyOf()) }?.takeIf { it.isCorrect } ?: TargetStateContainer() +} \ No newline at end of file diff --git a/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt b/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt index 09f388056e2a..809c0ce5b235 100644 --- a/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt +++ b/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt @@ -1,7 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.bazel.jvm.kotlin -import com.google.devtools.build.lib.worker.WorkerProtocol +import org.jetbrains.bazel.jvm.WorkRequest import org.jetbrains.bazel.jvm.WorkRequestExecutor import org.jetbrains.bazel.jvm.processRequests import java.io.Writer @@ -14,12 +14,12 @@ object KotlinBuildWorker : WorkRequestExecutor { processRequests(startupArgs, this, debugLogClassifier = "kotlin") } - override suspend fun execute(request: WorkerProtocol.WorkRequest, writer: Writer, baseDir: Path): Int { - val sources = request.inputsList.asSequence() + override suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path): Int { + val sources = request.inputs.asSequence() .filter { it.path.endsWith(".kt") || it.path.endsWith(".java") } - .map { baseDir.resolve(it.path) } + .map { baseDir.resolve(it.path).normalize() } .toList() - return buildKotlin(workingDir = baseDir, args = parseArgs(request.argumentsList), out = writer, sources = sources) + return buildKotlin(workingDir = baseDir, args = parseArgs(request.arguments), out = writer, sources = sources) } } diff --git a/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt b/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt index 9db94c7a481c..76a9ee615169 100644 --- a/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt +++ b/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt @@ -37,7 +37,7 @@ internal suspend fun compileKotlinForJvm( kotlinArgs.classpath = createClasspath(args, workingDir) kotlinArgs.moduleName = info.moduleName - kotlinArgs.destination = workingDir.resolve(args.mandatorySingle(JvmBuilderFlags.OUTPUT)).toString() + kotlinArgs.destination = workingDir.resolve(args.mandatorySingle(JvmBuilderFlags.OUT)).toString() val pluginConfigurations = configurePlugins(args = args, workingDir = workingDir, label = info.label) diff --git a/build/jvm-rules/src/kotlin-builder/compilation/plugins.kt b/build/jvm-rules/src/kotlin-builder/compilation/plugins.kt index 083cfe6d9b8a..d33ae8e030d4 100644 --- a/build/jvm-rules/src/kotlin-builder/compilation/plugins.kt +++ b/build/jvm-rules/src/kotlin-builder/compilation/plugins.kt @@ -85,7 +85,7 @@ internal fun configurePlugins( } } - args.optionalSingle(JvmBuilderFlags.KOTLIN_OUTPUT_JDEPS)?.let { workingDir.resolve(it) }?.let { jdeps -> + args.optionalSingle(JvmBuilderFlags.JDEPS_OUT)?.let { workingDir.resolve(it) }?.let { jdeps -> val options = mutableListOf( cliOptionValue("output", jdeps.toString()), cliOptionValue("target_label", label), @@ -101,7 +101,7 @@ internal fun configurePlugins( )) } - args.optionalSingle(JvmBuilderFlags.ABI_JAR)?.let { workingDir.resolve(it) }?.let { abiJar -> + args.optionalSingle(JvmBuilderFlags.ABI_OUT)?.let { workingDir.resolve(it) }?.let { abiJar -> addPlugin(RegisteredPluginInfo( componentRegistrar = null, compilerPluginRegistrar = JvmAbiComponentRegistrar(), diff --git a/build/jvm-rules/src/kotlin-plugins/abi/JvmAbiMetadataProcessor.kt b/build/jvm-rules/src/kotlin-plugins/abi/JvmAbiMetadataProcessor.kt index ff5e78c3ee5d..3edb47bcf771 100644 --- a/build/jvm-rules/src/kotlin-plugins/abi/JvmAbiMetadataProcessor.kt +++ b/build/jvm-rules/src/kotlin-plugins/abi/JvmAbiMetadataProcessor.kt @@ -5,218 +5,225 @@ package org.jetbrains.kotlin.jvm.abi -import kotlin.metadata.* -import kotlin.metadata.jvm.* import org.jetbrains.kotlin.load.java.JvmAnnotationNames.* import org.jetbrains.org.objectweb.asm.AnnotationVisitor import org.jetbrains.org.objectweb.asm.Opcodes +import kotlin.metadata.* +import kotlin.metadata.jvm.* /** * Wrap the visitor for a Kotlin Metadata annotation to strip out private and local * functions, properties, and type aliases as well as local delegated properties. */ fun abiMetadataProcessor( - annotationVisitor: AnnotationVisitor, - removeDataClassCopyIfConstructorIsPrivate: Boolean, - preserveDeclarationOrder: Boolean, - classesToBeDeleted: Set, - pruneClass: Boolean, - treatInternalAsPrivate: Boolean, -): AnnotationVisitor = - kotlinClassHeaderVisitor { header -> - // kotlinx-metadata only supports writing Kotlin metadata of version >= 1.4, so we need to - // update the metadata version if we encounter older metadata annotations. - val metadataVersion = header.metadataVersion.takeIf { v -> - val major = v.getOrNull(0) ?: 0 - val minor = v.getOrNull(1) ?: 0 - major > 1 || major == 1 && minor >= 4 - } ?: intArrayOf(1, 4) + annotationVisitor: AnnotationVisitor, + removeDataClassCopyIfConstructorIsPrivate: Boolean, + preserveDeclarationOrder: Boolean, + classesToBeDeleted: Set, + pruneClass: Boolean, + treatInternalAsPrivate: Boolean, +): AnnotationVisitor { + return kotlinClassHeaderVisitor { header -> + // kotlinx-metadata only supports writing Kotlin metadata of version >= 1.4, so we need to + // update the metadata version if we encounter older metadata annotations. + val metadataVersion = header.metadataVersion.takeIf { v -> + val major = v.getOrNull(0) ?: 0 + val minor = v.getOrNull(1) ?: 0 + major > 1 || major == 1 && minor >= 4 + } ?: intArrayOf(1, 4) - val newHeader = runCatching { - KotlinClassMetadata.transform(header) { metadata -> - when (metadata) { - is KotlinClassMetadata.Class -> { - metadata.kmClass.removePrivateDeclarations( - removeDataClassCopyIfConstructorIsPrivate, - preserveDeclarationOrder, - classesToBeDeleted, - pruneClass, - treatInternalAsPrivate, - ) - } - is KotlinClassMetadata.FileFacade -> { - metadata.kmPackage.removePrivateDeclarations(preserveDeclarationOrder, pruneClass, treatInternalAsPrivate) - } - is KotlinClassMetadata.MultiFileClassPart -> { - metadata.kmPackage.removePrivateDeclarations(preserveDeclarationOrder, pruneClass, treatInternalAsPrivate) - } - else -> Unit - } - } - }.getOrElse { cause -> - // TODO: maybe jvm-abi-gen should throw this exception by default, and not only in tests. - if (System.getProperty("idea.is.unit.test").toBoolean()) { - val actual = "${metadataVersion[0]}.${metadataVersion[1]}" - val expected = JvmMetadataVersion.LATEST_STABLE_SUPPORTED.toString() - throw AssertionError( - "jvm-abi-gen can't process class file with the new metadata version because the version of kotlinx-metadata-jvm " + - "it depends on is too old.\n" + - "Class file has metadata version $actual, but default metadata version of kotlinx-metadata-jvm is " + - "$expected, so it can process class files with metadata version up to +1 from that (because of " + - "Kotlin/JVM's one-version forward compatibility policy).\n" + - "To fix this error, ensure that jvm-abi-gen depends on the latest version of kotlinx-metadata-jvm.\n" + - "If this happens during the update of the default language version in the project, make sure that " + - "a version of kotlinx-metadata-jvm has been published that supports this version, and update " + - "\"versions.kotlinx-metadata-jvm\" in `gradle/versions.properties`.", - cause - ) - } - header + val newHeader = runCatching { + KotlinClassMetadata.transform(header) { metadata -> + when (metadata) { + is KotlinClassMetadata.Class -> { + metadata.kmClass.removePrivateDeclarations( + removeDataClassCopyIfConstructorIsPrivate, + preserveDeclarationOrder, + classesToBeDeleted, + pruneClass, + treatInternalAsPrivate, + ) + } + + is KotlinClassMetadata.FileFacade -> { + metadata.kmPackage.removePrivateDeclarations(preserveDeclarationOrder, pruneClass, treatInternalAsPrivate) + } + + is KotlinClassMetadata.MultiFileClassPart -> { + metadata.kmPackage.removePrivateDeclarations(preserveDeclarationOrder, pruneClass, treatInternalAsPrivate) + } + + else -> Unit } - - // Write out the stripped annotation - annotationVisitor.visitKotlinMetadata(newHeader) + } + }.getOrElse { cause -> + // TODO: maybe jvm-abi-gen should throw this exception by default, and not only in tests. + if (System.getProperty("idea.is.unit.test").toBoolean()) { + val actual = "${metadataVersion[0]}.${metadataVersion[1]}" + val expected = JvmMetadataVersion.LATEST_STABLE_SUPPORTED.toString() + throw AssertionError( + "jvm-abi-gen can't process class file with the new metadata version because the version of kotlinx-metadata-jvm " + + "it depends on is too old.\n" + + "Class file has metadata version $actual, but default metadata version of kotlinx-metadata-jvm is " + + "$expected, so it can process class files with metadata version up to +1 from that (because of " + + "Kotlin/JVM's one-version forward compatibility policy).\n" + + "To fix this error, ensure that jvm-abi-gen depends on the latest version of kotlinx-metadata-jvm.\n" + + "If this happens during the update of the default language version in the project, make sure that " + + "a version of kotlinx-metadata-jvm has been published that supports this version, and update " + + "\"versions.kotlinx-metadata-jvm\" in `gradle/versions.properties`.", + cause + ) + } + header } + // Write out the stripped annotation + annotationVisitor.visitKotlinMetadata(newHeader) + } +} + + /** * Parse a KotlinClassHeader from an existing Kotlin Metadata annotation visitor. */ -private fun kotlinClassHeaderVisitor(body: (Metadata) -> Unit): AnnotationVisitor = - object : AnnotationVisitor(Opcodes.API_VERSION) { - var kind: Int = 1 - var metadataVersion: IntArray = intArrayOf() - var data1: MutableList = mutableListOf() - var data2: MutableList = mutableListOf() - var extraString: String? = null - var packageName: String? = null - var extraInt: Int = 0 +private fun kotlinClassHeaderVisitor(body: (Metadata) -> Unit): AnnotationVisitor { + return object : AnnotationVisitor(Opcodes.API_VERSION) { + var kind: Int = 1 + var metadataVersion: IntArray = intArrayOf() + var data1: MutableList = mutableListOf() + var data2: MutableList = mutableListOf() + var extraString: String? = null + var packageName: String? = null + var extraInt: Int = 0 - override fun visit(name: String, value: Any?) { - when (name) { - KIND_FIELD_NAME -> kind = value as Int - METADATA_EXTRA_INT_FIELD_NAME -> extraInt = value as Int - METADATA_VERSION_FIELD_NAME -> metadataVersion = value as IntArray - METADATA_EXTRA_STRING_FIELD_NAME -> extraString = value as String - METADATA_PACKAGE_NAME_FIELD_NAME -> packageName = value as String - } - } - - override fun visitArray(name: String): AnnotationVisitor? { - val destination = when (name) { - METADATA_DATA_FIELD_NAME -> data1 - METADATA_STRINGS_FIELD_NAME -> data2 - else -> return null - } - return object : AnnotationVisitor(Opcodes.API_VERSION) { - override fun visit(name: String?, value: Any?) { - destination += value as String - } - } - } - - override fun visitEnd() { - body( - Metadata( - kind, - metadataVersion, - data1.toTypedArray(), - data2.toTypedArray(), - extraString, - packageName, - extraInt - ) - ) - } + override fun visit(name: String, value: Any?) { + when (name) { + KIND_FIELD_NAME -> kind = value as Int + METADATA_EXTRA_INT_FIELD_NAME -> extraInt = value as Int + METADATA_VERSION_FIELD_NAME -> metadataVersion = value as IntArray + METADATA_EXTRA_STRING_FIELD_NAME -> extraString = value as String + METADATA_PACKAGE_NAME_FIELD_NAME -> packageName = value as String + } } + override fun visitArray(name: String): AnnotationVisitor? { + val destination = when (name) { + METADATA_DATA_FIELD_NAME -> data1 + METADATA_STRINGS_FIELD_NAME -> data2 + else -> return null + } + return object : AnnotationVisitor(Opcodes.API_VERSION) { + override fun visit(name: String?, value: Any?) { + destination += value as String + } + } + } + + override fun visitEnd() { + body( + Metadata( + kind, + metadataVersion, + data1.toTypedArray(), + data2.toTypedArray(), + extraString, + packageName, + extraInt + ) + ) + } + } +} + /** * Serialize a KotlinClassHeader to an existing Kotlin Metadata annotation visitor. */ private fun AnnotationVisitor.visitKotlinMetadata(header: Metadata) { - visit(KIND_FIELD_NAME, header.kind) - visit(METADATA_VERSION_FIELD_NAME, header.metadataVersion) - if (header.data1.isNotEmpty()) { - visitArray(METADATA_DATA_FIELD_NAME).apply { - header.data1.forEach { visit(null, it) } - visitEnd() - } + visit(KIND_FIELD_NAME, header.kind) + visit(METADATA_VERSION_FIELD_NAME, header.metadataVersion) + if (header.data1.isNotEmpty()) { + visitArray(METADATA_DATA_FIELD_NAME).apply { + header.data1.forEach { visit(null, it) } + visitEnd() } - if (header.data2.isNotEmpty()) { - visitArray(METADATA_STRINGS_FIELD_NAME).apply { - header.data2.forEach { visit(null, it) } - visitEnd() - } + } + if (header.data2.isNotEmpty()) { + visitArray(METADATA_STRINGS_FIELD_NAME).apply { + header.data2.forEach { visit(null, it) } + visitEnd() } - if (header.extraString.isNotEmpty()) { - visit(METADATA_EXTRA_STRING_FIELD_NAME, header.extraString) - } - if (header.packageName.isNotEmpty()) { - visit(METADATA_PACKAGE_NAME_FIELD_NAME, header.packageName) - } - if (header.extraInt != 0) { - visit(METADATA_EXTRA_INT_FIELD_NAME, header.extraInt) - } - visitEnd() + } + if (header.extraString.isNotEmpty()) { + visit(METADATA_EXTRA_STRING_FIELD_NAME, header.extraString) + } + if (header.packageName.isNotEmpty()) { + visit(METADATA_PACKAGE_NAME_FIELD_NAME, header.packageName) + } + if (header.extraInt != 0) { + visit(METADATA_EXTRA_INT_FIELD_NAME, header.extraInt) + } + visitEnd() } private fun KmClass.removePrivateDeclarations( - removeCopyAlongWithConstructor: Boolean, - preserveDeclarationOrder: Boolean, - classesToBeDeleted: Set, - pruneClass: Boolean, - treatInternalAsPrivate: Boolean, + removeCopyAlongWithConstructor: Boolean, + preserveDeclarationOrder: Boolean, + classesToBeDeleted: Set, + pruneClass: Boolean, + treatInternalAsPrivate: Boolean, ) { - constructors.removeIf { pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) } - (this as KmDeclarationContainer).removePrivateDeclarations( - copyFunShouldBeDeleted(removeCopyAlongWithConstructor), - preserveDeclarationOrder, - pruneClass, - treatInternalAsPrivate, - ) - nestedClasses.removeIf { "$name\$$it" in classesToBeDeleted } - companionObject = companionObject.takeUnless { "$name\$$it" in classesToBeDeleted } - localDelegatedProperties.clear() - // TODO: do not serialize private type aliases once KT-17229 is fixed. + constructors.removeIf { pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) } + (this as KmDeclarationContainer).removePrivateDeclarations( + copyFunShouldBeDeleted(removeCopyAlongWithConstructor), + preserveDeclarationOrder, + pruneClass, + treatInternalAsPrivate, + ) + nestedClasses.removeIf { "$name\$$it" in classesToBeDeleted } + companionObject = companionObject.takeUnless { "$name\$$it" in classesToBeDeleted } + localDelegatedProperties.clear() + // TODO: do not serialize private type aliases once KT-17229 is fixed. } private fun KmPackage.removePrivateDeclarations( - preserveDeclarationOrder: Boolean, - pruneClass: Boolean, - treatInternalAsPrivate: Boolean, + preserveDeclarationOrder: Boolean, + pruneClass: Boolean, + treatInternalAsPrivate: Boolean, ) { - (this as KmDeclarationContainer).removePrivateDeclarations(false, preserveDeclarationOrder, pruneClass, treatInternalAsPrivate) - localDelegatedProperties.clear() - // TODO: do not serialize private type aliases once KT-17229 is fixed. + (this as KmDeclarationContainer).removePrivateDeclarations(false, preserveDeclarationOrder, pruneClass, treatInternalAsPrivate) + localDelegatedProperties.clear() + // TODO: do not serialize private type aliases once KT-17229 is fixed. } private fun KmDeclarationContainer.removePrivateDeclarations( - copyFunShouldBeDeleted: Boolean, - preserveDeclarationOrder: Boolean, - pruneClass: Boolean, - treatInternalAsPrivate: Boolean, + copyFunShouldBeDeleted: Boolean, + preserveDeclarationOrder: Boolean, + pruneClass: Boolean, + treatInternalAsPrivate: Boolean, ) { - functions.removeIf { pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) || (copyFunShouldBeDeleted && it.name == "copy") } - properties.removeIf { pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) } + functions.removeIf { pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) || (copyFunShouldBeDeleted && it.name == "copy") } + properties.removeIf { pruneClass || it.visibility.shouldRemove(treatInternalAsPrivate) } - if (!preserveDeclarationOrder) { - functions.sortWith(compareBy(KmFunction::name, { it.signature.toString() })) - properties.sortWith(compareBy(KmProperty::name, { it.getterSignature.toString() })) - } + if (!preserveDeclarationOrder) { + functions.sortWith(compareBy(KmFunction::name, { it.signature.toString() })) + properties.sortWith(compareBy(KmProperty::name, { it.getterSignature.toString() })) + } - for (property in properties) { - // Whether or not the *non-const* property is initialized by a compile-time constant is not a part of the ABI. - if (!property.isConst) { - property.hasConstant = false - } + for (property in properties) { + // Whether or not the *non-const* property is initialized by a compile-time constant is not a part of the ABI. + if (!property.isConst) { + property.hasConstant = false } + } } -private fun KmClass.copyFunShouldBeDeleted(removeDataClassCopy: Boolean): Boolean = - removeDataClassCopy && isData && constructors.none { !it.isSecondary } +private fun KmClass.copyFunShouldBeDeleted(removeDataClassCopy: Boolean): Boolean { + return removeDataClassCopy && isData && constructors.none { !it.isSecondary } +} private fun Visibility.shouldRemove(treatInternalAsPrivate: Boolean): Boolean { - return this == Visibility.PRIVATE || - this == Visibility.PRIVATE_TO_THIS || - this == Visibility.LOCAL || - (treatInternalAsPrivate && this == Visibility.INTERNAL) + return this == Visibility.PRIVATE || + this == Visibility.PRIVATE_TO_THIS || + this == Visibility.LOCAL || + (treatInternalAsPrivate && this == Visibility.INTERNAL) } diff --git a/build/jvm-rules/src/misc/JvmWorker.kt b/build/jvm-rules/src/misc/JvmWorker.kt index e94a8ab6028d..c7ba12ae24bb 100644 --- a/build/jvm-rules/src/misc/JvmWorker.kt +++ b/build/jvm-rules/src/misc/JvmWorker.kt @@ -1,7 +1,6 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.bazel.jvm -import com.google.devtools.build.lib.worker.WorkerProtocol import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.jetbrains.intellij.build.io.* @@ -16,8 +15,8 @@ object JvmWorker : WorkRequestExecutor { processRequests(startupArgs, this) } - override suspend fun execute(request: WorkerProtocol.WorkRequest, writer: Writer, baseDir: Path): Int { - val args = request.argumentsList + override suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path): Int { + val args = request.arguments if (args.isEmpty()) { writer.appendLine("Command is not specified") return 1 @@ -33,8 +32,8 @@ object JvmWorker : WorkRequestExecutor { when (taskKind) { "jar" -> { var stripPrefix = command[3] - if (stripPrefix == "" && request.inputsList.isNotEmpty()) { - val p = request.inputsList.first().path + if (stripPrefix == "" && request.inputs.isNotEmpty()) { + val p = request.inputs.first().path stripPrefix = command[4] val index = p.indexOf(stripPrefix) require(index != -1) @@ -42,7 +41,7 @@ object JvmWorker : WorkRequestExecutor { } createZip( outJar = Path.of(output), - inputs = request.inputsList, + inputs = request.inputs, baseDir = baseDir, stripPrefix = stripPrefix, ) @@ -51,7 +50,7 @@ object JvmWorker : WorkRequestExecutor { } "jdeps" -> { - val inputs = request.inputsList.asSequence() + val inputs = request.inputs.asSequence() .filter { it.path.endsWith(".jdeps") } .map { baseDir.resolve(it.path) } //Files.writeString(Path.of("${System.getProperty("user.home")}/f.txt"), inputs.joinToString("\n") { it.toString() }) @@ -73,7 +72,7 @@ object JvmWorker : WorkRequestExecutor { } } -private suspend fun createZip(outJar: Path, inputs: List, baseDir: Path, stripPrefix: String) { +private suspend fun createZip(outJar: Path, inputs: Array, baseDir: Path, stripPrefix: String) { //Files.writeString(Path.of("${System.getProperty("user.home")}/f.txt"), stripPrefix + "\n" + inputs.joinToString("\n") { it.toString() }) val stripPrefixWithSlash = stripPrefix.let { if (it.isEmpty()) "" else "$it/" } diff --git a/build/jvm-rules/src/worker-framework/AsyncLogger.kt b/build/jvm-rules/src/worker-framework/AsyncFileLogger.kt similarity index 95% rename from build/jvm-rules/src/worker-framework/AsyncLogger.kt rename to build/jvm-rules/src/worker-framework/AsyncFileLogger.kt index 74995ad8756e..8f7ed42cc154 100644 --- a/build/jvm-rules/src/worker-framework/AsyncLogger.kt +++ b/build/jvm-rules/src/worker-framework/AsyncFileLogger.kt @@ -12,7 +12,7 @@ import java.nio.file.StandardOpenOption import java.time.LocalDateTime import java.time.format.DateTimeFormatter -internal class AsyncLogger(file: Path, coroutineScope: CoroutineScope) { +internal class AsyncFileLogger(file: Path, coroutineScope: CoroutineScope) { private val logChannel = Channel(Channel.UNLIMITED) private val writer = Files.newOutputStream(file, StandardOpenOption.APPEND, StandardOpenOption.CREATE).bufferedWriter() diff --git a/build/jvm-rules/src/worker-framework/BUILD.bazel b/build/jvm-rules/src/worker-framework/BUILD.bazel index b5abc5fde4ab..5a70cf524970 100644 --- a/build/jvm-rules/src/worker-framework/BUILD.bazel +++ b/build/jvm-rules/src/worker-framework/BUILD.bazel @@ -10,8 +10,9 @@ java_proto_library( kt_jvm_library( name = "worker-framework", srcs = glob(["*.kt"], exclude = ["*Test.kt"]), + kotlinc_opts = "//:rules_jvm_bootstrap_kotlinc_options", deps = [ - ":worker_protocol_java_proto", + "@lib//:kotlin-stdlib", "//:kotlinx-coroutines-core", "//:protobuf-java", "@lib//:jetbrains-annotations", @@ -23,7 +24,9 @@ kt_jvm_library( name = "worker_test_lib", srcs = ["WorkRequestHandlerTest.kt"], associates = [":worker-framework"], + kotlinc_opts = "//:rules_jvm_bootstrap_kotlinc_options", deps = [ + "@lib//:kotlin-stdlib", ":worker_protocol_java_proto", "@junit_jupiter_api//jar", "@assertj//jar", diff --git a/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt b/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt index ff73e8ebf5eb..d3b422d64f10 100644 --- a/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt +++ b/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt @@ -3,7 +3,6 @@ package org.jetbrains.bazel.jvm -import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest import com.google.protobuf.CodedOutputStream import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel @@ -12,7 +11,6 @@ import kotlinx.coroutines.sync.withLock import org.jetbrains.annotations.VisibleForTesting import org.jetbrains.bazel.jvm.WorkRequestState.* import java.io.* -import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap @@ -25,13 +23,13 @@ fun interface WorkRequestExecutor { suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path): Int } -internal fun createLogger(classifier: String, coroutineScope: CoroutineScope): AsyncLogger? { +internal fun createLogger(classifier: String, coroutineScope: CoroutineScope): AsyncFileLogger? { val dir = Path.of(System.getProperty("user.home"), "$classifier-worker") // changing env or flag leads to rebuild, so, we create logger if log dir exists if (!Files.isDirectory(dir)) { return null } - return AsyncLogger(file = dir.resolve("log-" + System.currentTimeMillis() + ".txt"), coroutineScope = coroutineScope) + return AsyncFileLogger(file = dir.resolve("log-" + System.currentTimeMillis() + ".txt"), coroutineScope = coroutineScope) } fun processRequests(startupArgs: Array, executor: WorkRequestExecutor, debugLogClassifier: String? = null) { @@ -51,16 +49,8 @@ fun processRequests(startupArgs: Array, executor: WorkRequestExecutor, d } try { - val output = System.out - val input = System.`in` - // create workerIo only after creating messageProcessor, as messageProcessor must use original system input - val workerIo = wrapStandardSystemStreams() - WorkRequestHandler( - requestExecutor = executor, - input = input, - out = output, - logger = logger, - ).processRequests(workerIo) + WorkRequestHandler(requestExecutor = executor, input = System.`in`, out = System.out, logger = logger) + .processRequests() } finally { logger?.shutdown() @@ -101,7 +91,7 @@ internal class WorkRequestHandler internal constructor( * Must be quick and safe - executed in a read thread */ private val cancelHandler: ((Int) -> Unit)? = null, - private val logger: AsyncLogger? = null, + private val logger: AsyncFileLogger? = null, ) { private val workingDir = Path.of(".").toAbsolutePath().normalize() @@ -111,11 +101,11 @@ internal class WorkRequestHandler internal constructor( private val activeRequests = ConcurrentHashMap() @OptIn(DelicateCoroutinesApi::class) - internal suspend fun processRequests(workerIo: WorkerIo) { + internal suspend fun processRequests() { val requestChannel = Channel(Channel.UNLIMITED) try { coroutineScope { - startTaskProcessing(requestChannel, workerIo) + startTaskProcessing(requestChannel) readRequests(requestChannel) } @@ -136,22 +126,16 @@ internal class WorkRequestHandler internal constructor( } activeRequests.clear() } - - try { - // unwrap the system streams placing the original streams back - workerIo.close() - } - catch (e: Exception) { - System.err.println(e.message) - } } } private suspend fun readRequests(requestChannel: Channel) { + val inputListToReuse = ArrayList() + val argListToReuse = ArrayList() while (coroutineContext.isActive) { val request = try { runInterruptible(Dispatchers.IO) { - WorkRequest.parseDelimitedFrom(input) + readWorkRequestFromStream(input, inputListToReuse, argListToReuse) } } catch (e: InterruptedIOException) { @@ -179,7 +163,7 @@ internal class WorkRequestHandler internal constructor( if (cancelHandler != null) { cancelHandler(request.requestId) } - logger?.log("request(id=$requestId}) cancelled before handling") + logger?.log("request(id=$requestId) cancelled before handling") writeAndRemoveRequest(requestId = requestId, wasCancelled = true) } } @@ -204,10 +188,7 @@ internal class WorkRequestHandler internal constructor( } } - private fun CoroutineScope.startTaskProcessing( - requestChannel: Channel, - workerIo: WorkerIo - ) { + private fun CoroutineScope.startTaskProcessing(requestChannel: Channel) { repeat(Runtime.getRuntime().availableProcessors().coerceAtLeast(2)) { launch { for (item in requestChannel) { @@ -215,7 +196,7 @@ internal class WorkRequestHandler internal constructor( val stateRef = item.state when { stateRef.compareAndSet(NOT_STARTED, STARTED) -> { - handleRequest(workerIo = workerIo, request = item.request, requestState = stateRef) + handleRequest(request = item.request, requestState = stateRef) } else -> { val state = stateRef.get() @@ -289,20 +270,15 @@ internal class WorkRequestHandler internal constructor( * @throws IOException if there is an error talking to the server. Errors from calling the [][.callback] are reported with exit code 1. */ // visible for tests - internal suspend fun handleRequest(workerIo: WorkerIo, request: WorkRequest, requestState: AtomicReference) { + internal suspend fun handleRequest(request: WorkRequest, requestState: AtomicReference) { val baseDir = if (request.sandboxDir.isNullOrEmpty()) workingDir else workingDir.resolve(request.sandboxDir) var exitCode = 1 val stringWriter = StringWriter() var errorToThrow: Throwable? = null val requestId = request.requestId try { + //System.err.println("request: ${request.inputs.asSequence().take(4).joinToString { it.path }}, $baseDir, ${request.sandboxDir}") exitCode = requestExecutor.execute(request = request, writer = stringWriter, baseDir = baseDir) - - // read out the captured string for the final WorkResponse output - val captured = workerIo.readCapturedAsUtf8String()?.trim() - if (!captured.isNullOrEmpty()) { - stringWriter.write(captured) - } } catch (e: CancellationException) { errorToThrow = e @@ -332,82 +308,4 @@ internal class WorkRequestHandler internal constructor( throw errorToThrow } } -} - -/** - * A class that wraps the standard [System.in], [System.out], and [System.err] - * with our own ByteArrayOutputStream that allows [WorkRequestHandler] to safely capture - * outputs that can't be directly captured by the PrintStream associated with the work request. - * - * This is most useful when integrating JVM tools that write exceptions and logs directly to - * [System.out] and [System.err], which would corrupt the persistent worker protocol. - * We also redirect [System.in], just in case a tool should attempt to read it. - * - * WorkerIO implements [AutoCloseable] and will swap the original streams back into [System] once close has been called. - */ -@VisibleForTesting -internal class WorkerIo( - /** - * Returns the original input stream most commonly provided by [System.in] - */ - @JvmField val originalInputStream: InputStream?, - /** - * Returns the original output stream most commonly provided by [System.out] - */ - @JvmField val originalOutputStream: PrintStream?, - /** - * Returns the original error stream most commonly provided by [System.err] - */ - @JvmField val originalErrorStream: PrintStream?, - private val capturedStream: ByteArrayOutputStream, - private val restore: AutoCloseable) : AutoCloseable { - /** - * Returns the captured outputs as a UTF-8 string - */ - fun readCapturedAsUtf8String(): String? { - if (capturedStream.size() == 0) { - return null - } - - val captureOutput = capturedStream.toString(StandardCharsets.UTF_8) - capturedStream.reset() - return captureOutput - } - - override fun close() { - restore.close() - } -} - -/** - * Wraps the standard System streams and WorkerIO instance - */ -@VisibleForTesting -internal fun wrapStandardSystemStreams(): WorkerIo { - // save the original streams - val originalInputStream = System.`in` - val originalOutputStream = System.out - val originalErrorStream = System.err - - // replace the original streams with our own instances - val capturedStream = ByteArrayOutputStream() - val outputBuffer = PrintStream(capturedStream, true) - val byteArrayInputStream = ByteArray(0).inputStream() - System.setIn(byteArrayInputStream) - System.setOut(outputBuffer) - System.setErr(outputBuffer) - - return WorkerIo( - originalInputStream = originalInputStream, - originalOutputStream = originalOutputStream, - originalErrorStream = originalErrorStream, - capturedStream = capturedStream, - restore = AutoCloseable { - System.setIn(originalInputStream) - System.setOut(originalOutputStream) - System.setErr(originalErrorStream) - outputBuffer.close() - byteArrayInputStream.close() - } - ) } \ No newline at end of file diff --git a/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt b/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt index 93f6221922f6..a2812ac0169d 100644 --- a/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt +++ b/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt @@ -15,11 +15,10 @@ package org.jetbrains.bazel.jvm -import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest +import com.google.devtools.build.lib.worker.WorkerProtocol import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse import kotlinx.coroutines.* import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.Timeout import java.io.* @@ -35,26 +34,6 @@ import kotlin.coroutines.coroutineContext * Tests for the WorkRequestHandler */ class WorkRequestHandlerTest { - companion object { - private fun createTestWorkerIo(): WorkerIo { - val captured = ByteArrayOutputStream() - return WorkerIo( - originalInputStream = System.`in`, - originalOutputStream = System.out, - originalErrorStream = System.err, - capturedStream = captured, - restore = captured - ) - } - } - - private val testWorkerIo = createTestWorkerIo() - - @AfterEach - fun after() { - testWorkerIo.close() - } - @Test @Timeout(value = 10, unit = TimeUnit.SECONDS) fun normalWorkRequest() { @@ -65,9 +44,16 @@ class WorkRequestHandlerTest { out = out, ) - val request = WorkRequest.newBuilder().addAllArguments(listOf("--sources", "A.java")).build() + val request = WorkRequest( + arguments = arrayOf("--sources", "A.java"), + inputs = emptyArray(), + requestId = 0, + cancel = false, + verbosity = 0, + sandboxDir = null, + ) runBlocking { - handler.handleRequest(workerIo = testWorkerIo, request = request, requestState = AtomicReference(WorkRequestState.STARTED)) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(out.toByteArray().inputStream()) @@ -88,7 +74,7 @@ class WorkRequestHandlerTest { val request = newWorkRequest(listOf("--sources", "A.java")) runBlocking { - handler.handleRequest(workerIo = testWorkerIo, request = request, requestState = AtomicReference(WorkRequestState.STARTED)) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(out.toByteArray().inputStream()) @@ -127,8 +113,8 @@ class WorkRequestHandlerTest { useHandler(handler = handler, waitForProcessing = true, errorFilter = { it.message != "Intentional death!" }) { val args = listOf("--sources", "A.java") - newWorkRequest(args).writeDelimitedTo(src) - newWorkRequest(args, 43).writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().addAllArguments(args).setRequestId(42).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().addAllArguments(args).setRequestId(43).build().writeDelimitedTo(src) started.acquire(2) } @@ -149,9 +135,9 @@ class WorkRequestHandlerTest { ) val args = listOf("--sources", "A.java") - val request = WorkRequest.newBuilder().addAllArguments(args).build() + val request = newWorkRequest(args, requestId = 0) runBlocking { - handler.handleRequest(testWorkerIo, request, AtomicReference(WorkRequestState.STARTED)) + handler.handleRequest(request, AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(out.toByteArray().inputStream()) @@ -173,9 +159,9 @@ class WorkRequestHandlerTest { ) val args = listOf("--sources", "A.java") - val request = WorkRequest.newBuilder().addAllArguments(args).setRequestId(342).build() + val request = newWorkRequest(args, 342) runBlocking { - handler.handleRequest(workerIo = testWorkerIo, request = request, requestState = AtomicReference(WorkRequestState.STARTED)) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(ByteArrayInputStream(out.toByteArray())) @@ -206,8 +192,8 @@ class WorkRequestHandlerTest { ) useHandler(handler) { - WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) - WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) val response = WorkResponse.parseDelimitedFrom(dest) @@ -260,10 +246,10 @@ class WorkRequestHandlerTest { // this thread just makes sure the WorkRequestHandler does work asynchronously useHandler(handler, failures, waitForProcessing = true) { - WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) // make sure the handler is called before sending the cancel request, or we might process the cancellation entirely before that handlerCalled.acquire() - WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) waitForCancel.release() assertThat(WorkResponse.parseDelimitedFrom(dest).requestId).isEqualTo(42) @@ -308,9 +294,9 @@ class WorkRequestHandlerTest { ) useHandler(handler, failures, waitForProcessing = true) { - WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) - WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) - WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) waitForCancel.release() @@ -347,9 +333,9 @@ class WorkRequestHandlerTest { var r: WorkResponse? = null useHandler(handler, waitForProcessing = true) { - WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).build().writeDelimitedTo(src) r = WorkResponse.parseDelimitedFrom(dest) - WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) + WorkerProtocol.WorkRequest.newBuilder().setRequestId(42).setCancel(true).build().writeDelimitedTo(src) src.close() } @@ -370,15 +356,15 @@ class WorkRequestHandlerTest { fun workRequestHandlerWithWorkRequestCallback() { val out = ByteArrayOutputStream() val handler = WorkRequestHandler( - requestExecutor = { request, err, _ -> request.argumentsCount }, + requestExecutor = { request, err, _ -> request.arguments.size }, ByteArrayInputStream(ByteArray(0)), out, ) val args = listOf("--sources", "B.java") - val request = WorkRequest.newBuilder().addAllArguments(args).build() + val request = newWorkRequest(args, requestId = 0) runBlocking { - handler.handleRequest(workerIo = testWorkerIo, request = request, requestState = AtomicReference(WorkRequestState.STARTED)) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(ByteArrayInputStream(out.toByteArray())) @@ -386,75 +372,10 @@ class WorkRequestHandlerTest { assertThat(response.exitCode).isEqualTo(2) assertThat(response.getOutput()).isEmpty() } - - @Test - @Timeout(value = 10, unit = TimeUnit.SECONDS) - fun workerIODoesWrapSystemStreams() { - // Save the original streams - val originalInputStream = System.`in` - val originalOutputStream = System.out - val originalErrorStream = System.err - - // Swap in the test streams to assert against - val byteArrayInputStream = ByteArray(0).inputStream() - System.setIn(byteArrayInputStream) - val outputBuffer = PrintStream(ByteArrayOutputStream(), true) - System.setOut(outputBuffer) - System.setErr(outputBuffer) - - try { - outputBuffer.use { - byteArrayInputStream.use { - wrapStandardSystemStreams().use { io -> - // Assert that the WorkerIO returns the correct wrapped streams and the new System instance - // has been swapped out with the wrapped one - assertThat(io.originalInputStream).isSameAs(byteArrayInputStream) - assertThat(System.`in`).isNotSameAs(byteArrayInputStream) - - assertThat(io.originalOutputStream).isSameAs(outputBuffer) - assertThat(System.out).isNotSameAs(outputBuffer) - - assertThat(io.originalErrorStream).isSameAs(outputBuffer) - assertThat(System.err).isNotSameAs(outputBuffer) - } - } - } - } - finally { - // swap back in the original streams - System.setIn(originalInputStream) - System.setOut(originalOutputStream) - System.setErr(originalErrorStream) - } - } - - @Test - @Timeout(value = 10, unit = TimeUnit.SECONDS) - fun workerIODoesCaptureStandardOutAndErrorStreams() { - wrapStandardSystemStreams().use { io -> - // assert that nothing has been captured in the new instance - assertThat(io.readCapturedAsUtf8String()).isNullOrEmpty() - - // Assert that the standard out/error stream redirect to our own streams - print("This is a standard out message!") - System.err.print("This is a standard error message!") - assertThat(io.readCapturedAsUtf8String()).isEqualTo("This is a standard out message!This is a standard error message!") - - // Assert that readCapturedAsUtf8String calls reset on the captured stream after a read - assertThat(io.readCapturedAsUtf8String()).isNullOrEmpty() - - print("out 1") - System.err.print("err 1") - print("out 2") - System.err.print("err 2") - assertThat(io.readCapturedAsUtf8String()).isEqualTo("out 1err 1out 2err 2") - assertThat(io.readCapturedAsUtf8String()).isNullOrEmpty() - } - } } -private fun newWorkRequest(args: List, id: Int = 42): WorkRequest { - return WorkRequest.newBuilder().addAllArguments(args).setRequestId(id).build() +private fun newWorkRequest(args: List, requestId: Int = 42): WorkRequest { + return WorkRequest(arguments = args.toTypedArray(), requestId = requestId, inputs = emptyArray(), cancel = false, verbosity = 0, sandboxDir = null) } @OptIn(DelicateCoroutinesApi::class, ExperimentalCoroutinesApi::class) @@ -465,39 +386,37 @@ private inline fun useHandler( crossinline errorFilter: (Throwable) -> Boolean = { true }, task: () -> Unit ) { - wrapStandardSystemStreams().use { workerIo -> - val processor = GlobalScope.async(Dispatchers.Default) { - try { - handler.processRequests(workerIo) - } - catch (_: CancellationException) { - } - catch (e: Throwable) { - if (errorFilter(e)) { - failures.add(e.message!!) - } - } - } - + val processor = GlobalScope.async(Dispatchers.Default) { try { - task() - - if (waitForProcessing) { - runBlocking { - processor.await() - } - } - - // checks that there weren't other unexpected failures - assertThat(failures).isEmpty() + handler.processRequests() } - finally { - runBlocking { - if (processor.isCompleted) { - processor.getCompletionExceptionOrNull()?.let { throw it } - } - processor.cancel() + catch (_: CancellationException) { + } + catch (e: Throwable) { + if (errorFilter(e)) { + failures.add(e.message!!) } } } + + try { + task() + + if (waitForProcessing) { + runBlocking { + processor.await() + } + } + + // checks that there weren't other unexpected failures + assertThat(failures).isEmpty() + } + finally { + runBlocking { + if (processor.isCompleted) { + processor.getCompletionExceptionOrNull()?.let { throw it } + } + processor.cancel() + } + } } \ No newline at end of file diff --git a/build/jvm-rules/src/worker-framework/protocol.kt b/build/jvm-rules/src/worker-framework/protocol.kt new file mode 100644 index 000000000000..7838a528aee5 --- /dev/null +++ b/build/jvm-rules/src/worker-framework/protocol.kt @@ -0,0 +1,133 @@ +package org.jetbrains.bazel.jvm + +import com.google.protobuf.CodedInputStream +import java.io.FilterInputStream +import java.io.InputStream +import kotlin.math.min + +class Input( + @JvmField val path: String, + @JvmField val digest: ByteArray, +) + +class WorkRequest( + @JvmField val arguments: Array, + @JvmField val inputs: Array, + @JvmField val requestId: Int, + @JvmField val cancel: Boolean, + @JvmField val verbosity: Int, + @JvmField val sandboxDir: String? +) + +internal fun readWorkRequestFromStream( + input: InputStream, + inputListToReuse: MutableList, + argListToReuse: MutableList +): WorkRequest? { + // read the length-prefixed WorkRequest + val firstByte = input.read() + if (firstByte == -1) { + return null + } + + val size = CodedInputStream.readRawVarint32(firstByte, input) + + var requestId = 0 + var cancel = false + var verbosity = 0 + var sandboxDir: String? = null + + val codedInputStream = CodedInputStream.newInstance(LimitedInputStream(input, size)) + argListToReuse.clear() + inputListToReuse.clear() + while (true) { + val tag = codedInputStream.readTag() + if (tag == 0) { + break + } + + when (tag.shr(3)) { + 1 -> argListToReuse.add(codedInputStream.readString()) + 2 -> inputListToReuse.add(readInput(codedInputStream)) + 3 -> requestId = codedInputStream.readInt32() + 4 -> cancel = codedInputStream.readBool() + 5 -> verbosity = codedInputStream.readInt32() + 6 -> sandboxDir = codedInputStream.readString() + else -> codedInputStream.skipField(tag) + } + } + + return WorkRequest( + arguments = argListToReuse.toTypedArray(), + inputs = inputListToReuse.toTypedArray(), + requestId = requestId, + cancel = cancel, + verbosity = verbosity, + sandboxDir = sandboxDir + ) +} + +private fun readInput(codedInputStream: CodedInputStream): Input { + var path = "" + var digest: ByteArray? = null + + val messageSize = codedInputStream.readRawVarint32() + val limit = codedInputStream.pushLimit(messageSize) + while (!codedInputStream.isAtEnd) { + val tag = codedInputStream.readTag() + when (tag.shr(3)) { + 1 -> { + path = codedInputStream.readString() + } + + 2 -> { + digest = codedInputStream.readByteArray() + } + + else -> { + codedInputStream.skipField(tag) + } + } + } + codedInputStream.popLimit(limit) + + return Input(path = path, digest = digest!!) +} + +private class LimitedInputStream(input: InputStream, private var limit: Int) : FilterInputStream(input) { + override fun available(): Int = min(super.available(), limit) + + override fun read(): Int { + if (limit <= 0) { + return -1 + } + + val result = super.read() + if (result >= 0) { + --limit + } + return result + } + + override fun read(b: ByteArray?, off: Int, len: Int): Int { + var len = len + if (limit <= 0) { + return -1 + } + + len = min(len, limit) + val result = super.read(b, off, len) + if (result >= 0) { + limit -= result + } + return result + } + + override fun skip(n: Long): Long { + val result = super.skip(min(n, limit.toLong())).toInt() + if (result >= 0) { + limit -= result + } + return result.toLong() + } +} \ No newline at end of file diff --git a/build/jvm-rules/zip/BUILD.bazel b/build/jvm-rules/zip/BUILD.bazel index bb362c4b8935..fda4300be90c 100644 --- a/build/jvm-rules/zip/BUILD.bazel +++ b/build/jvm-rules/zip/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") kt_jvm_library( name = "build-zip", module_name = "intellij.idea.community.build.zip", + kotlinc_opts = "//:rules_jvm_bootstrap_kotlinc_options", visibility = ["//visibility:public"], srcs = glob(["src/*.kt"]), deps = [ diff --git a/build/jvm-rules/zip/src/ZipFileWriter.kt b/build/jvm-rules/zip/src/ZipFileWriter.kt index b555899a95db..08ac013f26bf 100644 --- a/build/jvm-rules/zip/src/ZipFileWriter.kt +++ b/build/jvm-rules/zip/src/ZipFileWriter.kt @@ -22,7 +22,7 @@ import java.util.zip.ZipEntry import kotlin.math.min val W_CREATE_NEW: EnumSet = EnumSet.of(StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW) -private val WRITE = EnumSet.of(StandardOpenOption.WRITE) +val WRITE: EnumSet = EnumSet.of(StandardOpenOption.WRITE) private val READ = EnumSet.of(StandardOpenOption.READ) // 1 MB @@ -43,6 +43,25 @@ fun ZipArchiveOutputStream.file(nameString: String, file: Path) { return } +@Suppress("DuplicatedCode") +fun ZipArchiveOutputStream.fileAndGetData(name: ByteArray, file: Path): ByteArray { + return FileChannel.open(file, READ).use { channel -> + val size = channel.size() + assert(size <= Int.MAX_VALUE) + writeEntryHeaderWithoutCrc(name = name, size = size.toInt()) + transferFrom(channel, size.toLong()) + + val buffer = ByteBuffer.allocate(size.toInt()) + while (buffer.hasRemaining()) { + val bytesRead = channel.read(buffer, 0) + if (bytesRead == -1) { + throw EOFException("End of stream reached before filling the buffer") + } + } + buffer.array() + } +} + fun transformZipUsingTempFile(file: Path, indexWriter: IkvIndexBuilder?, task: (ZipFileWriter) -> Unit) { val tempFile = Files.createTempFile(file.parent, file.fileName.toString(), ".tmp") try { diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index d52804daef82..47aa40d214c8 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -7499,62 +7499,6 @@ java_import( visibility = ["//visibility:public"] ) -java_library( - name = "platform-build_scripts-code_optimizer-proguard", - exports = [ - ":proguard-base-7_5_0_http_import", - ":proguard-core-9_1_4_http_import", - ":kotlin-metadata-jvm-2_0_0_http_import", - ":annotations-24_0_0_http_import", - ":log4j-api-2_19_0_http_import", - ":log4j-core-2_19_0_http_import", - ":json-20231013_http_import", - ], - visibility = ["//visibility:public"] -) - -kt_jvm_import( - name = "proguard-base-7_5_0_http_import", - jar = "@proguard-base-7_5_0_http//file", - srcjar = "@proguard-base-7_5_0-sources_http//file" -) - -kt_jvm_import( - name = "proguard-core-9_1_4_http_import", - jar = "@proguard-core-9_1_4_http//file", - srcjar = "@proguard-core-9_1_4-sources_http//file" -) - -kt_jvm_import( - name = "kotlin-metadata-jvm-2_0_0_http_import", - jar = "@kotlin-metadata-jvm-2_0_0_http//file", - srcjar = "@kotlin-metadata-jvm-2_0_0-sources_http//file" -) - -kt_jvm_import( - name = "annotations-24_0_0_http_import", - jar = "@annotations-24_0_0_http//file", - srcjar = "@annotations-24_0_0-sources_http//file" -) - -kt_jvm_import( - name = "log4j-api-2_19_0_http_import", - jar = "@log4j-api-2_19_0_http//file", - srcjar = "@log4j-api-2_19_0-sources_http//file" -) - -kt_jvm_import( - name = "log4j-core-2_19_0_http_import", - jar = "@log4j-core-2_19_0_http//file", - srcjar = "@log4j-core-2_19_0-sources_http//file" -) - -kt_jvm_import( - name = "json-20231013_http_import", - jar = "@json-20231013_http//file", - srcjar = "@json-20231013-sources_http//file" -) - java_library( name = "platform-build_scripts-jetbrains-apple-notary-api-kotlin-client", exports = [ @@ -9113,6 +9057,41 @@ kt_jvm_import( visibility = ["//visibility:public"] ) +java_library( + name = "proguard", + exports = [ + ":proguard-base-7_6_1_http_import", + ":proguard-core-9_1_7_http_import", + ":log4j-api-2_24_2_http_import", + ":log4j-core-2_24_2_http_import", + ], + visibility = ["//visibility:public"] +) + +kt_jvm_import( + name = "proguard-base-7_6_1_http_import", + jar = "@proguard-base-7_6_1_http//file", + srcjar = "@proguard-base-7_6_1-sources_http//file" +) + +kt_jvm_import( + name = "proguard-core-9_1_7_http_import", + jar = "@proguard-core-9_1_7_http//file", + srcjar = "@proguard-core-9_1_7-sources_http//file" +) + +kt_jvm_import( + name = "log4j-api-2_24_2_http_import", + jar = "@log4j-api-2_24_2_http//file", + srcjar = "@log4j-api-2_24_2-sources_http//file" +) + +kt_jvm_import( + name = "log4j-core-2_24_2_http_import", + jar = "@log4j-core-2_24_2_http//file", + srcjar = "@log4j-core-2_24_2-sources_http//file" +) + java_import( name = "protobuf", jars = ["@protobuf-java-3_24_4-jb_2_http//file"], diff --git a/lib/MODULE.bazel b/lib/MODULE.bazel index 740efac50db6..1ea278be7b3d 100644 --- a/lib/MODULE.bazel +++ b/lib/MODULE.bazel @@ -10569,90 +10569,6 @@ http_file( downloaded_file_path = "oshi-core-6.6.0-sources.jar" ) -http_file( - name = "proguard-base-7_5_0_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-base/7.5.0/proguard-base-7.5.0.jar", - sha256 = "04f507bf11e57d798d972b6640cd026302e0b54800cd75813dc4715da66f3ab5", - downloaded_file_path = "proguard-base-7.5.0.jar" -) - -http_file( - name = "proguard-core-9_1_4_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-core/9.1.4/proguard-core-9.1.4.jar", - sha256 = "c51bb89ac5668280e3db7e4ab6c16f454ec5132ed356c840a4f930f16ef6ed51", - downloaded_file_path = "proguard-core-9.1.4.jar" -) - -http_file( - name = "kotlin-metadata-jvm-2_0_0_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-metadata-jvm/2.0.0/kotlin-metadata-jvm-2.0.0.jar", - sha256 = "ad8f1c7dbc5ac46f5cbd2d2e5de39c56c9db65dd7de716a84e01ce208758aee6", - downloaded_file_path = "kotlin-metadata-jvm-2.0.0.jar" -) - -http_file( - name = "log4j-api-2_19_0_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.19.0/log4j-api-2.19.0.jar", - sha256 = "5ccb24ad9f92e768d0bc456d3061a737951262df803e004d2cad096b75a88d60", - downloaded_file_path = "log4j-api-2.19.0.jar" -) - -http_file( - name = "log4j-core-2_19_0_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.19.0/log4j-core-2.19.0.jar", - sha256 = "b4a1796fab7bfc36df015c1b4052459147997e8d215a7199d71d05f9e747e4f4", - downloaded_file_path = "log4j-core-2.19.0.jar" -) - -http_file( - name = "json-20231013_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/json/json/20231013/json-20231013.jar", - sha256 = "0f18192df289114e17aa1a0d0a7f8372cc9f5c7e4f7e39adcf8906fe714fa7d3", - downloaded_file_path = "json-20231013.jar" -) - -http_file( - name = "proguard-base-7_5_0-sources_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-base/7.5.0/proguard-base-7.5.0-sources.jar", - sha256 = "8be2f5fd5875b4ccd01d149059e7abacae56c9b0471212d0b80ceb8b211c360c", - downloaded_file_path = "proguard-base-7.5.0-sources.jar" -) - -http_file( - name = "proguard-core-9_1_4-sources_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-core/9.1.4/proguard-core-9.1.4-sources.jar", - sha256 = "4ca2942f6e7ce02431ba3692b4b47962e6b726eee34ef5508653676eb8e37200", - downloaded_file_path = "proguard-core-9.1.4-sources.jar" -) - -http_file( - name = "kotlin-metadata-jvm-2_0_0-sources_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-metadata-jvm/2.0.0/kotlin-metadata-jvm-2.0.0-sources.jar", - sha256 = "11cc619a970e643786053993b53286b6a0dc7f1bd787d99ed506c8d21e558424", - downloaded_file_path = "kotlin-metadata-jvm-2.0.0-sources.jar" -) - -http_file( - name = "log4j-api-2_19_0-sources_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.19.0/log4j-api-2.19.0-sources.jar", - sha256 = "b068cfc8d7d975c6d8910c0a0650874ae6445ec77b2e1511f3910384dee29f7f", - downloaded_file_path = "log4j-api-2.19.0-sources.jar" -) - -http_file( - name = "log4j-core-2_19_0-sources_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.19.0/log4j-core-2.19.0-sources.jar", - sha256 = "7f2c06b81cbf7fba6fc5e4f607c7b47cbd893c315cd2c237da65b941ca8b1278", - downloaded_file_path = "log4j-core-2.19.0-sources.jar" -) - -http_file( - name = "json-20231013-sources_http", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/json/json/20231013/json-20231013-sources.jar", - sha256 = "fc65cb53ae955d07f62478485446a544a0aba1e0ab8522688ee875dbbf11766e", - downloaded_file_path = "json-20231013-sources.jar" -) - http_file( name = "apple-notary-api-kotlin-client-1_0_5_http", url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/org/jetbrains/apple-notary-api-kotlin-client/1.0.5/apple-notary-api-kotlin-client-1.0.5.jar", @@ -11619,6 +11535,62 @@ http_file( downloaded_file_path = "poko-compiler-plugin-0.18.0-sources.jar" ) +http_file( + name = "proguard-base-7_6_1_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-base/7.6.1/proguard-base-7.6.1.jar", + sha256 = "c2650e666c3443781dc5e9eaee5de8721593f00be44e9d87cec761406357c770", + downloaded_file_path = "proguard-base-7.6.1.jar" +) + +http_file( + name = "proguard-core-9_1_7_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-core/9.1.7/proguard-core-9.1.7.jar", + sha256 = "db06fa9f6b43bb7d386420f854f41aa55a11023de36bad68e043c3ac3f33b8d5", + downloaded_file_path = "proguard-core-9.1.7.jar" +) + +http_file( + name = "log4j-api-2_24_2_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.24.2/log4j-api-2.24.2.jar", + sha256 = "0ca3ecbd4c315bdd5f2ef6af127712df718c78334cce5bf6fb7b4aa17fdab126", + downloaded_file_path = "log4j-api-2.24.2.jar" +) + +http_file( + name = "log4j-core-2_24_2_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.24.2/log4j-core-2.24.2.jar", + sha256 = "7a7b90db866c86a1093b3fde758ca28398ebb2a533da0d79a30cc0a78506b9df", + downloaded_file_path = "log4j-core-2.24.2.jar" +) + +http_file( + name = "proguard-base-7_6_1-sources_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-base/7.6.1/proguard-base-7.6.1-sources.jar", + sha256 = "9aed0e666f457f755e0756176821ef1d31a928790450e8e18f5bb6dcd4a9bac5", + downloaded_file_path = "proguard-base-7.6.1-sources.jar" +) + +http_file( + name = "proguard-core-9_1_7-sources_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-core/9.1.7/proguard-core-9.1.7-sources.jar", + sha256 = "495063fbf5a1989435ee15ed6c6caf39b7dc7af22dd495e38fc7e6e7c00f1fcc", + downloaded_file_path = "proguard-core-9.1.7-sources.jar" +) + +http_file( + name = "log4j-api-2_24_2-sources_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.24.2/log4j-api-2.24.2-sources.jar", + sha256 = "c17fb95a3ac61a8ed77fa13170f0081f0e6cf4966fb747752e9c0afe3b9b5b15", + downloaded_file_path = "log4j-api-2.24.2-sources.jar" +) + +http_file( + name = "log4j-core-2_24_2-sources_http", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.24.2/log4j-core-2.24.2-sources.jar", + sha256 = "ac9d7678ca2550758ddc2034af8f2fb97c36bf8ef2bf71d5ab22a6d3cf04c18e", + downloaded_file_path = "log4j-core-2.24.2-sources.jar" +) + http_file( name = "protobuf-java-3_24_4-jb_2_http", url = "https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-dependencies/com/google/protobuf/protobuf-java/3.24.4-jb.2/protobuf-java-3.24.4-jb.2.jar", diff --git a/platform/build-scripts/codeOptimizer/BUILD.bazel b/platform/build-scripts/codeOptimizer/BUILD.bazel index bc8dfd7253d7..de661a287f0e 100644 --- a/platform/build-scripts/codeOptimizer/BUILD.bazel +++ b/platform/build-scripts/codeOptimizer/BUILD.bazel @@ -7,7 +7,8 @@ jvm_test( srcs = glob(["src/**/*.kt", "src/**/*.java"], allow_empty = True), deps = [ "@lib//:kotlin-stdlib", - "@lib//:platform-build_scripts-code_optimizer-proguard", - ] + "@lib//:proguard", + ], + runtime_deps = ["@lib//:kotlin-metadata"] ) ### auto-generated section `build intellij.platform.buildScripts.codeOptimizer` end \ No newline at end of file