From e16162f1db38d076bee489a119f47bd6fe3da732 Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Tue, 4 Feb 2025 07:38:14 +0100 Subject: [PATCH] jps bazel compiler - part 13 GitOrigin-RevId: 42b64017e196b527fd2281de3da749a2ec35ab62 --- .bazelversion | 2 +- MODULE.bazel | 2 +- build/jvm-rules/.bazelrc | 5 +- build/jvm-rules/.bazelversion | 2 +- build/jvm-rules/BUILD.bazel | 24 +- build/jvm-rules/MODULE.bazel | 67 ++- build/jvm-rules/MODULE.bazel.lock | 22 +- build/jvm-rules/jaeger-config.yaml | 57 ++ build/jvm-rules/jaeger.sh | 20 + build/jvm-rules/libs.lock.json | 18 +- build/jvm-rules/libs.yaml | 14 +- build/jvm-rules/rules/impl/builder-args.bzl | 2 +- build/jvm-rules/rules/impl/compile.bzl | 68 +-- build/jvm-rules/src/abi/BUILD.bazel | 1 - .../src/compiler-util/JvmBuilderFlags.kt | 4 +- .../jps-builder-test/TestJpsBuildWorker.kt | 19 +- build/jvm-rules/src/jps-builder/BUILD.bazel | 2 + .../src/jps-builder/BazelJpsModelLoader.kt | 4 +- .../src/jps-builder/BazelJpsServiceManager.kt | 3 +- build/jvm-rules/src/jps-builder/JpsBuilder.kt | 226 ++++---- .../jps-builder/JpsKotlinCompilerRunner.kt | 195 +++++++ .../src/jps-builder/impl/BazelCompileScope.kt | 130 +++++ .../jps-builder/impl/BazelKotlinBuilder.kt | 150 ++---- .../src/jps-builder/impl/JavaBuilder.kt | 133 +---- .../src/jps-builder/impl/JpsTargetBuilder.kt | 37 +- .../src/jps-builder/impl/RequestLog.kt | 9 + .../jps-builder/state/BuildStateStorage.kt | 5 +- .../src/kotlin-builder/KotlinBuilder.kt | 3 +- .../kotlin-builder/KotlinJvmTaskExecutor.kt | 2 +- build/jvm-rules/src/misc/JvmWorker.kt | 3 +- .../src/worker-framework/BUILD.bazel | 1 + .../worker-framework/WorkRequestHandler.kt | 489 +++++++++--------- .../WorkRequestHandlerTest.kt | 42 +- .../src/worker-framework/testHelper.kt | 8 +- .../jvm-rules/src/worker-framework/tracing.kt | 35 +- lib/.bazelversion | 2 +- lib/MODULE.bazel | 2 +- 37 files changed, 1095 insertions(+), 713 deletions(-) create mode 100644 build/jvm-rules/jaeger-config.yaml create mode 100644 build/jvm-rules/jaeger.sh create mode 100644 build/jvm-rules/src/jps-builder/JpsKotlinCompilerRunner.kt create mode 100644 build/jvm-rules/src/jps-builder/impl/BazelCompileScope.kt diff --git a/.bazelversion b/.bazelversion index 5210382a7a62..51be2ce6c7ed 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.0.1 \ No newline at end of file +8.1.0rc1 \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel index 40c055c820cf..be89ddb6f752 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,7 +1,7 @@ module(name = "community") bazel_dep(name = "rules_kotlin", version = "2.0.0-jb.6") -bazel_dep(name = "rules_java", version = "8.7.1") +bazel_dep(name = "rules_java", version = "8.8.0") bazel_dep(name = "rules_jvm", version = "0.0.1") bazel_dep(name = "lib") diff --git a/build/jvm-rules/.bazelrc b/build/jvm-rules/.bazelrc index f3d0ee30a190..63ad13c020d8 100644 --- a/build/jvm-rules/.bazelrc +++ b/build/jvm-rules/.bazelrc @@ -21,4 +21,7 @@ test --test_verbose_timeout_warnings --test_timeout=60 --test_output=all # https://www.buildbuddy.io/blog/debugging-slow-bazel-builds/ common --nolegacy_important_outputs --remote_build_event_upload=minimal --remote_cache_async --remote_cache_compression -build --disk_cache=~/.bazel-ij-cache --experimental_disk_cache_gc_max_size=32G --experimental_disk_cache_gc_max_age=28d \ No newline at end of file +build --disk_cache=~/.bazel-ij-cache --experimental_disk_cache_gc_max_size=32G --experimental_disk_cache_gc_max_age=28d + +# protobuf +common --incompatible_enable_proto_toolchain_resolution \ No newline at end of file diff --git a/build/jvm-rules/.bazelversion b/build/jvm-rules/.bazelversion index 5210382a7a62..51be2ce6c7ed 100644 --- a/build/jvm-rules/.bazelversion +++ b/build/jvm-rules/.bazelversion @@ -1 +1 @@ -8.0.1 \ No newline at end of file +8.1.0rc1 \ No newline at end of file diff --git a/build/jvm-rules/BUILD.bazel b/build/jvm-rules/BUILD.bazel index ac76e66aaebc..ee645a4c68d6 100644 --- a/build/jvm-rules/BUILD.bazel +++ b/build/jvm-rules/BUILD.bazel @@ -185,9 +185,27 @@ jvm_import( ) jvm_import( - name = "proguard-core", - jar = "@proguard-core//file", - source_jar = "@proguard-core-sources//file", + name = "opentelemetry-sdk-extension-autoconfigure-spi", + jar = "@opentelemetry-sdk-extension-autoconfigure-spi//file", + source_jar = "@opentelemetry-sdk-extension-autoconfigure-spi-sources//file", + visibility = ["//visibility:public"], +) + +jvm_import( + name = "opentelemetry-exporter-otlp", + jar = "@opentelemetry-exporter-otlp//file", + source_jar = "@opentelemetry-exporter-otlp-sources//file", + visibility = ["//visibility:public"], + runtime_deps = [ + ":opentelemetry-exporter-sender-jdk", + ":opentelemetry-sdk-extension-autoconfigure-spi" + ], +) + +jvm_import( + name = "opentelemetry-exporter-sender-jdk", + jar = "@opentelemetry-exporter-sender-jdk//file", + source_jar = "@opentelemetry-exporter-sender-jdk-sources//file", visibility = ["//visibility:public"], ) diff --git a/build/jvm-rules/MODULE.bazel b/build/jvm-rules/MODULE.bazel index 11b0c9990610..0a56bf601fc2 100644 --- a/build/jvm-rules/MODULE.bazel +++ b/build/jvm-rules/MODULE.bazel @@ -1,12 +1,21 @@ module(name = "rules_jvm", version = "0.1") bazel_dep(name = "bazel_worker_api", version = "0.0.4") -bazel_dep(name = "rules_java", version = "8.7.1") +bazel_dep(name = "rules_java", version = "8.8.0") bazel_dep(name = "rules_kotlin", version = "2.0.0-jb.5") # bazel_dep(name = "rules_graalvm", version = "0.11.3") bazel_dep(name = "lib") -bazel_dep(name = "protobuf", version = "29.2") +bazel_dep(name = "protobuf", version = "29.3") bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "toolchains_protoc", version = "0.3.7") + +protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc") +protoc.toolchain( + google_protobuf = "com_google_protobuf", + version = "v29.3", +) +use_repo(protoc, "com_google_protobuf", "toolchains_protoc_hub") +register_toolchains("@toolchains_protoc_hub//:all") http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") @@ -209,15 +218,15 @@ http_file( http_file( name = "flatbuffers-java", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/google/flatbuffers/flatbuffers-java/24.12.23/flatbuffers-java-24.12.23.jar", - sha256 = "67c87e2e588734cf0166423e89ca99761479a580f1bd959c052b82a2546440e8", - downloaded_file_path = "flatbuffers-java-24.12.23.jar", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/google/flatbuffers/flatbuffers-java/25.1.24/flatbuffers-java-25.1.24.jar", + sha256 = "dbc103dfd4eb25664f87492968d8e01812c13cf939e5b72db2af38bfce530af3", + downloaded_file_path = "flatbuffers-java-25.1.24.jar", ) http_file( name = "flatbuffers-java-sources", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/google/flatbuffers/flatbuffers-java/24.12.23/flatbuffers-java-24.12.23-sources.jar", - sha256 = "822f6a95531ec757019d3e3e8af68885b0a682e53825d571f7cecf0c71b4edde", - downloaded_file_path = "flatbuffers-java-24.12.23-sources.jar", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/google/flatbuffers/flatbuffers-java/25.1.24/flatbuffers-java-25.1.24-sources.jar", + sha256 = "5cd75b7500b07f93c5308ffd8d430aa0b62cd162dff7a5394d5e513cf3a7d534", + downloaded_file_path = "flatbuffers-java-25.1.24-sources.jar", ) http_file( @@ -312,16 +321,42 @@ http_file( ) http_file( - name = "proguard-core", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-core/9.1.7/proguard-core-9.1.7.jar", - sha256 = "db06fa9f6b43bb7d386420f854f41aa55a11023de36bad68e043c3ac3f33b8d5", - downloaded_file_path = "proguard-core-9.1.7.jar", + name = "opentelemetry-sdk-extension-autoconfigure-spi", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-extension-autoconfigure-spi/1.46.0/opentelemetry-sdk-extension-autoconfigure-spi-1.46.0.jar", + sha256 = "ffe8ac4ddce31b13eab1c6280dc18dcf734fbbeb0cf86feacc084ac7ca3f6725", + downloaded_file_path = "opentelemetry-sdk-extension-autoconfigure-spi-1.46.0.jar", ) http_file( - name = "proguard-core-sources", - url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/com/guardsquare/proguard-core/9.1.7/proguard-core-9.1.7-sources.jar", - sha256 = "495063fbf5a1989435ee15ed6c6caf39b7dc7af22dd495e38fc7e6e7c00f1fcc", - downloaded_file_path = "proguard-core-9.1.7-sources.jar", + name = "opentelemetry-sdk-extension-autoconfigure-spi-sources", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/io/opentelemetry/opentelemetry-sdk-extension-autoconfigure-spi/1.46.0/opentelemetry-sdk-extension-autoconfigure-spi-1.46.0-sources.jar", + sha256 = "d3f7b7f5b2041a7c7d3f31b3619336f4c72f945d89e5c90a83292e308431fe67", + downloaded_file_path = "opentelemetry-sdk-extension-autoconfigure-spi-1.46.0-sources.jar", +) + +http_file( + name = "opentelemetry-exporter-otlp", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-otlp/1.46.0/opentelemetry-exporter-otlp-1.46.0.jar", + sha256 = "9df3776f9bb2ca5e63058006931b2806c177cd74e77e2875fdc54a49302ba733", + downloaded_file_path = "opentelemetry-exporter-otlp-1.46.0.jar", +) +http_file( + name = "opentelemetry-exporter-otlp-sources", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-otlp/1.46.0/opentelemetry-exporter-otlp-1.46.0-sources.jar", + sha256 = "1703983de4b3820336f2e2b55f9cd4e2323996863e0c8ba59d44397d780a7d22", + downloaded_file_path = "opentelemetry-exporter-otlp-1.46.0-sources.jar", +) + +http_file( + name = "opentelemetry-exporter-sender-jdk", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-sender-jdk/1.46.0/opentelemetry-exporter-sender-jdk-1.46.0.jar", + sha256 = "ae7f7b4ec65b814e45f8388863ccf453d3ae0c4fc513c8dc6127bec4507524a0", + downloaded_file_path = "opentelemetry-exporter-sender-jdk-1.46.0.jar", +) +http_file( + name = "opentelemetry-exporter-sender-jdk-sources", + url = "https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/io/opentelemetry/opentelemetry-exporter-sender-jdk/1.46.0/opentelemetry-exporter-sender-jdk-1.46.0-sources.jar", + sha256 = "13eb527901c0d3e0b1760f1f5ade26b340f61a31ff72a6e5c0fcafb2c81206d7", + downloaded_file_path = "opentelemetry-exporter-sender-jdk-1.46.0-sources.jar", ) ### auto-generated section `libraries` end diff --git a/build/jvm-rules/MODULE.bazel.lock b/build/jvm-rules/MODULE.bazel.lock index 41ec95149064..6f37bc1031e1 100644 --- a/build/jvm-rules/MODULE.bazel.lock +++ b/build/jvm-rules/MODULE.bazel.lock @@ -1,5 +1,5 @@ { - "lockFileVersion": 16, + "lockFileVersion": 18, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", @@ -20,6 +20,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", "https://bcr.bazel.build/modules/bazel_features/1.21.0/source.json": "3e8379efaaef53ce35b7b8ba419df829315a880cb0a030e5bb45c96d6d5ecb5f", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", @@ -61,8 +62,9 @@ "https://bcr.bazel.build/modules/protobuf/27.2/MODULE.bazel": "32450b50673882e4c8c3d10a83f3bc82161b213ed2f80d17e38bece8f165c295", "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", - "https://bcr.bazel.build/modules/protobuf/29.2/MODULE.bazel": "5435497c190d86f79b0568698c45044df7c8d97692886cda9fe9cf9053aea712", - "https://bcr.bazel.build/modules/protobuf/29.2/source.json": "fe7090cc34072609b26d9beafb122916dabc1d47ba61b242c26c4b06c51384ab", + "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", + "https://bcr.bazel.build/modules/protobuf/29.3/MODULE.bazel": "77480eea5fb5541903e49683f24dc3e09f4a79e0eea247414887bb9fc0066e94", + "https://bcr.bazel.build/modules/protobuf/29.3/source.json": "c460e6550ddd24996232c7542ebf201f73c4e01d2183a31a041035fb50f19681", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022", @@ -76,7 +78,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.14/MODULE.bazel": "5e343a3aac88b8d7af3b1b6d2093b55c347b8eefc2e7d1442f7a02dc8fea48ac", "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc", "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", - "https://bcr.bazel.build/modules/rules_cc/0.0.16/source.json": "227e83737046aa4f50015da48e98e0d8ab42fd0ec74d8d653b6cc9f9a357f200", + "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", + "https://bcr.bazel.build/modules/rules_cc/0.0.17/source.json": "4db99b3f55c90ab28d14552aa0632533e3e8e5e9aea0f5c24ac0014282c2a7c5", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", @@ -94,8 +97,8 @@ "https://bcr.bazel.build/modules/rules_java/7.4.0/MODULE.bazel": "a592852f8a3dd539e82ee6542013bf2cadfc4c6946be8941e189d224500a8934", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", - "https://bcr.bazel.build/modules/rules_java/8.7.1/MODULE.bazel": "123a57f84c7f80d6f66b0c2486db3460ed8c4389f788ccbd35bb489b1ab23634", - "https://bcr.bazel.build/modules/rules_java/8.7.1/source.json": "3a98d057e5638a980e0b9e3a8f1cdb798f8b377b6016fb455d132ea2aa4ea41e", + "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", + "https://bcr.bazel.build/modules/rules_java/8.8.0/source.json": "63c0c14a917082cddc3956eedbf2098410651e75fc7d29aa3e732c30a0099458", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", @@ -114,7 +117,8 @@ "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", - "https://bcr.bazel.build/modules/rules_proto/7.0.2/source.json": "1e5e7260ae32ef4f2b52fd1d0de8d03b606a44c91b694d2f1afb1d3b28a48ce1", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", @@ -130,6 +134,8 @@ "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", "https://bcr.bazel.build/modules/stardoc/0.7.1/source.json": "b6500ffcd7b48cd72c29bb67bcac781e12701cc0d6d55d266a652583cfcdab01", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/MODULE.bazel": "d2758e5af5838970175274c4fcb67edc8d2f4f8f348885da90c42fafc221236d", + "https://bcr.bazel.build/modules/toolchains_protoc/0.3.7/source.json": "07e0eab9be4aded2a8074504a80c55c5a48e811a9824061642af639baaabf985", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", @@ -354,7 +360,7 @@ }, "@@rules_python+//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "+zEWYyNfKsf13xNoIs5u4bH4EEgN37u2T/f35J7yWKI=", + "bzlTransitiveDigest": "RmAvYYQQXxke38mrivExLNDhbFDbXKKstfbUwaEXgt0=", "usagesDigest": "OLoIStnzNObNalKEMRq99FqenhPGLFZ5utVLV4sz7OI=", "recordedFileInputs": { "@@rules_python+//tools/publish/requirements_darwin.txt": "2994136eab7e57b083c3de76faf46f70fad130bc8e7360a7fed2b288b69e79dc", diff --git a/build/jvm-rules/jaeger-config.yaml b/build/jvm-rules/jaeger-config.yaml new file mode 100644 index 000000000000..de6e17a96b87 --- /dev/null +++ b/build/jvm-rules/jaeger-config.yaml @@ -0,0 +1,57 @@ +service: + extensions: [jaeger_storage, jaeger_query, healthcheckv2] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [jaeger_storage_exporter] + telemetry: + resource: + service.name: jaeger + metrics: + level: detailed + address: 0.0.0.0:8888 + logs: + level: info + # TODO Initialize telemetry tracer once OTEL released new feature. + # https://github.com/open-telemetry/opentelemetry-collector/issues/10663 + +extensions: + healthcheckv2: + use_v2: true + http: + + jaeger_query: + storage: + traces: some_store + traces_archive: another_store + ui: + #config_file: ./cmd/jaeger/config-ui.json + + jaeger_storage: + backends: + some_store: + badger: + directories: + keys: "/tmp/jaeger/" + values: "/tmp/jaeger/" + ephemeral: false + another_store: + badger: + directories: + keys: "/tmp/jaeger_archive/" + values: "/tmp/jaeger_archive/" + ephemeral: false + +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + +exporters: + jaeger_storage_exporter: + trace_storage: some_store \ No newline at end of file diff --git a/build/jvm-rules/jaeger.sh b/build/jvm-rules/jaeger.sh new file mode 100644 index 000000000000..7b52671fe07f --- /dev/null +++ b/build/jvm-rules/jaeger.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +TARBALL_NAME="jaeger-2.2.0-darwin-amd64.tar.gz" +JAERGER_URL="https://github.com/jaegertracing/jaeger/releases/download/v1.65.0/$TARBALL_NAME" +EXTRACT_DIR="$HOME/Applications/jaeger" + +handle_error() { + echo "An error occurred. Exiting..." + exit 1 +} + +curl -L -o "$TARBALL_NAME" "$JAERGER_URL" || handle_error + +if [ ! -d "$EXTRACT_DIR" ]; then + mkdir -p "$EXTRACT_DIR" || handle_error +fi + +tar -xzf "$TARBALL_NAME" -C "$EXTRACT_DIR" || handle_error +rm "$TARBALL_NAME" || handle_error diff --git a/build/jvm-rules/libs.lock.json b/build/jvm-rules/libs.lock.json index 303b916d5880..ccac3fe20c5a 100644 --- a/build/jvm-rules/libs.lock.json +++ b/build/jvm-rules/libs.lock.json @@ -1,7 +1,7 @@ { - "com.google.flatbuffers:flatbuffers-java:24.12.23": [ - "67c87e2e588734cf0166423e89ca99761479a580f1bd959c052b82a2546440e8", - "822f6a95531ec757019d3e3e8af68885b0a682e53825d571f7cecf0c71b4edde" + "com.google.flatbuffers:flatbuffers-java:25.1.24": [ + "dbc103dfd4eb25664f87492968d8e01812c13cf939e5b72db2af38bfce530af3", + "5cd75b7500b07f93c5308ffd8d430aa0b62cd162dff7a5394d5e513cf3a7d534" ], "com.google.protobuf:protobuf-java:4.29.3": [ "442db5991a11974d72127353be7d7e7abdf8d943d83b16668cb2e336d7392f54", @@ -15,6 +15,18 @@ "afbed5afecd48ebc66ffe16c438cd92da31759af56bdcbde0f260b131e43042a", "10c69d372df6c0e80486edb3cbd4191b75a99325e5faba10406242b3e609311e" ], + "io.opentelemetry:opentelemetry-exporter-otlp:1.46.0": [ + "9df3776f9bb2ca5e63058006931b2806c177cd74e77e2875fdc54a49302ba733", + "1703983de4b3820336f2e2b55f9cd4e2323996863e0c8ba59d44397d780a7d22" + ], + "io.opentelemetry:opentelemetry-exporter-sender-jdk:1.46.0": [ + "ae7f7b4ec65b814e45f8388863ccf453d3ae0c4fc513c8dc6127bec4507524a0", + "13eb527901c0d3e0b1760f1f5ade26b340f61a31ff72a6e5c0fcafb2c81206d7" + ], + "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.46.0": [ + "ffe8ac4ddce31b13eab1c6280dc18dcf734fbbeb0cf86feacc084ac7ca3f6725", + "d3f7b7f5b2041a7c7d3f31b3619336f4c72f945d89e5c90a83292e308431fe67" + ], "org.apache.arrow:arrow-algorithm:18.1.0": [ "8627d00c1fa3341ff49d894fb88f003e9a7ca4a3846527e31d881c162cf146ff", "d5c6c4ed38c33a55e126f86bccfb4d7c87341d23b5f820392b332e011d22c7a0" diff --git a/build/jvm-rules/libs.yaml b/build/jvm-rules/libs.yaml index 01d9c420f530..4d5530ed0e47 100644 --- a/build/jvm-rules/libs.yaml +++ b/build/jvm-rules/libs.yaml @@ -31,7 +31,7 @@ - id: org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm version: 1.10.1 - id: com.google.flatbuffers:flatbuffers-java - version: 24.12.23 + version: 25.1.24 - id: org.apache.arrow:arrow-format version: 18.1.0 - id: org.apache.arrow:arrow-algorithm @@ -49,7 +49,15 @@ - ":arrow-memory-netty-buffer-patch" - id: org.apache.arrow:arrow-memory-netty-buffer-patch version: 18.1.0 + # OpenTelemetry - id: io.opentelemetry:opentelemetry-exporter-logging-otlp version: 1.46.0 - - id: com.guardsquare:proguard-core - version: 9.1.7 \ No newline at end of file + - id: io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi + version: 1.46.0 + - id: io.opentelemetry:opentelemetry-exporter-otlp + runtimeDeps: + - ":opentelemetry-exporter-sender-jdk" + - ":opentelemetry-sdk-extension-autoconfigure-spi" + version: 1.46.0 + - id: io.opentelemetry:opentelemetry-exporter-sender-jdk + version: 1.46.0 \ No newline at end of file diff --git a/build/jvm-rules/rules/impl/builder-args.bzl b/build/jvm-rules/rules/impl/builder-args.bzl index ed7f29ac7e96..7cdcdacaf801 100644 --- a/build/jvm-rules/rules/impl/builder-args.bzl +++ b/build/jvm-rules/rules/impl/builder-args.bzl @@ -31,7 +31,7 @@ def init_builder_args(ctx, rule_kind, associates, transitiveInputs, plugins, com kotlinc_options_to_flags(kotlinc_options, args) args.add_all("--opt-in", kotlinc_options.opt_in) - args.add_all("--classpath", compile_deps.compile_jars) + args.add_all("--cp", compile_deps.compile_jars) if ctx.attr._reduced_classpath: args.add("--reduced-classpath-mode", "true") diff --git a/build/jvm-rules/rules/impl/compile.bzl b/build/jvm-rules/rules/impl/compile.bzl index 056074a5f334..e3d4c1bed301 100644 --- a/build/jvm-rules/rules/impl/compile.bzl +++ b/build/jvm-rules/rules/impl/compile.bzl @@ -271,7 +271,6 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): perTargetPlugins = ctx.attr.plugins if hasattr(ctx.attr, "plugins") else [] plugins = _new_plugins_from(perTargetPlugins + _exported_plugins(deps = ctx.attr.deps)) - # merge outputs into final runtime jar output_jar = ctx.actions.declare_file(ctx.label.name + ".jar") outputs_struct = None @@ -284,32 +283,34 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): _collect_runtime_jars(perTargetPlugins, transitiveInputs) _collect_runtime_jars(ctx.attr.deps, transitiveInputs) - if jps_threshold != -1 and len(srcs.kt) >= jps_threshold: - outputs_struct = _run_jps_builder( - ctx = ctx, - output_jar = output_jar, - rule_kind = rule_kind, - toolchains = toolchains, - srcs = srcs, - associates = associates, - compile_deps = compile_deps, - transitiveInputs = transitiveInputs, - plugins = plugins, - ) - else: - outputs_struct = _run_kt_java_builder_actions( - ctx = ctx, - output_jar = output_jar, - rule_kind = rule_kind, - toolchains = toolchains, - srcs = srcs, - generated_ksp_src_jars = [], - associates = associates, - compile_deps = compile_deps, - annotation_processors = [], - transitiveInputs = transitiveInputs, - plugins = plugins, - ) + isIncremental = jps_threshold != -1 and len(srcs.kt) >= jps_threshold +# if jps_threshold == -2 or (jps_threshold != -1 and len(srcs.kt) >= jps_threshold): + outputs_struct = _run_jps_builder( + ctx = ctx, + isIncremental = isIncremental, + output_jar = output_jar, + rule_kind = rule_kind, + toolchains = toolchains, + srcs = srcs, + associates = associates, + compile_deps = compile_deps, + transitiveInputs = transitiveInputs, + plugins = plugins, + ) +# else: +# outputs_struct = _run_kt_java_builder_actions( +# ctx = ctx, +# output_jar = output_jar, +# rule_kind = rule_kind, +# toolchains = toolchains, +# srcs = srcs, +# generated_ksp_src_jars = [], +# associates = associates, +# compile_deps = compile_deps, +# annotation_processors = [], +# transitiveInputs = transitiveInputs, +# plugins = plugins, +# ) compile_jar = outputs_struct.compile_jar generated_src_jars = outputs_struct.generated_src_jars @@ -569,6 +570,7 @@ def _run_kt_java_builder_actions( def _run_jps_builder( ctx, + isIncremental, output_jar, rule_kind, toolchains, @@ -603,6 +605,9 @@ def _run_jps_builder( outputs.append(jdeps) args.add("--jdeps-out", jdeps) + if isIncremental: + args.add("--incremental", jdeps) + javac_opts = ctx.attr.javac_opts[JavacOptions] if ctx.attr.javac_opts else None if javac_opts and javac_opts.add_exports: args.add_all("--add-export", javac_opts.add_exports) @@ -617,14 +622,11 @@ def _run_jps_builder( "supports-workers": "1", "supports-multiplex-workers": "1", "supports-worker-cancellation": "1", -# "supports-path-mapping": "1", -# "supports-multiplex-sandboxing": "1", + "supports-path-mapping": "1", + "supports-multiplex-sandboxing": "1", }, arguments = [args], - progress_message = "Incremental compile %%{label} { kt: %d, java: %d }" % (len(srcs.kt), len(srcs.java)), - env = { - "LC_CTYPE": "en_US.UTF-8", - }, + progress_message = "compile %%{label} { kt: %d, java: %d%s}" % (len(srcs.kt), len(srcs.java), ", incremental" if isIncremental else ""), ) return struct( diff --git a/build/jvm-rules/src/abi/BUILD.bazel b/build/jvm-rules/src/abi/BUILD.bazel index 8956015eed8e..3984beba2ff4 100644 --- a/build/jvm-rules/src/abi/BUILD.bazel +++ b/build/jvm-rules/src/abi/BUILD.bazel @@ -9,7 +9,6 @@ kt_jvm_library( "@lib//:kotlin-stdlib", "//:kotlinx-coroutines-core", "@lib//:asm", - "//:proguard-core", "//:kotlin-metadata", "//zip:build-zip", ], diff --git a/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt b/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt index a368317c7ec6..600ac1d1325c 100644 --- a/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt +++ b/build/jvm-rules/src/compiler-util/JvmBuilderFlags.kt @@ -22,8 +22,10 @@ fun parseArgs(args: Array): ArgMap { } enum class JvmBuilderFlags { + INCREMENTAL, TARGET_LABEL, - CLASSPATH, + // classpath + CP, DIRECT_DEPENDENCIES, DEPS_ARTIFACTS, diff --git a/build/jvm-rules/src/jps-builder-test/TestJpsBuildWorker.kt b/build/jvm-rules/src/jps-builder-test/TestJpsBuildWorker.kt index f64eb8fc61d4..f9001311d9d5 100644 --- a/build/jvm-rules/src/jps-builder-test/TestJpsBuildWorker.kt +++ b/build/jvm-rules/src/jps-builder-test/TestJpsBuildWorker.kt @@ -3,7 +3,6 @@ package org.jetbrains.bazel.jvm.jps.test -import io.opentelemetry.context.Context import org.apache.arrow.memory.RootAllocator import org.jetbrains.bazel.jvm.TestModules import org.jetbrains.bazel.jvm.collectSources @@ -14,7 +13,7 @@ import org.jetbrains.bazel.jvm.jps.configureGlobalJps import org.jetbrains.bazel.jvm.kotlin.JvmBuilderFlags import org.jetbrains.bazel.jvm.kotlin.parseArgs import org.jetbrains.bazel.jvm.performTestInvocation -import org.jetbrains.bazel.jvm.use +import org.jetbrains.bazel.jvm.span import java.nio.file.Files import java.security.MessageDigest import kotlin.io.path.ExperimentalPathApi @@ -26,7 +25,7 @@ internal object TestJpsBuildWorker { val testPaths = getTestWorkerPaths() val baseDir = testPaths.baseDir - val testModule = TestModules.PLATFORM_IMPL + val testModule = TestModules.STAT_CONFIG val sources = collectSources(sourceDirPath = testModule.sourcePath, paths = testPaths) val testParams = testModule.getParams(baseDir) @@ -35,19 +34,20 @@ internal object TestJpsBuildWorker { // ${dateTimeFormatter.format(LocalDateTime.now())} val logFile = testPaths.userHomeDir.resolve("kotlin-worker/log.jsonl") Files.createDirectories(logFile.parent) - val tracer = configureOpenTelemetry(Files.newOutputStream(logFile), "test-builder").getTracer("test-builder") + val openTelemetryAndOnClose = configureOpenTelemetry(Files.newOutputStream(logFile), "test-builder") + val tracer = openTelemetryAndOnClose.first.getTracer("test-builder") configureGlobalJps(tracer, coroutineScope) val args = parseArgs(testParams.lines().toTypedArray()) val messageDigest = MessageDigest.getInstance("SHA-512") - RootAllocator(Long.MAX_VALUE).use { allocator -> - tracer.spanBuilder("build").use { span -> + val exitCode = RootAllocator(Long.MAX_VALUE).use { allocator -> + tracer.span("build") { span -> buildUsingJps( baseDir = baseDir, args = args, out = out, sources = sources, - dependencyFileToDigest = args.optionalList(JvmBuilderFlags.CLASSPATH).associate { + dependencyFileToDigest = args.optionalList(JvmBuilderFlags.CP).associate { val file = baseDir.resolve(it).normalize() val digest = messageDigest.digest(Files.readAllBytes(file)) messageDigest.reset() @@ -61,13 +61,16 @@ internal object TestJpsBuildWorker { }, isDebugEnabled = true, allocator = allocator, - tracingContext = Context.current(), parentSpan = span, tracer = tracer, cachePrefix = "test-builder-", ) } } + + openTelemetryAndOnClose.second() + + exitCode } } } \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/BUILD.bazel b/build/jvm-rules/src/jps-builder/BUILD.bazel index 412eca209071..4aa36b766873 100644 --- a/build/jvm-rules/src/jps-builder/BUILD.bazel +++ b/build/jvm-rules/src/jps-builder/BUILD.bazel @@ -67,6 +67,7 @@ java_binary( data = [ "@kotlinc//:kotlinc_dist", "@kotlin-serialization-compiler-plugin//file", + "@kotlin-compose-compiler-plugin//file", ], main_class = "org.jetbrains.bazel.jvm.jps.JpsBuildWorker", jvm_flags = get_jvm_flags([ @@ -74,6 +75,7 @@ java_binary( "-Djps.use.experimental.storage=true", "-Djps.kotlin.home=$(rlocationpath @kotlinc//:kotlinc_dist)", "-Dorg.jetbrains.kotlin.kotlin-serialization-compiler-plugin.path=$(rlocationpath @kotlin-serialization-compiler-plugin//file)", + "-Dorg.jetbrains.kotlin.kotlin-compose-compiler-plugin.path=$(rlocationpath @kotlin-serialization-compiler-plugin//file)", ]), visibility = ["//visibility:public"], ) \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt b/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt index 1bb95a5e3429..83ce025b7d4b 100644 --- a/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt +++ b/build/jvm-rules/src/jps-builder/BazelJpsModelLoader.kt @@ -185,8 +185,8 @@ private fun configureClasspath( dependencyFileToDigest: Map, digests: TargetConfigurationDigestContainer, ) { - // REDUCED_CLASSPATH_MODE is not supported for JPS - val classPathRaw = args.mandatory(JvmBuilderFlags.CLASSPATH) + // no classpath if no source file (jvm_test without own sources) + val classPathRaw = args.optionalList(JvmBuilderFlags.CP) val files = Array(classPathRaw.size) { baseDir.resolve(classPathRaw[it]).normalize() } diff --git a/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt b/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt index 82228ecb438d..ad3a3d00a960 100644 --- a/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt +++ b/build/jvm-rules/src/jps-builder/BazelJpsServiceManager.kt @@ -13,7 +13,6 @@ import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType import org.jetbrains.jps.incremental.BuilderService import org.jetbrains.jps.incremental.ModuleLevelBuilder import org.jetbrains.jps.model.* -import org.jetbrains.jps.model.java.JpsJavaModuleType import org.jetbrains.jps.service.JpsServiceManager import org.jetbrains.jps.service.SharedThreadPool import java.io.File @@ -57,7 +56,7 @@ internal class BazelJpsServiceManager : JpsServiceManager() { // confine costly service initialization to single thread for defined startup profile return synchronized(services) { - return services.computeIfAbsent(serviceClass) { + services.computeIfAbsent(serviceClass) { doComputeService(it) as T } as T } diff --git a/build/jvm-rules/src/jps-builder/JpsBuilder.kt b/build/jvm-rules/src/jps-builder/JpsBuilder.kt index b28bcc46260e..657340640c65 100644 --- a/build/jvm-rules/src/jps-builder/JpsBuilder.kt +++ b/build/jvm-rules/src/jps-builder/JpsBuilder.kt @@ -9,7 +9,6 @@ import com.intellij.openapi.util.io.FileUtilRt import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.Tracer -import io.opentelemetry.context.Context import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap import kotlinx.coroutines.* import org.apache.arrow.memory.RootAllocator @@ -26,12 +25,10 @@ import org.jetbrains.bazel.jvm.kotlin.ArgMap import org.jetbrains.bazel.jvm.kotlin.JvmBuilderFlags import org.jetbrains.bazel.jvm.kotlin.parseArgs import org.jetbrains.bazel.jvm.processRequests +import org.jetbrains.bazel.jvm.span import org.jetbrains.bazel.jvm.use import org.jetbrains.jps.api.GlobalOptions import org.jetbrains.jps.backwardRefs.JavaBackwardReferenceIndexBuilder -import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType -import org.jetbrains.jps.incremental.CompileContextImpl -import org.jetbrains.jps.incremental.CompileScopeImpl import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jps.incremental.RebuildRequestedException import org.jetbrains.jps.incremental.relativizer.PathRelativizerService @@ -41,7 +38,6 @@ import org.jetbrains.kotlin.jps.incremental.KotlinCompilerReferenceIndexBuilder import java.io.Writer import java.nio.file.Files import java.nio.file.Path -import java.util.Set import kotlin.coroutines.coroutineContext // if more than 50% files were changed, perform a full rebuild @@ -92,7 +88,12 @@ internal class JpsBuildWorker private constructor(private val allocator: RootAll } @OptIn(ExperimentalStdlibApi::class) - override suspend fun execute(request: WorkRequestWithDigests, writer: Writer, baseDir: Path, tracingContext: Context, tracer: Tracer): Int { + override suspend fun execute( + request: WorkRequestWithDigests, + writer: Writer, + baseDir: Path, + tracer: Tracer, + ): Int { val dependencyFileToDigest = hashMap() val sourceFileToDigest = hashMap(request.inputPaths.size) val sources = ArrayList() @@ -118,29 +119,28 @@ internal class JpsBuildWorker private constructor(private val allocator: RootAll } } - if (isDebugEnabled) { + return if (isDebugEnabled) { tracer.spanBuilder("build") - .setParent(tracingContext) .setAttribute(AttributeKey.stringKey("sourceFileToDigest"), sourceFileToDigestDebugString.toString()) .setAttribute(AttributeKey.stringKey("dependencyFileToDigest"), dependencyFileToDigestDebugString.toString()) } else { tracer.spanBuilder("build") - }.use { span -> - return buildUsingJps( - baseDir = baseDir, - args = parseArgs(request.arguments), - out = writer, - sources = sources, - dependencyFileToDigest = dependencyFileToDigest, - isDebugEnabled = isDebugEnabled, - sourceFileToDigest = sourceFileToDigest, - allocator = allocator, - parentSpan = span, - tracer = tracer, - tracingContext = tracingContext.with(span), - ) } + .use { span -> + buildUsingJps( + baseDir = baseDir, + args = parseArgs(request.arguments), + out = writer, + sources = sources, + dependencyFileToDigest = dependencyFileToDigest, + isDebugEnabled = isDebugEnabled, + sourceFileToDigest = sourceFileToDigest, + allocator = allocator, + parentSpan = span, + tracer = tracer, + ) + } } } @@ -164,7 +164,6 @@ suspend fun buildUsingJps( allocator: RootAllocator, parentSpan: Span, tracer: Tracer, - tracingContext: Context, cachePrefix: String = "", ): Int { val log = RequestLog(out = out, parentSpan = parentSpan, tracer = tracer) @@ -175,9 +174,10 @@ suspend fun buildUsingJps( val bazelOutDir = outJar.parent val dataDir = bazelOutDir.resolve("$cachePrefix$prefix-jps-data") - // incremental compilation - we do not clear dir val classOutDir = bazelOutDir.resolve("$cachePrefix$prefix-classes") + val isIncrementalCompilation = args.boolFlag(JvmBuilderFlags.INCREMENTAL) + val (jpsModel, targetDigests) = loadJpsModel( sources = sources, args = args, @@ -203,32 +203,34 @@ suspend fun buildUsingJps( // if class output dir doesn't exist, make sure that we do not to use existing cache - // set `isRebuild` to true and clear caches in this case - var isRebuild = false - if (Files.notExists(dataDir)) { - FileUtilRt.deleteRecursively(classOutDir) - isRebuild = true + var isRebuild = !isIncrementalCompilation + if (isIncrementalCompilation) { + if (Files.notExists(dataDir)) { + FileUtilRt.deleteRecursively(classOutDir) + isRebuild = true + } + else if (Files.notExists(classOutDir)) { + FileUtilRt.deleteRecursively(dataDir) + isRebuild = true + } } - else if (Files.notExists(classOutDir)) { + else { + FileUtilRt.deleteRecursively(classOutDir) FileUtilRt.deleteRecursively(dataDir) - isRebuild = true } val buildStateFile = dataDir.resolve("$prefix-state-v1.arrow") val typeAwareRelativizer = relativizer.typeAwareRelativizer!! - val buildState = tracer.spanBuilder("load and check state").setParent(tracingContext).use { parentSpan -> - val buildState = if (isRebuild) { - null - } - else { - loadBuildState( - buildStateFile = buildStateFile, - relativizer = typeAwareRelativizer, - allocator = allocator, - sourceFileToDigest = sourceFileToDigest, - targetDigests = targetDigests, - parentSpan = parentSpan, - ) - } + + fun computeBuildState(parentSpan: Span): LoadStateResult? { + val buildState = loadBuildState( + buildStateFile = buildStateFile, + relativizer = typeAwareRelativizer, + allocator = allocator, + sourceFileToDigest = sourceFileToDigest, + targetDigests = targetDigests, + parentSpan = parentSpan, + ) val forceFullRebuild = buildState != null && checkIsFullRebuildRequired( buildState = buildState, @@ -236,19 +238,24 @@ suspend fun buildUsingJps( sourceFileCount = sourceFileToDigest.size, parentSpan = parentSpan, ) - if (forceFullRebuild) { FileUtilRt.deleteRecursively(dataDir) FileUtilRt.deleteRecursively(classOutDir) isRebuild = true + return null } + else { + return buildState + } + } - buildState + val buildState = if (isRebuild) null else tracer.span("load and check state") { parentSpan -> + computeBuildState(parentSpan) } var exitCode = initAndBuild( - isRebuild = isRebuild, + compileScope = BazelCompileScope(isIncrementalCompilation = isIncrementalCompilation, isRebuild = isRebuild), messageHandler = log, dataDir = dataDir, classOutDir = classOutDir, @@ -265,14 +272,14 @@ suspend fun buildUsingJps( allocator = allocator, isCleanBuild = isRebuild, ), - buildState = buildState.takeIf { !isRebuild }, - tracingContext = tracingContext, + buildState = buildState, parentSpan = parentSpan, ) + if (exitCode == -1) { log.resetState() exitCode = initAndBuild( - isRebuild = true, + compileScope = BazelCompileScope(isIncrementalCompilation = isIncrementalCompilation, isRebuild = true), messageHandler = log, dataDir = dataDir, classOutDir = classOutDir, @@ -284,13 +291,12 @@ suspend fun buildUsingJps( jpsModel = jpsModel, buildDataProvider = BazelBuildDataProvider( relativizer = typeAwareRelativizer, - sourceToDescriptor = hashMap(sourceFileToDigest.size), + sourceToDescriptor = createInitialSourceMap(sourceFileToDigest), storeFile = buildStateFile, allocator = allocator, isCleanBuild = true, ), buildState = null, - tracingContext = tracingContext, parentSpan = parentSpan, ) } @@ -327,7 +333,7 @@ private fun checkIsFullRebuildRequired( } private suspend fun initAndBuild( - isRebuild: Boolean, + compileScope: BazelCompileScope, messageHandler: RequestLog, dataDir: Path, classOutDir: Path, @@ -339,12 +345,12 @@ private suspend fun initAndBuild( jpsModel: JpsModel, buildDataProvider: BazelBuildDataProvider, buildState: LoadStateResult?, - tracingContext: Context, parentSpan: Span, ): Int { + val isRebuild = compileScope.isRebuild val tracer = messageHandler.tracer val storageInitializer = StorageInitializer(dataDir = dataDir, classOutDir = classOutDir) - val storageManager = tracer.spanBuilder("init storage").setParent(tracingContext).use { span -> + val storageManager = tracer.span("init storage") { span -> if (isRebuild) { storageInitializer.clearAndInit(span) } @@ -362,30 +368,26 @@ private suspend fun initAndBuild( span = parentSpan, ) try { - val compileScope = CompileScopeImpl( - /* types = */ Set.of(JavaModuleBuildTargetType.PRODUCTION), - /* typesToForceBuild = */ Set.of(), - /* targets = */ if (isRebuild) Set.of(moduleTarget) else Set.of(), - /* files = */ java.util.Map.of() + val context = BazelCompileContext( + scope = compileScope, + projectDescriptor = projectDescriptor, + delegateMessageHandler = messageHandler, + coroutineContext = coroutineContext, ) - val coroutineContext = coroutineContext - val context = CompileContextImpl( - compileScope, - projectDescriptor, - messageHandler, - emptyMap(), - ) { !coroutineContext.isActive } - val exitCode = tracer.spanBuilder("compile") - .setParent(tracingContext) .setAttribute(AttributeKey.booleanKey("isRebuild"), isRebuild) .use { span -> val builders = arrayOf( JavaBuilder(span, messageHandler.out), //NotNullInstrumentingBuilder(), JavaBackwardReferenceIndexBuilder(), - BazelKotlinBuilder(isKotlinBuilderInDumbMode = false, span = span, dataManager = buildDataProvider), + BazelKotlinBuilder( + isIncrementalCompilation = compileScope.isIncrementalCompilation, + isKotlinBuilderInDumbMode = false, + span = span, + dataManager = buildDataProvider, + ), KotlinCompilerReferenceIndexBuilder(), ) builders.sortBy { it.category.ordinal } @@ -400,23 +402,24 @@ private suspend fun initAndBuild( moduleTarget = moduleTarget, builders = builders, buildState = buildState, - tracingContext = tracingContext.with(span), parentSpan = span, ) } try { - postBuild( - success = exitCode == 0, - moduleTarget = moduleTarget, - outJar = outJar, - abiJar = abiJar, - classOutDir = classOutDir, - context = context, - targetDigests = targetDigests, - buildDataProvider = buildDataProvider, - tracingContext = tracingContext, - tracer = tracer, - ) + coroutineScope { + postBuild( + success = exitCode == 0, + moduleTarget = moduleTarget, + outJar = outJar, + abiJar = abiJar, + classOutDir = classOutDir, + context = context, + targetDigests = targetDigests, + buildDataProvider = buildDataProvider, + tracer = tracer, + parentSpan = parentSpan, + ) + } } catch (e: Throwable) { // in case of any error during packaging - clear build @@ -445,21 +448,21 @@ private suspend fun initAndBuild( private val stateFileMetaNames: Array = TargetConfigurationDigestProperty.entries .let { entries -> Array(entries.size) { entries.get(it).name } } -private suspend fun postBuild( +private fun CoroutineScope.postBuild( moduleTarget: ModuleBuildTarget, outJar: Path, abiJar: Path?, classOutDir: Path, - context: CompileContextImpl, + context: BazelCompileContext, targetDigests: TargetConfigurationDigestContainer, buildDataProvider: BazelBuildDataProvider, - tracingContext: Context, tracer: Tracer, success: Boolean, + parentSpan: Span, ) { - coroutineScope { - val dataManager = context.projectDescriptor.dataManager - val sourceDescriptors = buildDataProvider.getFinalList() + val dataManager = context.projectDescriptor.dataManager + val sourceDescriptors = buildDataProvider.getFinalList() + if (context.scope.isIncrementalCompilation) { launch(CoroutineName("save caches")) { dataManager.flush(/* memoryCachesOnly = */ false) @@ -481,33 +484,34 @@ private suspend fun postBuild( allocator = buildDataProvider.allocator, ) } + } - launch { - // deletes class loader classpath index files for changed output roots - // todo remove when we will produce JAR directly - Files.deleteIfExists(classOutDir.resolve("classpath.index")) - Files.deleteIfExists(classOutDir.resolve(".unmodified")) - } + launch { + // deletes class loader classpath index files for changed output roots + // todo remove when we will produce JAR directly + Files.deleteIfExists(classOutDir.resolve("classpath.index")) + Files.deleteIfExists(classOutDir.resolve(".unmodified")) + } - if (success) { - launch(CoroutineName("create output JAR and ABI JAR")) { - // pack to jar - tracer.spanBuilder("create output JAR and ABI JAR").setParent(tracingContext).use { span -> - packageToJar( - outJar = outJar, - abiJar = abiJar, - sourceDescriptors = sourceDescriptors, - classOutDir = classOutDir, - span = span, - ) - } + if (success) { + launch(CoroutineName("create output JAR and ABI JAR")) { + // pack to jar + tracer.span("create output JAR and ABI JAR") { span -> + packageToJar( + outJar = outJar, + abiJar = abiJar, + sourceDescriptors = sourceDescriptors, + classOutDir = classOutDir, + span = span, + ) } } + } - launch(CoroutineName("report build state")) { - dataManager.reportUnhandledRelativizerPaths() - reportRebuiltModules(context) - reportUnprocessedChanges(context, moduleTarget) + launch(CoroutineName("report build state")) { + dataManager.reportUnhandledRelativizerPaths() + if (context.projectDescriptor.fsState.hasUnprocessedChanges(context, moduleTarget)) { + parentSpan.addEvent("Some files were changed during the build. Additional compilation may be required.") } } } \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/JpsKotlinCompilerRunner.kt b/build/jvm-rules/src/jps-builder/JpsKotlinCompilerRunner.kt new file mode 100644 index 000000000000..11aa6e41a088 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/JpsKotlinCompilerRunner.kt @@ -0,0 +1,195 @@ +@file:Suppress("PackageDirectoryMismatch") + +package org.jetbrains.kotlin.compilerRunner + +import com.intellij.util.xmlb.XmlSerializerUtil +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.mergeBeans +import org.jetbrains.kotlin.config.CompilerSettings +import org.jetbrains.kotlin.config.splitArgumentString +import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter +import org.jetbrains.kotlin.preloading.ClassCondition +import org.jetbrains.kotlin.preloading.ClassPreloadingUtils +import org.jetbrains.kotlin.preloading.Preloader +import org.jetbrains.kotlin.utils.KotlinPaths +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.PrintStream +import java.lang.IllegalStateException +import java.lang.reflect.Constructor +import java.lang.reflect.Method +import java.nio.file.Files +import java.nio.file.Path + +internal object DummyKotlinPaths : KotlinPaths { + override val homePath: File + get() = throw kotlin.IllegalStateException() + + override val libPath: File + get() = throw kotlin.IllegalStateException() + + override fun jar(jar: KotlinPaths.Jar): File = throw kotlin.IllegalStateException() + + override fun klib(jar: KotlinPaths.Jar): File = throw kotlin.IllegalStateException() + + override fun sourcesJar(jar: KotlinPaths.Jar): File? = throw kotlin.IllegalStateException() +} + +internal val classesToLoadByParent = ClassCondition { className -> + for (it in arrayOf( + "org.apache.log4j.", // For logging from compiler + "org.jetbrains.kotlin.incremental.components.", + "org.jetbrains.kotlin.incremental.js", + "org.jetbrains.kotlin.load.kotlin.incremental.components." + )) { + if (className.startsWith(it)) { + return@ClassCondition true + } + } + for (it in arrayOf( + "org.jetbrains.kotlin.config.Services", + "org.jetbrains.kotlin.progress.CompilationCanceledStatus", + "org.jetbrains.kotlin.progress.CompilationCanceledException", + "org.jetbrains.kotlin.modules.TargetId", + "org.jetbrains.kotlin.cli.common.ExitCode" + )) { + if (className == it) { + return@ClassCondition true + } + } + + return@ClassCondition false +} + + +// replace Kotlin JPS plugin version by a more tailored for Bazel +@Suppress("unused") +class JpsKotlinCompilerRunner { + companion object { + private val compilerConstructor: Constructor<*> + private val exec: Method + + fun triggerInit() { + requireNotNull(exec) + } + + init { + val libDir = getLibPath(computeKotlinPathsForJpsPlugin()) + val jarFiles = sequenceOf(KotlinPaths.Jar.StdLibJdk7, KotlinPaths.Jar.StdLibJdk8, KotlinPaths.Jar.Compiler, KotlinPaths.Jar.Reflect, KotlinPaths.Jar.Trove4j, KotlinPaths.Jar.CoroutinesCore) + .map { libDir.resolve("${it.baseName}.jar").toFile() } + .toList() + val classLoader = ClassPreloadingUtils.preloadClasses( + jarFiles, + Preloader.DEFAULT_CLASS_NUMBER_ESTIMATE, + JpsKotlinCompilerRunner::class.java.classLoader, + classesToLoadByParent, + ) + + val compiler = Class.forName(KotlinCompilerClass.JVM, true, classLoader) + exec = compiler.getMethod("execAndOutputXml", PrintStream::class.java, Class.forName("org.jetbrains.kotlin.config.Services", true, classLoader), Array::class.java) + compilerConstructor = compiler.getDeclaredConstructor() + compilerConstructor.setAccessible(true) + exec.setAccessible(true) + } + } + + @Suppress("unused") + fun runK2MetadataCompiler( + commonArguments: CommonCompilerArguments, + k2MetadataArguments: K2MetadataCompilerArguments, + compilerSettings: CompilerSettings, + environment: JpsCompilerEnvironment, + destination: String, + classpath: Collection, + sourceFiles: Collection, + buildMetricReporter: JpsBuilderMetricReporter?, + ) { + throw UnsupportedOperationException("") + } + + @Suppress("unused") + fun runK2JvmCompiler( + commonArguments: CommonCompilerArguments, + k2jvmArguments: K2JVMCompilerArguments, + compilerSettings: CompilerSettings, + environment: JpsCompilerEnvironment, + moduleFile: File, + buildMetricReporter: JpsBuilderMetricReporter?, + ) { + val arguments = mergeBeans(commonArguments, XmlSerializerUtil.createCopy(k2jvmArguments)) + setupK2JvmArguments(moduleFile, arguments) + val stream = ByteArrayOutputStream() + val out = PrintStream(stream) + val rc = environment.withProgressReporter { progress -> + progress.compilationStarted() + exec.invoke(compilerConstructor.newInstance(), out, environment.services, withAdditionalCompilerArgs(arguments, compilerSettings)) + } + // exec() returns an ExitCode object, class of which is loaded with a different class loader, + // so we take its contents through reflection + val exitCode = ExitCode.valueOf(getReturnCodeFromObject(rc)) + processCompilerOutput(environment.messageCollector, environment.outputItemsCollector, stream, exitCode) + } +} + +private fun withAdditionalCompilerArgs(compilerArgs: CommonCompilerArguments, compilerSettings: CompilerSettings): Array { + val allArgs = ArgumentUtils.convertArgumentsToStringList(compilerArgs).asSequence() + splitArgumentString(compilerSettings.additionalArguments).asSequence() + val filteredArguments = mutableListOf() + val knownPluginOptions = HashSet() + val argumentsIterator = allArgs.iterator() + while (argumentsIterator.hasNext()) { + val argument = argumentsIterator.next() + // try to find pair -P plugin::= + if (argument == "-P" && argumentsIterator.hasNext()) { + val pluginOption = argumentsIterator.next() // expected plugin::= + val elementIsUnique = knownPluginOptions.add(pluginOption) + if (elementIsUnique) { + filteredArguments.add(argument) // add -P + filteredArguments.add(pluginOption) // add the plugin option + } + } + else { + // skip filtering for all other arguments + filteredArguments.add(argument) + } + } + return filteredArguments.toTypedArray() +} + +private fun setupK2JvmArguments(moduleFile: File, settings: K2JVMCompilerArguments) { + settings.buildFile = moduleFile.absolutePath + settings.destination = null + settings.noStdlib = true + settings.noReflect = true + settings.noJdk = true +} + +private fun getReturnCodeFromObject(rc: Any?): String { + return when { + rc == null -> ExitCode.INTERNAL_ERROR.toString() + ExitCode::class.java.name == rc::class.java.name -> rc.toString() + else -> throw IllegalStateException("Unexpected return: $rc") + } +} + +private const val JPS_KOTLIN_HOME_PROPERTY = "jps.kotlin.home" + +internal fun computeKotlinPathsForJpsPlugin(): Path { + val jpsKotlinHome = System.getProperty(JPS_KOTLIN_HOME_PROPERTY)?.let { Path.of(it) } + ?: throw RuntimeException("Make sure that '$JPS_KOTLIN_HOME_PROPERTY' system property is set in JPS process") + if (Files.isDirectory(jpsKotlinHome)) { + return jpsKotlinHome.resolve("lib") + } + else { + throw RuntimeException("Cannot find kotlinc home at $jpsKotlinHome") + } +} + +private fun getLibPath(libDir: Path): Path { + if (Files.isDirectory(libDir)) { + return libDir + } + throw IllegalStateException("Broken compiler at '$libDir'. Make sure plugin is properly installed") +} diff --git a/build/jvm-rules/src/jps-builder/impl/BazelCompileScope.kt b/build/jvm-rules/src/jps-builder/impl/BazelCompileScope.kt new file mode 100644 index 000000000000..bd915f402365 --- /dev/null +++ b/build/jvm-rules/src/jps-builder/impl/BazelCompileScope.kt @@ -0,0 +1,130 @@ +@file:Suppress("ReplaceGetOrSet") + +package org.jetbrains.bazel.jvm.jps.impl + +import com.intellij.openapi.util.UserDataHolderBase +import com.intellij.util.EventDispatcher +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.isActive +import org.jetbrains.jps.ModuleChunk +import org.jetbrains.jps.api.CanceledStatus +import org.jetbrains.jps.builders.BuildTarget +import org.jetbrains.jps.builders.BuildTargetType +import org.jetbrains.jps.builders.logging.BuildLoggingManager +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.incremental.* +import org.jetbrains.jps.incremental.messages.BuildMessage +import org.jetbrains.jps.incremental.messages.FileDeletedEvent +import org.jetbrains.jps.incremental.messages.FileGeneratedEvent +import org.jetbrains.jps.incremental.messages.ProgressMessage +import java.nio.file.Path +import java.util.* +import kotlin.concurrent.Volatile +import kotlin.coroutines.CoroutineContext + +internal class BazelCompileScope( + @JvmField val isIncrementalCompilation: Boolean, + @JvmField val isRebuild: Boolean, +) : CompileScope() { + private val typesToForceBuild = HashSet?>() + + init { + var forceBuildAllModuleBasedTargets = false + for (type in typesToForceBuild) { + typesToForceBuild.add(type) + forceBuildAllModuleBasedTargets = false + } + } + + override fun isAffected(target: BuildTarget<*>): Boolean = isWholeTargetAffected(target) + + override fun isWholeTargetAffected(target: BuildTarget<*>): Boolean = !isIncrementalCompilation || isRebuild + + override fun isAllTargetsOfTypeAffected(type: BuildTargetType<*>): Boolean = !isIncrementalCompilation || isRebuild + + override fun isBuildForced(target: BuildTarget<*>): Boolean = !isIncrementalCompilation || isRebuild + + override fun isBuildForcedForAllTargets(targetType: BuildTargetType<*>): Boolean = !isIncrementalCompilation || isRebuild + + override fun isBuildIncrementally(targetType: BuildTargetType<*>): Boolean = isIncrementalCompilation && !isRebuild + + override fun isAffected(target: BuildTarget<*>, file: Path): Boolean = true + + override fun markIndirectlyAffected(target: BuildTarget<*>?, file: Path) { + } +} + +internal class BazelCompileContext( + private val scope: BazelCompileScope, + private val projectDescriptor: ProjectDescriptor, + private val delegateMessageHandler: MessageHandler, + private val coroutineContext: CoroutineContext, +) : UserDataHolderBase(), CompileContext { + private val cancelStatus = CanceledStatus { !coroutineContext.isActive } + + private var isMarkedAsNonIncremental = false + @Volatile + private var compilationStartStamp = 0L + + @Volatile + private var done = -1.0f + private val listeners = EventDispatcher.create(BuildListener::class.java) + + override fun getCompilationStartStamp(target: BuildTarget<*>): Long = compilationStartStamp + + override fun setCompilationStartStamp(targets: Collection>, stamp: Long) { + compilationStartStamp = stamp + } + + override fun getLoggingManager(): BuildLoggingManager = projectDescriptor.loggingManager + + override fun getBuilderParameter(paramName: String?): String? = null + + override fun addBuildListener(listener: BuildListener) { + listeners.addListener(listener) + } + + override fun removeBuildListener(listener: BuildListener) { + listeners.removeListener(listener) + } + + override fun markNonIncremental(target: ModuleBuildTarget) { + isMarkedAsNonIncremental = true + } + + override fun shouldDifferentiate(chunk: ModuleChunk): Boolean = scope.isIncrementalCompilation && !isMarkedAsNonIncremental + + override fun getCancelStatus(): CanceledStatus = cancelStatus + + override fun checkCanceled() { + coroutineContext.ensureActive() + } + + override fun clearNonIncrementalMark(target: ModuleBuildTarget) { + isMarkedAsNonIncremental = false + } + + override fun getScope(): BazelCompileScope = scope + + override fun processMessage(message: BuildMessage) { + if (message.kind == BuildMessage.Kind.ERROR) { + Utils.ERRORS_DETECTED_KEY.set(this, true) + } + if (message is ProgressMessage) { + message.done = done + } + delegateMessageHandler.processMessage(message) + if (message is FileGeneratedEvent) { + listeners.getMulticaster().filesGenerated(message) + } + else if (message is FileDeletedEvent) { + listeners.getMulticaster().filesDeleted(message) + } + } + + override fun setDone(done: Float) { + this.done = done + } + + override fun getProjectDescriptor(): ProjectDescriptor = projectDescriptor +} \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/BazelKotlinBuilder.kt b/build/jvm-rules/src/jps-builder/impl/BazelKotlinBuilder.kt index 6600b7de611c..962f9a004f8d 100644 --- a/build/jvm-rules/src/jps-builder/impl/BazelKotlinBuilder.kt +++ b/build/jvm-rules/src/jps-builder/impl/BazelKotlinBuilder.kt @@ -21,6 +21,7 @@ import org.jetbrains.kotlin.build.report.ICReporterBase import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments import org.jetbrains.kotlin.compilerRunner.* +import org.jetbrains.kotlin.compilerRunner.CompilerRunnerUtil import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.components.* @@ -31,46 +32,23 @@ import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.metadata.deserialization.MetadataVersion import org.jetbrains.kotlin.name.FqName -import org.jetbrains.kotlin.preloading.ClassCondition -import org.jetbrains.kotlin.utils.KotlinPaths -import org.jetbrains.kotlin.utils.KotlinPathsFromHomeDir +import org.jetbrains.kotlin.progress.CompilationCanceledException import java.io.File +import kotlin.coroutines.cancellation.CancellationException import kotlin.system.measureTimeMillis -private val classesToLoadByParent = ClassCondition { className -> - for (it in arrayOf( - "org.apache.log4j.", // For logging from compiler - "org.jetbrains.kotlin.incremental.components.", - "org.jetbrains.kotlin.incremental.js", - "org.jetbrains.kotlin.load.kotlin.incremental.components." - )) { - if (className.startsWith(it)) { - return@ClassCondition true - } - } - for (it in arrayOf( - "org.jetbrains.kotlin.config.Services", - "org.jetbrains.kotlin.progress.CompilationCanceledStatus", - "org.jetbrains.kotlin.progress.CompilationCanceledException", - "org.jetbrains.kotlin.modules.TargetId", - "org.jetbrains.kotlin.cli.common.ExitCode" - )) { - if (className == it) { - return@ClassCondition true - } - } - - return@ClassCondition false -} - internal class BazelKotlinBuilder( private val isKotlinBuilderInDumbMode: Boolean, + private val isIncrementalCompilation: Boolean, private val enableLookupStorageFillingInDumbMode: Boolean = false, private val dataManager: BazelBuildDataProvider, private val span: Span, ) : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { companion object { - const val JPS_KOTLIN_HOME_PROPERTY = "jps.kotlin.home" + init { + // force load our version, preload compiler classLoader + JpsKotlinCompilerRunner.triggerInit() + } } override fun getPresentableName() = "Kotlin Builder" @@ -134,18 +112,16 @@ internal class BazelKotlinBuilder( } override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { - super.chunkBuildStarted(context, chunk) - val buildLogger = context.testingContext?.buildLogger buildLogger?.chunkBuildStarted(context, chunk) - if (JavaBuilderUtil.isForcedRecompilationAllJavaModules(context)) { + val kotlinContext = ensureKotlinContextInitialized(context) + + if (JavaBuilderUtil.isForcedRecompilationAllJavaModules(context.scope)) { return } - val targets = chunk.targets - val kotlinContext = ensureKotlinContextInitialized(context) - if (targets.none { kotlinContext.hasKotlinMarker[it] == true }) { + if (chunk.targets.none { kotlinContext.hasKotlinMarker.get(it) == true }) { return } @@ -174,7 +150,6 @@ internal class BazelKotlinBuilder( } } ) - val fsOperations = BazelKotlinFsOperationsHelper(context, chunk, dirtyFilesHolder, span, dataManager = dataManager) val representativeTarget = kotlinContext.targetsBinding[chunk.representativeTarget()] ?: return @@ -197,31 +172,32 @@ internal class BazelKotlinBuilder( val removedClasses = HashSet() for (target in kotlinChunk.targets) { - val cache = incrementalCaches[target] ?: continue + val cache = incrementalCaches.get(target) ?: continue val dirtyFiles = dirtyFilesHolder.getDirtyFiles(target.jpsModuleBuildTarget).keys val removedFiles = dirtyFilesHolder.getRemovedFiles(target.jpsModuleBuildTarget) - val existingClasses = JpsKotlinCompilerRunner().classesFqNamesByFiles(environment, dirtyFiles) + val existingClasses = CompilerRunnerUtil.invokeClassesFqNames(environment, dirtyFiles) val previousClasses = cache.classesFqNamesBySources(dirtyFiles + removedFiles) for (jvmClassName in previousClasses) { val fqName = jvmClassName.asString() - if (fqName !in existingClasses) { + if (!existingClasses.contains(fqName)) { removedClasses.add(fqName) } } } - val changesCollector = ChangesCollector() + val changeCollector = ChangesCollector() for (it in removedClasses) { - changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) + changeCollector.collectSignature(FqName(it), areSubclassesAffected = true) } val affectedByRemovedClasses = getDirtyFiles( - changeCollector = changesCollector, + changeCollector = changeCollector, caches = incrementalCaches.values, lookupStorageManager = kotlinContext.lookupStorageManager, reporter = BazelJpsICReporter(span), ) + val fsOperations = BazelKotlinFsOperationsHelper(context, chunk, dirtyFilesHolder, span, dataManager = dataManager) fsOperations.markFilesForCurrentRound(affectedByRemovedClasses.dirtyFiles.asSequence() + affectedByRemovedClasses.forceRecompileTogether) } @@ -240,15 +216,21 @@ internal class BazelKotlinBuilder( dataManager = dataManager, span = span, ) - val proposedExitCode = doBuild( - chunk = chunk, - representativeTarget = kotlinTarget, - context = context, - kotlinDirtyFilesHolder = kotlinDirtyFilesHolder, - messageCollector = MessageCollectorAdapter(context, kotlinTarget), - outputConsumer = outputConsumer, - fsOperations = fsOperations - ) + val proposedExitCode = try { + doBuild( + chunk = chunk, + representativeTarget = kotlinTarget, + context = context, + kotlinDirtyFilesHolder = kotlinDirtyFilesHolder, + messageCollector = MessageCollectorAdapter(context, kotlinTarget), + outputConsumer = outputConsumer, + fsOperations = fsOperations + ) + } + catch (e: CompilationCanceledException) { + // https://youtrack.jetbrains.com/issue/KTI-2139 + throw CancellationException(e) + } val actualExitCode = if (proposedExitCode == OK && fsOperations.hasMarkedDirty) ADDITIONAL_PASS_REQUIRED else proposedExitCode context.testingContext?.buildLogger?.buildFinished(actualExitCode) @@ -276,8 +258,8 @@ internal class BazelKotlinBuilder( val targets = chunk.targets - val isChunkRebuilding = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context) || - targets.any { kotlinContext.rebuildAfterCacheVersionChanged[it] == true } + val isChunkRebuilding = JavaBuilderUtil.isForcedRecompilationAllJavaModules(context.scope) || + targets.any { kotlinContext.rebuildAfterCacheVersionChanged.get(it) == true } if (!kotlinDirtyFilesHolder.hasDirtyOrRemovedFiles) { if (isChunkRebuilding) { @@ -295,20 +277,17 @@ internal class BazelKotlinBuilder( // but they are not dirty, so Groovy builder does not generate source stubs, // and Kotlin builder is filtering out output directory from classpath // (because it may contain outdated Java classes). - if (!isChunkRebuilding && !representativeTarget.isIncrementalCompilationEnabled) { - targets.forEach { kotlinContext.rebuildAfterCacheVersionChanged[it] = true } + if (!isChunkRebuilding && !isIncrementalCompilation) { + for (target in targets) { + kotlinContext.rebuildAfterCacheVersionChanged.set(target, true) + } return CHUNK_REBUILD_REQUIRED } - val targetsWithoutOutputDir = targets.filter { it.outputDir == null } - if (targetsWithoutOutputDir.isNotEmpty()) { - throw RuntimeException("Output directory not specified for ${targetsWithoutOutputDir.joinToString()}") - } - val project = context.projectDescriptor.project - val lookupTracker = getLookupTracker(project, representativeTarget) + val lookupTracker = getLookupTracker(project, isIncrementalCompilation) val exceptActualTracker = ExpectActualTrackerImpl() - val incrementalCaches = kotlinChunk.loadCaches() + val incrementalCaches = kotlinChunk.loadCaches(loadDependent = false) val inlineConstTracker = InlineConstTrackerImpl() val enumWhenTracker = EnumWhenTrackerImpl() val importTracker = ImportTrackerImpl() @@ -357,7 +336,7 @@ internal class BazelKotlinBuilder( generatedFiles = generatedFiles, kotlinContext = kotlinContext, incrementalCaches = incrementalCaches, - fsOperations = fsOperations + fsOperations = fsOperations, ) } @@ -390,7 +369,7 @@ internal class BazelKotlinBuilder( environment = environment ) - if (!representativeTarget.isIncrementalCompilationEnabled) { + if (!isIncrementalCompilation) { return OK } @@ -399,15 +378,15 @@ internal class BazelKotlinBuilder( environment.withProgressReporter { progress -> progress.progress("performing incremental compilation analysis") - val changesCollector = ChangesCollector() + val changeCollector = ChangesCollector() for ((target, files) in generatedFiles) { - val kotlinModuleBuilderTarget = kotlinContext.targetsBinding[target]!! + val kotlinModuleBuilderTarget = kotlinContext.targetsBinding.get(target)!! kotlinModuleBuilderTarget.updateCaches( dirtyFilesHolder = kotlinDirtyFilesHolder, jpsIncrementalCache = incrementalCaches.get(kotlinModuleBuilderTarget)!!, files = files, - changesCollector = changesCollector, + changesCollector = changeCollector, environment = environment, ) } @@ -418,7 +397,7 @@ internal class BazelKotlinBuilder( if (!isKotlinBuilderInDumbMode && !isChunkRebuilding) { processChangesUsingLookups( - collector = changesCollector, + collector = changeCollector, compiledFiles = kotlinDirtyFilesHolder.allDirtyFiles, lookupStorageManager = kotlinContext.lookupStorageManager, fsOperations = fsOperations, @@ -447,12 +426,12 @@ internal class BazelKotlinBuilder( it.nextRound(context) } - if (representativeTarget.isIncrementalCompilationEnabled) { + if (isIncrementalCompilation) { for (target in kotlinChunk.targets) { val cache = incrementalCaches[target] val jpsTarget = target.jpsModuleBuildTarget - val targetDirtyFiles = dirtyFilesHolder.byTarget[jpsTarget] + val targetDirtyFiles = dirtyFilesHolder.byTarget.get(jpsTarget) if (cache != null && targetDirtyFiles != null) { val complementaryFiles = cache.getComplementaryFilesRecursive(targetDirtyFiles.dirty.keys + targetDirtyFiles.removed) context.testingContext?.buildLogger?.markedAsComplementaryFiles(ArrayList(complementaryFiles)) @@ -505,7 +484,7 @@ internal class BazelKotlinBuilder( } return JpsCompilerEnvironment( - kotlinPaths = computeKotlinPathsForJpsPlugin() ?: return null, + kotlinPaths = DummyKotlinPaths, services = compilerServices, classesToLoadByParent = classesToLoadByParent, messageCollector = messageCollector, @@ -514,17 +493,6 @@ internal class BazelKotlinBuilder( ) } - private fun computeKotlinPathsForJpsPlugin(): KotlinPaths? { - val jpsKotlinHome = System.getProperty(JPS_KOTLIN_HOME_PROPERTY)?.let { File(it) } - ?: throw RuntimeException("Make sure that '$JPS_KOTLIN_HOME_PROPERTY' system property is set in JPS process") - if (jpsKotlinHome.exists()) { - return KotlinPathsFromHomeDir(jpsKotlinHome) - } - else { - throw RuntimeException("Cannot find kotlinc home at $jpsKotlinHome") - } - } - private fun getGeneratedFiles( context: CompileContext, chunk: ModuleChunk, @@ -542,7 +510,7 @@ internal class BazelKotlinBuilder( val representativeTarget = chunk.representativeTarget() fun SimpleOutputItem.target(): ModuleBuildTarget { - return sourceFiles.firstOrNull()?.let { sourceToTarget[it] } + return sourceFiles.firstOrNull()?.let { sourceToTarget.get(it) } ?: chunk.targets.singleOrNull { target -> target.outputDir?.let { outputDir -> outputFile.startsWith(outputDir) @@ -561,8 +529,9 @@ internal class BazelKotlinBuilder( lookupStorageManager: JpsLookupStorageManager, dirtyFilesHolder: KotlinDirtySourceFilesHolder ) { - if (lookupTracker !is LookupTrackerImpl) + if (lookupTracker !is LookupTrackerImpl) { throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}") + } lookupStorageManager.withLookupStorage { lookupStorage -> lookupStorage.removeLookupsFrom(dirtyFilesHolder.allDirtyFiles.asSequence() + dirtyFilesHolder.allRemovedFilesFiles.asSequence()) @@ -650,12 +619,7 @@ private fun getDirtyFiles( ) } -private fun getLookupTracker(project: JpsProject, representativeTarget: KotlinModuleBuildTarget<*>): LookupTracker { - val testLookupTracker = project.testingContext?.lookupTracker ?: LookupTracker.DO_NOTHING - if (representativeTarget.isIncrementalCompilationEnabled) { - return LookupTrackerImpl(testLookupTracker) - } - else { - return testLookupTracker - } +private fun getLookupTracker(project: JpsProject, isIncrementalCompilation: Boolean): LookupTracker { + val lookupTracker = project.testingContext?.lookupTracker ?: LookupTracker.DO_NOTHING + return if (isIncrementalCompilation) LookupTrackerImpl(lookupTracker) else lookupTracker } \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/JavaBuilder.kt b/build/jvm-rules/src/jps-builder/impl/JavaBuilder.kt index 377427c054b6..fee37b060ba6 100644 --- a/build/jvm-rules/src/jps-builder/impl/JavaBuilder.kt +++ b/build/jvm-rules/src/jps-builder/impl/JavaBuilder.kt @@ -81,12 +81,11 @@ private const val USE_MODULE_PATH_ONLY_OPTION = "compiler.force.module.path" @Suppress("RemoveRedundantQualifierName") internal val javaModuleTypes = java.util.Set.of(JpsJavaModuleType.INSTANCE) -private val PREFER_TARGET_JDK_COMPILER: Key = GlobalContextKey.create("_prefer_target_jdk_javac_") -private val SHOWN_NOTIFICATIONS: Key> = GlobalContextKey.create>("_shown_notifications_") +private val PREFER_TARGET_JDK_COMPILER: Key = GlobalContextKey.create("_prefer_target_jdk_javac_") +private val SHOWN_NOTIFICATIONS: Key> = GlobalContextKey.create("_shown_notifications_") private val COMPILING_TOOL = Key.create("_java_compiling_tool_") -private val COMPILER_USAGE_STATISTICS = Key.create>>("_java_compiler_usage_stats_") -private val MODULE_PATH_SPLITTER: Key = GlobalContextKey.create("_module_path_splitter_") -private val COMPILABLE_EXTENSIONS = mutableListOf(JAVA_EXTENSION) +private val MODULE_PATH_SPLITTER: Key = GlobalContextKey.create("_module_path_splitter_") +private val COMPILABLE_EXTENSIONS = mutableListOf(JAVA_EXTENSION) private const val PROC_ONLY_OPTION = "-proc:only" private const val PROC_FULL_OPTION = "-proc:full" @@ -102,7 +101,7 @@ private const val SOURCE_OPTION = "-source" private const val SYSTEM_OPTION = "--system" @Suppress("RemoveRedundantQualifierName") -private val FILTERED_OPTIONS = java.util.Set.of(TARGET_OPTION, RELEASE_OPTION, "-d") +private val FILTERED_OPTIONS = java.util.Set.of(TARGET_OPTION, RELEASE_OPTION, "-d") @Suppress("RemoveRedundantQualifierName", "SpellCheckingInspection") private val FILTERED_SINGLE_OPTIONS: MutableSet = java.util.Set.of( @@ -114,26 +113,26 @@ private val POSSIBLY_CONFLICTING_OPTIONS = java.util.Set.of( SOURCE_OPTION, SYSTEM_OPTION, "--boot-class-path", "-bootclasspath", "--class-path", "-classpath", "-cp", PROCESSORPATH_OPTION, "-sourcepath", "--module-path", "-p", "--module-source-path" ) -internal class JavaBuilder(private val span: Span, private val out: Appendable) : ModuleLevelBuilder(BuilderCategory.TRANSLATOR) { +internal class JavaBuilder( + private val span: Span, + private val out: Appendable, +) : ModuleLevelBuilder(BuilderCategory.TRANSLATOR) { private val refRegistrars = ArrayList() companion object { const val BUILDER_ID: String = "java" - private val ourDefaultRtJar: Path? private val ourProcFullRequiredFrom: Int // 0, if not set init { - var rtJar: Path? = null val tokenizer = StringTokenizer(System.getProperty("sun.boot.class.path", ""), File.pathSeparator, false) while (tokenizer.hasMoreTokens()) { val file = Path.of(tokenizer.nextToken()) if (file.endsWith("rt.jar")) { - rtJar = file.normalize().toAbsolutePath() + error("rt.jar should not exists") break } } - ourDefaultRtJar = rtJar ourProcFullRequiredFrom = 0 } @@ -250,44 +249,6 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) return result } - // If platformCp of the build process is the same as the target platform, do not specify platformCp explicitly - // this will allow javac to resolve against ct.sym file, which is required for the "compilation profiles" feature - private fun calcEffectivePlatformCp( - platformCp: Collection, - options: Iterable - ): Collection? { - if (ourDefaultRtJar == null) { - return platformCp - } - - var profileFeatureRequested = false - for (option in options) { - if ("-profile".equals(option, ignoreCase = true)) { - profileFeatureRequested = true - break - } - } - if (!profileFeatureRequested) { - return platformCp - } - - var isTargetPlatformSameAsBuildRuntime = false - for (file in platformCp) { - if (file == ourDefaultRtJar) { - isTargetPlatformSameAsBuildRuntime = true - break - } - } - if (!isTargetPlatformSameAsBuildRuntime) { - // compact profile was requested, but we have to use alternative platform classpath to meet project settings - // consider this a compiler error and let user re-configure the project - return null - } - // returning an empty list will force default behavior for platform classpath calculation - // javac will resolve against its own bootclasspath and use ct.sym file when available - return emptyList() - } - @Synchronized private fun ensureJavacServerStarted(context: CompileContext): ExternalJavacManager { var server = ExternalJavacManagerKey.KEY.get(context) @@ -763,7 +724,6 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) val compilingTool = JavaBuilderUtil.findCompilingTool(compilerId) COMPILING_TOOL.set(context, compilingTool) SHOWN_NOTIFICATIONS.set(context, Collections.synchronizedSet(HashSet())) - COMPILER_USAGE_STATISTICS.set(context, ConcurrentHashMap>()) val dataManager = context.projectDescriptor.dataManager if (!isJavac(compilingTool)) { dataManager.isProcessConstantsIncrementally = false @@ -777,35 +737,22 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) } } - override fun chunkBuildStarted(context: CompileContext?, chunk: ModuleChunk) { + override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { + if (!(context as BazelCompileContext).scope.isIncrementalCompilation) { + return + } + // before the first compilation round starts: find and mark dirty all classes that depend on removed or moved classes so // that all such files are compiled in the first round. JavaBuilderUtil.markDirtyDependenciesForInitialRound(context, object : DirtyFilesHolderBase(context) { override fun processDirtyFiles(processor: FileProcessor) { - FSOperations.processFilesToRecompile(context, chunk, processor) + context.projectDescriptor.fsState.processFilesToRecompile(context, chunk.targets.single(), processor) } }, chunk) } override fun buildFinished(context: CompileContext) { refRegistrars.clear() - val stats = COMPILER_USAGE_STATISTICS.get(context) - if (stats.size == 1) { - val entry = stats.entries.iterator().next() - val compilerName = entry.key - context.processMessage(CompilerMessage("", BuildMessage.Kind.JPS_INFO, - JpsBuildBundle.message("build.message.0.was.used.to.compile.java.sources", compilerName))) - } - else { - for (entry in stats.entries) { - val compilerName = entry.key - val moduleNames = entry.value - context.processMessage(CompilerMessage("", BuildMessage.Kind.JPS_INFO, - if (moduleNames.size == 1) JpsBuildBundle.message("build.message.0.was.used.to.compile.1", compilerName, moduleNames.iterator().next()) else JpsBuildBundle.message("build.message.0.was.used.to.compile.1.modules", compilerName, moduleNames.size) - )) - span.addEvent("$compilerName was used to compile $moduleNames") - } - } } override fun getCompilableFileExtensions(): List = COMPILABLE_EXTENSIONS @@ -898,7 +845,6 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) JavaBuilderUtil.ensureModuleHasJdk(chunk.representativeTarget().module, context, builderName) val classpath = ProjectPaths.getCompilationClasspath(chunk, false).map { it.toPath() } - val platformCp = ProjectPaths.getPlatformCompilationClasspath(chunk, false).map { it.toPath() } // begin compilation round val outputSink = OutputFilesSink( @@ -927,7 +873,6 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) AttributeKey.stringKey("module.name"), chunkName, AttributeKey.stringArrayKey("files"), files.map { it.path }, AttributeKey.stringArrayKey("classpath"), classpath.map { it.toString() }, - AttributeKey.stringArrayKey("platform classpath"), platformCp.map { it.toString() }, ) ) } @@ -938,7 +883,6 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) chunk = chunk, files = files, originalClassPath = classpath, - originalPlatformCp = platformCp, sourcePath = emptyList(), diagnosticSink = diagnosticSink, outputSink = outputSink, @@ -981,7 +925,6 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) chunk: ModuleChunk, files: Collection, originalClassPath: Collection, - originalPlatformCp: Collection, sourcePath: Collection, diagnosticSink: DiagnosticOutputConsumer, outputSink: OutputFileConsumer, @@ -1024,19 +967,17 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) val vmOptions = vm_compilerOptions.first val options = vm_compilerOptions.second - val effectivePlatformCp = calcEffectivePlatformCp(platformCp = originalPlatformCp, options = options) - if (effectivePlatformCp == null) { - context.processMessage(CompilerMessage(builderName, BuildMessage.Kind.ERROR, JpsBuildBundle.message( - "build.message.unsupported.compact.compilation.profile.was.requested", chunk.name, System.getProperty("java.version") - ))) - return false - } - val platformCp: Iterable? val classPath: Iterable? val modulePath: ModulePath val upgradeModulePath: Iterable? - if (moduleInfoFile != null) { + if (moduleInfoFile == null) { + modulePath = ModulePath.EMPTY + upgradeModulePath = emptyList() + platformCp = emptyList() + classPath = originalClassPath + } + else { // has modules val splitter = MODULE_PATH_SPLITTER.get(context) val pair = splitter.splitPath( @@ -1058,29 +999,13 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) modulePath = pair.first classPath = pair.second.map { it.toPath() } } + // modules above the JDK in the order entry list make a module upgrade path - upgradeModulePath = effectivePlatformCp + upgradeModulePath = emptyList() platformCp = emptyList() } - else { - modulePath = ModulePath.EMPTY - upgradeModulePath = mutableListOf() - if (!effectivePlatformCp.iterator().hasNext() && getChunkSdkVersion(chunk) >= 9) { - // If chunk's SDK is 9 or higher, there is no way to specify full platform classpath - // because platform classes are stored in `jimage` binary files with unknown format. - // Because of this, we are clearing platform classpath so that javac will resolve against its own boot classpath - // and prepending additional jars from the JDK configuration to compilation classpath - platformCp = emptyList() - classPath = Iterators.flat(effectivePlatformCp, originalClassPath) - } - else { - platformCp = effectivePlatformCp - classPath = originalClassPath - } - } if (forkSdk != null) { - updateCompilerUsageStatistics(context, "javac ${forkSdk.getSecond()}", chunk) val server = ensureJavacServerStarted(context) val paths = CompilationPaths.create( platformCp.map { it.toFile() }, @@ -1117,7 +1042,6 @@ internal class JavaBuilder(private val span: Span, private val out: Appendable) } } - updateCompilerUsageStatistics(context = context, compilerName = compilingTool.description, chunk = chunk) return invokeJavac( compilerSdkVersion = compilerSdkVersion, context = context, @@ -1442,11 +1366,4 @@ private fun collectAdditionalRequires(options: Iterable): Collection, files: Iterable, outSink: OutputFileConsumer): Boolean -} - -private fun updateCompilerUsageStatistics(context: CompileContext, compilerName: String, chunk: ModuleChunk) { - val names = COMPILER_USAGE_STATISTICS.get(context).computeIfAbsent(compilerName) { Collections.synchronizedSet(hashSet()) } - for (module in chunk.modules) { - names.add(module.name) - } } \ No newline at end of file diff --git a/build/jvm-rules/src/jps-builder/impl/JpsTargetBuilder.kt b/build/jvm-rules/src/jps-builder/impl/JpsTargetBuilder.kt index 86b9ed48368e..0d01f1c19bf2 100644 --- a/build/jvm-rules/src/jps-builder/impl/JpsTargetBuilder.kt +++ b/build/jvm-rules/src/jps-builder/impl/JpsTargetBuilder.kt @@ -4,12 +4,10 @@ package org.jetbrains.bazel.jvm.jps.impl import com.intellij.openapi.util.text.Formats.formatDuration -import com.intellij.tracing.Tracer.start import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.Tracer -import io.opentelemetry.context.Context import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap import it.unimi.dsi.fastutil.objects.ObjectArraySet import kotlinx.coroutines.ensureActive @@ -17,6 +15,7 @@ import org.jetbrains.bazel.jvm.jps.hashMap import org.jetbrains.bazel.jvm.jps.linkedSet import org.jetbrains.bazel.jvm.jps.state.LoadStateResult import org.jetbrains.bazel.jvm.jps.state.RemovedFileInfo +import org.jetbrains.bazel.jvm.span import org.jetbrains.bazel.jvm.use import org.jetbrains.jps.ModuleChunk import org.jetbrains.jps.builders.BuildRootDescriptor @@ -31,7 +30,6 @@ import org.jetbrains.jps.incremental.fs.BuildFSState.CURRENT_ROUND_DELTA_KEY import org.jetbrains.jps.incremental.fs.BuildFSState.NEXT_ROUND_DELTA_KEY import org.jetbrains.jps.incremental.fs.CompilationRound import org.jetbrains.jps.incremental.messages.* -import org.jetbrains.jps.incremental.storage.BuildTargetConfiguration import java.io.IOException import java.nio.file.Files import java.nio.file.Path @@ -53,34 +51,27 @@ internal class JpsTargetBuilder( private val numberOfSourcesProcessedByBuilder = hashMap() suspend fun build( - context: CompileContextImpl, + context: CompileContext, moduleTarget: BazelModuleBuildTarget, builders: Array, buildState: LoadStateResult?, parentSpan: Span, - tracingContext: Context, ): Int { try { - val buildSpan = start("JpsTargetBuilder.runBuild") context.setDone(0.0f) context.addBuildListener(ChainedTargetsBuildListener(context, dataManager)) - val allModuleLevelBuildersBuildStartedSpan = start("All ModuleLevelBuilder.buildStarted") for (builder in builders) { builder.buildStarted(context) } - allModuleLevelBuildersBuildStartedSpan.complete() try { - val span = start("build target") - buildTarget(context = context, target = moduleTarget, builders = builders, buildState = buildState, tracingContext) - span.complete() + buildTarget(context = context, target = moduleTarget, builders = builders, buildState = buildState) } finally { for (builder in builders) { builder.buildFinished(context) } } - buildSpan.complete() for ((builder, time) in builderToDuration) { val processedSources = numberOfSourcesProcessedByBuilder.get(builder)?.get() ?: 0 @@ -258,7 +249,6 @@ internal class JpsTargetBuilder( target: BazelModuleBuildTarget, builders: Array, buildState: LoadStateResult?, - tracingContext: Context, ) { val targets = java.util.Set.of>(target) try { @@ -271,7 +261,6 @@ internal class JpsTargetBuilder( val fsState = context.projectDescriptor.fsState require(!fsState.isInitialScanPerformed(target)) tracer.spanBuilder("fs state init") - .setParent(tracingContext) .setAttribute("isCleanBuild", isCleanBuild) .use { span -> if (isCleanBuild || buildState == null) { @@ -311,7 +300,7 @@ internal class JpsTargetBuilder( fsState.beforeChunkBuildStart(context, targets) - tracer.spanBuilder("runModuleLevelBuilders").setParent(tracingContext).use { span -> + tracer.span("runModuleLevelBuilders") { span -> if (runModuleLevelBuilders(context, target, builders, span)) { doneSomething = true } @@ -349,7 +338,7 @@ internal class JpsTargetBuilder( } } -private class ChainedTargetsBuildListener(private val context: CompileContextImpl, private val dataManager: BazelBuildDataProvider) : BuildListener { +private class ChainedTargetsBuildListener(private val context: CompileContext, private val dataManager: BazelBuildDataProvider) : BuildListener { override fun filesGenerated(event: FileGeneratedEvent) { val projectDescriptor = context.projectDescriptor val fsState = projectDescriptor.fsState @@ -430,22 +419,6 @@ private fun deleteOutputsAssociatedWithDeletedPaths( return doneSomething } -internal fun reportRebuiltModules(context: CompileContextImpl) { - val modules = BuildTargetConfiguration.MODULES_WITH_TARGET_CONFIG_CHANGED_KEY.get(context) - if (modules.isNullOrEmpty()) { - return - } - - val text = "${modules.joinToString { m -> "'" + m.name + "'" }} was fully rebuilt due to project configuration changes" - context.processMessage(CompilerMessage("", BuildMessage.Kind.INFO, text)) -} - -internal fun reportUnprocessedChanges(context: CompileContextImpl, moduleTarget: ModuleBuildTarget) { - if (context.projectDescriptor.fsState.hasUnprocessedChanges(context, moduleTarget)) { - context.processMessage(UnprocessedFSChangesNotification()) - } -} - /** * if an output file is generated from multiple sources, make sure all of them are added for recompilation */ diff --git a/build/jvm-rules/src/jps-builder/impl/RequestLog.kt b/build/jvm-rules/src/jps-builder/impl/RequestLog.kt index e150ed3d4aed..c6528650527e 100644 --- a/build/jvm-rules/src/jps-builder/impl/RequestLog.kt +++ b/build/jvm-rules/src/jps-builder/impl/RequestLog.kt @@ -9,6 +9,7 @@ import io.opentelemetry.api.trace.Tracer import org.jetbrains.jps.incremental.MessageHandler import org.jetbrains.jps.incremental.messages.BuildMessage import org.jetbrains.jps.incremental.messages.CompilerMessage +import org.jetbrains.jps.incremental.messages.CustomBuilderMessage import org.jetbrains.jps.incremental.messages.ProgressMessage internal class RequestLog( @@ -33,6 +34,14 @@ internal class RequestLog( } } is ProgressMessage -> return + is CustomBuilderMessage -> { + if (message.messageType == "processed module") { + return + } + else { + message.messageText + } + } else -> message.messageText } diff --git a/build/jvm-rules/src/jps-builder/state/BuildStateStorage.kt b/build/jvm-rules/src/jps-builder/state/BuildStateStorage.kt index 83898722de0e..6cf37f447ee0 100644 --- a/build/jvm-rules/src/jps-builder/state/BuildStateStorage.kt +++ b/build/jvm-rules/src/jps-builder/state/BuildStateStorage.kt @@ -7,9 +7,7 @@ import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.BaseValueVector import org.apache.arrow.vector.BitVector -import org.apache.arrow.vector.FixedSizeBinaryVector import org.apache.arrow.vector.VarBinaryVector import org.apache.arrow.vector.VarCharVector import org.apache.arrow.vector.VectorSchemaRoot @@ -212,7 +210,6 @@ fun saveBuildState( } } -// do not use an open-addressing hash map or immutable map - see https://stackoverflow.com/a/16303438 data class LoadStateResult( @JvmField val rebuildRequested: String?, @@ -252,7 +249,7 @@ private fun doLoad( val start = outputListVector.getElementStartIndex(rowIndex) val end = outputListVector.getElementEndIndex(rowIndex) val size = end - start - val outputs = if (size == 0) null else Array(size) { + val outputs = if (size == 0) null else Array(size) { String(outputListInnerVector.get(it + start)) } diff --git a/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt b/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt index ee392c0770bc..cc9b29ccea89 100644 --- a/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt +++ b/build/jvm-rules/src/kotlin-builder/KotlinBuilder.kt @@ -2,7 +2,6 @@ package org.jetbrains.bazel.jvm.kotlin import io.opentelemetry.api.trace.Tracer -import io.opentelemetry.context.Context import org.jetbrains.bazel.jvm.WorkRequest import org.jetbrains.bazel.jvm.WorkRequestExecutor import org.jetbrains.bazel.jvm.WorkRequestReaderWithoutDigest @@ -17,7 +16,7 @@ object KotlinBuildWorker : WorkRequestExecutor { processRequests(startupArgs = startupArgs, executor = this, reader = WorkRequestReaderWithoutDigest(System.`in`), serviceName = "kotlin-builder") } - override suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path, tracingContext: Context, tracer: Tracer): Int { + override suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path, tracer: Tracer): Int { val sources = request.inputPaths.asSequence() .filter { it.endsWith(".kt") || it.endsWith(".java") } .map { baseDir.resolve(it).normalize() } diff --git a/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt b/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt index e55e874ff415..c5975044f70e 100644 --- a/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt +++ b/build/jvm-rules/src/kotlin-builder/KotlinJvmTaskExecutor.kt @@ -189,7 +189,7 @@ private inline fun wrapOutput(out: Writer, label: String, classifier: String, ta private fun createClasspath(args: ArgMap, baseDir: Path): List { if (!args.boolFlag(JvmBuilderFlags.REDUCED_CLASSPATH_MODE)) { - return args.mandatory(JvmBuilderFlags.CLASSPATH).map { baseDir.resolve(it).normalize() } + return args.mandatory(JvmBuilderFlags.CP).map { baseDir.resolve(it).normalize() } } val directDependencies = args.mandatory(JvmBuilderFlags.DIRECT_DEPENDENCIES) diff --git a/build/jvm-rules/src/misc/JvmWorker.kt b/build/jvm-rules/src/misc/JvmWorker.kt index 919e893da13f..70935125e7ac 100644 --- a/build/jvm-rules/src/misc/JvmWorker.kt +++ b/build/jvm-rules/src/misc/JvmWorker.kt @@ -2,7 +2,6 @@ package org.jetbrains.bazel.jvm import io.opentelemetry.api.trace.Tracer -import io.opentelemetry.context.Context import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.jetbrains.intellij.build.io.* @@ -17,7 +16,7 @@ object JvmWorker : WorkRequestExecutor { processRequests(startupArgs = startupArgs, executor = this, reader = WorkRequestReaderWithoutDigest(System.`in`), serviceName = "jvm-worker") } - override suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path, tracingContext: Context, tracer: Tracer): Int { + override suspend fun execute(request: WorkRequest, writer: Writer, baseDir: Path, tracer: Tracer): Int { val args = request.arguments if (args.isEmpty()) { writer.appendLine("Command is not specified") diff --git a/build/jvm-rules/src/worker-framework/BUILD.bazel b/build/jvm-rules/src/worker-framework/BUILD.bazel index 45500c651c14..d99d08541e2f 100644 --- a/build/jvm-rules/src/worker-framework/BUILD.bazel +++ b/build/jvm-rules/src/worker-framework/BUILD.bazel @@ -15,6 +15,7 @@ kt_jvm_library( "@lib//:opentelemetry", "@lib//:opentelemetry-semconv", "//:opentelemetry-exporter-logging-otlp", + "//:opentelemetry-exporter-otlp", "//:kotlinx-coroutines-core", "//:protobuf-java", "@lib//:netty-buffer", diff --git a/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt b/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt index 4010f2e747c5..6974a6940d4f 100644 --- a/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt +++ b/build/jvm-rules/src/worker-framework/WorkRequestHandler.kt @@ -4,13 +4,14 @@ package org.jetbrains.bazel.jvm import com.google.protobuf.CodedOutputStream +import io.netty.buffer.PooledByteBufAllocator import io.opentelemetry.api.OpenTelemetry import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.Tracer import io.opentelemetry.context.Context -import io.opentelemetry.exporter.logging.otlp.internal.traces.OtlpStdoutSpanExporter +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter import io.opentelemetry.sdk.OpenTelemetrySdk import io.opentelemetry.sdk.common.export.MemoryMode import io.opentelemetry.sdk.resources.Resource @@ -19,7 +20,6 @@ import io.opentelemetry.sdk.trace.export.BatchSpanProcessor import io.opentelemetry.sdk.trace.samplers.Sampler import io.opentelemetry.semconv.ServiceAttributes import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.jetbrains.annotations.VisibleForTesting @@ -29,17 +29,21 @@ import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference import kotlin.coroutines.cancellation.CancellationException -import kotlin.coroutines.coroutineContext import kotlin.system.exitProcess fun interface WorkRequestExecutor { - suspend fun execute(request: T, writer: Writer, baseDir: Path, tracingContext: Context, tracer: Tracer): Int + suspend fun execute(request: T, writer: Writer, baseDir: Path, tracer: Tracer): Int } -fun configureOpenTelemetry(out: OutputStream, serviceName: String): OpenTelemetrySdk { - val spanExporter = OtlpStdoutSpanExporter.builder() - .setOutput(out) +@Suppress("SpellCheckingInspection") +fun configureOpenTelemetry(@Suppress("unused") out: OutputStream, serviceName: String): Pair Unit> { + //val spanExporter = OtlpStdoutSpanExporter.builder() + // .setOutput(out) + // .setMemoryMode(MemoryMode.REUSABLE_DATA) + // .build() + val spanExporter = OtlpHttpSpanExporter.builder() .setMemoryMode(MemoryMode.REUSABLE_DATA) + //.setEndpoint("https://jaeger-dev.labs.jb.gg/v1/traces") .build() val batchSpanProcessor = BatchSpanProcessor.builder(spanExporter) @@ -57,7 +61,10 @@ fun configureOpenTelemetry(out: OutputStream, serviceName: String): OpenTelemetr val openTelemetrySdk = OpenTelemetrySdk.builder() .setTracerProvider(tracerProvider) .build() - return openTelemetrySdk + return openTelemetrySdk to { + spanExporter.close() + batchSpanProcessor.close() + } } private val noopTracer = OpenTelemetry.noop().getTracer("noop") @@ -67,38 +74,56 @@ fun processRequests( executor: WorkRequestExecutor, serviceName: String?, reader: WorkRequestReader, - setup: (Tracer, CoroutineScope) -> Unit = { _, _ -> }, + setup: (Tracer, CoroutineScope) -> Unit = { _, _ -> }, ) { if (!startupArgs.contains("--persistent_worker")) { System.err.println("Only persistent worker mode is supported") - exitProcess(1) + exitProcess(3) } - val tracer = if (serviceName == null) { - noopTracer - } - else { - configureOpenTelemetry(System.err, serviceName).getTracer(serviceName) - } + var onClose = {} + var exitCode: Int try { - runBlocking(Dispatchers.Default) { - tracer.spanBuilder("process requests").use { span -> - setup(tracer, this@runBlocking) - WorkRequestHandler(requestExecutor = executor, out = System.out, tracer = tracer) - .processRequests(reader, Context.current().with(span)) + runBlocking(Dispatchers.Default + OpenTelemetryContextElement(Context.root())) { + val tracer = if (serviceName == null) { + System.err.println("worker started (no OpenTelemetry)") + noopTracer + } + else { + val otAndOnClose = configureOpenTelemetry(out = System.err, serviceName = serviceName) + onClose = otAndOnClose.second + System.err.println("worker started (serviceName=$serviceName)") + otAndOnClose.first.getTracer(serviceName) } - exitProcess(0) + tracer.span("process requests") { span -> + setup(tracer, this@runBlocking) + WorkRequestHandler(requestExecutor = executor, out = System.out, tracer = tracer) + .processRequests(reader) + } + exitCode = 0 } } + catch (_: CancellationException) { + exitCode = 2 + } catch (e: Throwable) { e.printStackTrace(System.err) - exitProcess(1) + exitCode = 1 } + finally { + onClose() + } + + exitProcess(exitCode) } -private class RequestState(@JvmField val request: T) { - @JvmField val state = AtomicReference(NOT_STARTED) +private class RequestState( + @JvmField val request: T, + @JvmField val job: Job, +) { + @JvmField + val state = AtomicReference(NOT_STARTED) } @VisibleForTesting @@ -132,185 +157,235 @@ internal class WorkRequestHandler internal constructor( private val activeRequests = ConcurrentHashMap>() @OptIn(DelicateCoroutinesApi::class) - internal suspend fun processRequests(reader: WorkRequestReader, tracingContext: Context) { - val requestChannel = Channel>(Channel.UNLIMITED) + internal suspend fun processRequests(reader: WorkRequestReader) { try { - coroutineScope { - tracer.spanBuilder("process requests").setParent(tracingContext).use { span -> - startTaskProcessing(requestChannel, tracingContext.with(span)) - } + val isTracingEnabled = tracer != noopTracer + // Bazel already ensures that tasks are submitted at a manageable rate (ensures the system isn’t overwhelmed), + // no need to use channel + tracer.span("read requests") { span -> + coroutineScope { + while (coroutineContext.isActive) { + val request = try { + runInterruptible(Dispatchers.IO) { + reader.readWorkRequestFromStream() + } + } + catch (e: InterruptedIOException) { + span.recordException(e) + null + } - tracer.spanBuilder("read requests").setParent(tracingContext).use { span -> - readRequests(requestChannel, reader, span) + if (request == null) { + span.addEvent("stop processing - no more requests") + return@coroutineScope + } + + val requestId = request.requestId + + if (isTracingEnabled) { + span.addEvent("request received", Attributes.of( + AttributeKey.longKey("id"), requestId.toLong(), + AttributeKey.booleanKey("cancel"), request.cancel, + )) + } + + if (request.cancel) { + // Theoretically, we could have gotten two singleplex requests, and we can't tell those apart. + // However, that's a violation of the protocol, so we don't try to handle it + // (not least because handling it would be quite error-prone). + val item = activeRequests.get(requestId) + val stateRef = item?.state + if (stateRef != null) { + if (stateRef.compareAndSet(STARTED, FINISHED) || stateRef.compareAndSet(NOT_STARTED, FINISHED)) { + if (cancelHandler != null) { + cancelHandler(request.requestId) + } + + item.job.cancel() + span.addEvent("request cancelled before handling", Attributes.of(AttributeKey.longKey("id"), requestId.toLong())) + writeRequest(requestId = requestId, wasCancelled = true) { true } + } + } + } + else { + if (requestId == 0) { + span.addEvent("waiting for singleplex requests to finish") + while (activeRequests.containsKey(0)) { + // Previous singleplex requests can still be in activeRequests for a bit after the response has been sent. + // We need to wait for them to vanish. + delay(1) + } + span.addEvent("end of waiting for singleplex requests to finish") + } + + val requestJob = launch(start = CoroutineStart.LAZY) { + val item = activeRequests.get(requestId) ?: return@launch + try { + checkStateAndExecuteRequest(item.state, request) + } + catch (e: CancellationException) { + ensureActive() + + // ok, only this task was cancelled + span.recordException(e, Attributes.of( + AttributeKey.stringKey("message"), "request cancelled", + AttributeKey.longKey("id"), requestId.toLong(), + )) + } + } + + val item = RequestState(request = request, job = requestJob) + val previous = activeRequests.putIfAbsent(requestId, item) + require(previous == null) { + "Request still active: $requestId" + } + requestJob.start() + } + } } } } finally { - withContext(NonCancellable) { - if (!requestChannel.isClosedForSend) { - requestChannel.close() - } - - //logger?.log("Unprocessed requests: ${activeRequests.keys.joinToString(", ")}") - - tracer.spanBuilder("cancelRequests on shutdown").setParent(tracingContext).use { span -> - for (item in requestChannel) { - cancelRequestOnShutdown(item, span) - } - for (item in activeRequests.values) { - cancelRequestOnShutdown(item, span) - } - activeRequests.clear() + tracer.span("cancelRequests on shutdown") { span -> + for (item in activeRequests.values) { + cancelRequestOnShutdown(item, span) } + activeRequests.clear() } } } - private suspend fun readRequests(requestChannel: Channel>, reader: WorkRequestReader, span: Span) { - while (coroutineContext.isActive) { - val request = try { - runInterruptible(Dispatchers.IO) { - reader.readWorkRequestFromStream() - } - } - catch (e: InterruptedIOException) { - span.recordException(e) - null - } - - if (request == null) { - span.addEvent("stop processing - no more requests") - requestChannel.close() - break - } - - val requestId = request.requestId - - //logger?.log("request(id=$requestId${if (request.cancel) ", cancel=true" else ""}) start") - if (request.cancel) { - withContext(NonCancellable) { - // Theoretically, we could have gotten two singleplex requests, and we can't tell those apart. - // However, that's a violation of the protocol, so we don't try to handle it - // (not least because handling it would be quite error-prone). - val stateRef = activeRequests.get(requestId)?.state - if (stateRef != null) { - if (stateRef.compareAndSet(STARTED, FINISHED) || stateRef.compareAndSet(NOT_STARTED, FINISHED)) { - if (cancelHandler != null) { - cancelHandler(request.requestId) - } - //logger?.log("request(id=$requestId) cancelled before handling") - writeAndRemoveRequest(requestId = requestId, wasCancelled = true) - } - } - } + private suspend fun checkStateAndExecuteRequest(stateRef: AtomicReference, request: T) { + if (stateRef.compareAndSet(NOT_STARTED, STARTED)) { + if (request.verbosity == 0) { + executeRequest(request = request, requestState = stateRef, parentSpan = null, tracer = noopTracer) } else { - if (requestId == 0) { - while (activeRequests.containsKey(0)) { - // Previous singleplex requests can still be in activeRequests for a bit after the response has been sent. - // We need to wait for them to vanish. - delay(1) + tracer.spanBuilder("execute request") + .setAttribute(AttributeKey.stringArrayKey("arguments"), request.arguments.toList()) + .setAttribute("id", request.requestId.toLong()) + .setAttribute("sandboxDir", request.sandboxDir ?: "") + .use { + executeRequest(request = request, requestState = stateRef, parentSpan = it, tracer = tracer) } - } - - val item = RequestState(request = request) - val previous = activeRequests.putIfAbsent(requestId, item) - require(previous == null) { - "Request still active: $requestId" - } - requestChannel.send(item) } } - } - - private fun CoroutineScope.startTaskProcessing(requestChannel: Channel>, tracingContext: Context) { - repeat(Runtime.getRuntime().availableProcessors().coerceAtLeast(2)) { - launch { - for (item in requestChannel) { - val stateRef = item.state - when { - stateRef.compareAndSet(NOT_STARTED, STARTED) -> { - val request = item.request - val span = if (request.verbosity > 0) { - tracer.spanBuilder("execute request") - .setAllAttributes(Attributes.of( - AttributeKey.stringArrayKey("arguments"), request.arguments.toList(), - //AttributeKey.stringArrayKey("inputs"), request.inputs.map { it.path }, - AttributeKey.longKey("id"), request.requestId.toLong(), - AttributeKey.stringKey("sandboxDir"), request.sandboxDir ?: "", - )) - .setParent(tracingContext) - .startSpan() - } - else { - null - } - - try { - handleRequest( - request = request, - requestState = stateRef, - tracingContext = span?.let { tracingContext.with(it) }, - parentSpan = span, - ) - } - catch (e: Throwable) { - span?.recordException(e) - throw e - } - finally { - span?.end() - } - } - else -> { - val state = stateRef.get() - if (state != FINISHED) { - throw IllegalStateException("Already started (state=$state)") - } - } - } - } + else { + val state = stateRef.get() + if (state != FINISHED) { + throw IllegalStateException("Already started (state=$state)") } } } private val outputWriteMutex = Mutex() - private suspend fun writeAndRemoveRequest(requestId: Int, exitCode: Int = 0, outString: String? = null, wasCancelled: Boolean = false) { - withContext(NonCancellable) { - var size = 0 + private suspend inline fun writeRequest( + requestId: Int, + exitCode: Int = 0, + outString: String? = null, + wasCancelled: Boolean = false, + beforeWrite: () -> Boolean, + ) { + var size = 0 + if (exitCode != 0) { + size += CodedOutputStream.computeInt32Size(1, exitCode) + } + if (outString != null) { + size += CodedOutputStream.computeStringSize(2, outString) + } + size += CodedOutputStream.computeInt32Size(3, requestId) + if (wasCancelled) { + size += CodedOutputStream.computeBoolSize(4, true) + } + + val messageSizeWithSizePrefix = CodedOutputStream.computeUInt32SizeNoTag(size) + size + val buffer = PooledByteBufAllocator.DEFAULT.heapBuffer(messageSizeWithSizePrefix, messageSizeWithSizePrefix) + try { + val codedOutput = CodedOutputStream.newInstance(buffer.array(), buffer.arrayOffset(), messageSizeWithSizePrefix) + codedOutput.writeUInt32NoTag(size) if (exitCode != 0) { - size += CodedOutputStream.computeInt32Size(1, exitCode) + codedOutput.writeInt32(1, exitCode) } if (outString != null) { - size += CodedOutputStream.computeStringSize(2, outString) + codedOutput.writeString(2, outString) } - size += CodedOutputStream.computeInt32Size(3, requestId) + codedOutput.writeInt32(3, requestId) if (wasCancelled) { - size += CodedOutputStream.computeBoolSize(4, true) + codedOutput.writeBool(4, true) } outputWriteMutex.withLock { - val bufferSize = (CodedOutputStream.computeUInt32SizeNoTag(size) + size).coerceAtMost(4096) - val codedOutput = CodedOutputStream.newInstance(out, bufferSize) - codedOutput.writeUInt32NoTag(size) + if (!beforeWrite()) { + activeRequests.remove(requestId) + return@withLock + } - if (exitCode != 0) { - codedOutput.writeInt32(1, exitCode) + try { + runInterruptible(Dispatchers.IO) { + out.write(buffer.array(), buffer.arrayOffset(), messageSizeWithSizePrefix) + out.flush() + } } - if (outString != null) { - codedOutput.writeString(2, outString) + finally { + activeRequests.remove(requestId) } - codedOutput.writeInt32(3, requestId) - if (wasCancelled) { - codedOutput.writeBool(4, true) - } - codedOutput.flush() - - activeRequests.remove(requestId) - out.flush() } } + finally { + buffer.release() + } + } + + internal suspend fun executeRequest( + request: T, + requestState: AtomicReference, + parentSpan: Span?, + tracer: Tracer, + ) { + val stringWriter = StringBuilderWriter() + var errorToThrow: Throwable? = null + val exitCode = try { + tracer.span("requestExecutor.execute") { + val baseDir = if (request.sandboxDir.isNullOrEmpty()) workingDir else workingDir.resolve(request.sandboxDir) + requestExecutor.execute(request = request, writer = stringWriter, baseDir = baseDir, tracer = tracer) + } + } + catch (e: CancellationException) { + errorToThrow = e + -1 + } + catch (e: Throwable) { + PrintWriter(stringWriter).use { e.printStackTrace(it) } + if (e is Error) { + errorToThrow = e + } + -1 + } + + val outString = stringWriter.toString() + writeRequest(requestId = request.requestId, exitCode = exitCode, outString = outString.takeIf { it.isNotEmpty() }) { + if (requestState.compareAndSet(STARTED, FINISHED)) { + true + } + else { + parentSpan?.addEvent( + "request state was modified during processing", + Attributes.of(AttributeKey.stringKey("state"), requestState.get().name), + ) + false + } + } + if (request.verbosity > 0) { + parentSpan?.addEvent( + "request processed", + Attributes.of(AttributeKey.longKey("exitCode"), exitCode.toLong(), AttributeKey.stringKey("out"), outString), + ) + } + + if (errorToThrow != null) { + throw errorToThrow + } } private suspend fun cancelRequestOnShutdown(item: RequestState, span: Span) { @@ -321,7 +396,7 @@ internal class WorkRequestHandler internal constructor( "request failed because it was not handled due to worker being forced to exit", Attributes.of(AttributeKey.longKey("request"), requestId.toLong()), ) - writeAndRemoveRequest(requestId = requestId, exitCode = 2) + writeRequest(requestId = requestId, exitCode = 2) { true } } else { val state = stateRef.get() @@ -330,80 +405,6 @@ internal class WorkRequestHandler internal constructor( } } } - - internal suspend fun handleRequest( - request: T, - requestState: AtomicReference, - tracingContext: Context?, - parentSpan: Span?, - ) { - val baseDir = if (request.sandboxDir.isNullOrEmpty()) workingDir else workingDir.resolve(request.sandboxDir) - var exitCode = 1 - val stringWriter = StringBuilderWriter() - var errorToThrow: Throwable? = null - val requestId = request.requestId - val span = if (tracingContext == null) { - null - } - else { - tracer.spanBuilder("requestExecutor.execute").setParent(tracingContext).startSpan() - } - try { - try { - val tracingContext = if (tracingContext == null) Context.root() else tracingContext.with(span!!) - exitCode = requestExecutor.execute( - request = request, - writer = stringWriter, - baseDir = baseDir, - tracingContext = tracingContext, - tracer = if (tracingContext == null) noopTracer else tracer, - ) - } - finally { - span?.end() - } - } - catch (e: CancellationException) { - errorToThrow = e - } - catch (e: Throwable) { - PrintWriter(stringWriter).use { e.printStackTrace(it) } - if (e is Error) { - errorToThrow = e - } - } - finally { - span?.end() - } - - withContext(NonCancellable) { - if (!requestState.compareAndSet(STARTED, FINISHED)) { - parentSpan?.addEvent( - "request state was modified during processing", - Attributes.of(AttributeKey.stringKey("state"), requestState.get().name), - ) - return@withContext - } - - val outString = stringWriter.toString() - writeAndRemoveRequest( - requestId = requestId, - exitCode = exitCode, - outString = outString.takeIf { it.isNotEmpty() }, - ) - - if (request.verbosity > 0) { - parentSpan?.addEvent( - "request processed", - Attributes.of(AttributeKey.longKey("exitCode"), exitCode.toLong(), AttributeKey.stringKey("out"), outString), - ) - } - } - - if (errorToThrow != null) { - throw errorToThrow - } - } } private class StringBuilderWriter : Writer() { diff --git a/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt b/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt index b62d614dcee6..695fc4a6077f 100644 --- a/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt +++ b/build/jvm-rules/src/worker-framework/WorkRequestHandlerTest.kt @@ -18,7 +18,6 @@ package org.jetbrains.bazel.jvm import com.google.devtools.build.lib.worker.WorkerProtocol import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.context.Context import kotlinx.coroutines.* import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test @@ -41,7 +40,7 @@ class WorkRequestHandlerTest { fun normalWorkRequest() { val out = ByteArrayOutputStream() val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> 1 }, + requestExecutor = { args, err, _, _ -> 1 }, //input = ByteArrayInputStream(ByteArray(0)), out = out, tracer = OpenTelemetry.noop().getTracer("noop"), @@ -56,7 +55,7 @@ class WorkRequestHandlerTest { sandboxDir = null, ) runBlocking { - handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED), tracingContext = null, parentSpan = null) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(out.toByteArray().inputStream()) @@ -70,7 +69,7 @@ class WorkRequestHandlerTest { fun multiplexWorkRequest() { val out = ByteArrayOutputStream() val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> 0 }, + requestExecutor = { args, err, _, _ -> 0 }, //input = ByteArray(0).inputStream(), out, tracer = OpenTelemetry.noop().getTracer("noop"), @@ -78,7 +77,7 @@ class WorkRequestHandlerTest { val request = newWorkRequest(listOf("--sources", "A.java")) runBlocking { - handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED), tracingContext = null, parentSpan = null) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(out.toByteArray().inputStream()) @@ -96,7 +95,7 @@ class WorkRequestHandlerTest { val started = Semaphore(0) val workerThreads = AtomicInteger() val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> + requestExecutor = { args, err, _, _ -> // each call to this, runs in its own thread workerThreads.incrementAndGet() started.release() @@ -130,7 +129,7 @@ class WorkRequestHandlerTest { fun testOutput() { val out = ByteArrayOutputStream() val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> + requestExecutor = { args, err, _, _ -> err.appendLine("Failed!") 1 }, @@ -142,7 +141,7 @@ class WorkRequestHandlerTest { val args = listOf("--sources", "A.java") val request = newWorkRequest(args, requestId = 0) runBlocking { - handler.handleRequest(request, AtomicReference(WorkRequestState.STARTED), tracingContext = null, parentSpan = null) + handler.handleRequest(request, AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(out.toByteArray().inputStream()) @@ -156,7 +155,7 @@ class WorkRequestHandlerTest { fun testException() { val out = ByteArrayOutputStream() val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> + requestExecutor = { args, err, _, _ -> throw RuntimeException("Exploded!") }, //input = ByteArray(0).inputStream(), @@ -167,7 +166,7 @@ class WorkRequestHandlerTest { val args = listOf("--sources", "A.java") val request = newWorkRequest(args, 342) runBlocking { - handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED), tracingContext = null, parentSpan = null) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(ByteArrayInputStream(out.toByteArray())) @@ -186,7 +185,7 @@ class WorkRequestHandlerTest { val input = PipedInputStream(src) val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> + requestExecutor = { args, err, _, _ -> handlerCalled = true err.appendLine("Such work! Much progress! Wow!") 1 @@ -234,7 +233,7 @@ class WorkRequestHandlerTest { // we force the regular handling to not finish until after we have read the cancel response, to avoid flakiness val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> + requestExecutor = { args, err, _, _ -> // this handler waits until the main thread has sent a cancel request handlerCalled.release() try { @@ -283,7 +282,7 @@ class WorkRequestHandlerTest { // we force the regular handling to not finish until after we have read the cancel response, to avoid flakiness val inputStream = PipedInputStream(src) val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> + requestExecutor = { args, err, _, _ -> try { waitForCancel.acquire() } @@ -329,7 +328,7 @@ class WorkRequestHandlerTest { // we force the cancel request to not happen until after we have read the normal response, to avoid flakiness val handler = WorkRequestHandler( - requestExecutor = { args, err, _, _, _ -> + requestExecutor = { args, err, _, _ -> handlerCalled.release() err.appendLine("Such work! Much progress! Wow!") 2 @@ -363,7 +362,7 @@ class WorkRequestHandlerTest { fun workRequestHandlerWithWorkRequestCallback() { val out = ByteArrayOutputStream() val handler = WorkRequestHandler( - requestExecutor = { request, err, _, _, _ -> request.arguments.size }, + requestExecutor = { request, err, _, _ -> request.arguments.size }, //ByteArrayInputStream(ByteArray(0)), out, tracer = OpenTelemetry.noop().getTracer("noop"), @@ -372,7 +371,7 @@ class WorkRequestHandlerTest { val args = listOf("--sources", "B.java") val request = newWorkRequest(args, requestId = 0) runBlocking { - handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED), tracingContext = null, parentSpan = null) + handler.handleRequest(request = request, requestState = AtomicReference(WorkRequestState.STARTED)) } val response = WorkResponse.parseDelimitedFrom(ByteArrayInputStream(out.toByteArray())) @@ -405,7 +404,7 @@ private inline fun useHandler( ) { val processor = GlobalScope.async(Dispatchers.Default) { try { - handler.processRequests(WorkRequestReaderWithoutDigest(input), Context.current()) + handler.processRequests(WorkRequestReaderWithoutDigest(input)) } catch (_: CancellationException) { } @@ -436,4 +435,11 @@ private inline fun useHandler( processor.cancel() } } -} \ No newline at end of file +} + +private suspend fun WorkRequestHandler.handleRequest( + request: WorkRequest, + requestState: AtomicReference, +) { + executeRequest(request = request, requestState = requestState, parentSpan = null, tracer = OpenTelemetry.noop().getTracer("noop")) +} diff --git a/build/jvm-rules/src/worker-framework/testHelper.kt b/build/jvm-rules/src/worker-framework/testHelper.kt index 3b363da6ffc8..62bd4c3ed897 100644 --- a/build/jvm-rules/src/worker-framework/testHelper.kt +++ b/build/jvm-rules/src/worker-framework/testHelper.kt @@ -1,6 +1,7 @@ // Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.bazel.jvm +import io.opentelemetry.context.Context import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking @@ -14,6 +15,7 @@ import kotlin.io.path.walk import kotlin.system.exitProcess enum class TestModules(@JvmField val sourcePath: String, private val paramsPath: String) { + STAT_CONFIG("platform/statistics/config/src", "platform/statistics/config/config.jar-0.params"), XML_DOM("platform/util/xmlDom/src", "platform/util/xmlDom/xmlDom.jar-0.params"), PLATFORM_IMPL("platform/platform-impl/src", "platform/platform-impl/ide-impl.jar-0.params"), LANG_IMPL("platform/lang-impl/src", "platform/lang-impl/lang-impl.jar-0.params"), @@ -82,12 +84,14 @@ fun performTestInvocation(execute: suspend (out: Writer, coroutineScope: Corouti val out = StringWriter() val exitCode: Int try { - exitCode = runBlocking(Dispatchers.Default) { + exitCode = runBlocking(Dispatchers.Default + OpenTelemetryContextElement(Context.root())) { execute(out, this) } } finally { - System.out.append(out.toString()) + System.out.appendLine("╔══════════════════════════════════ OUT ══════════════════════════════════╗\n") + System.out.append(out.toString().trim().prependIndent("║ ") + "\n") + System.out.appendLine("╚═════════════════════════════════════════════════════════════════════════╝") } exitProcess(exitCode) diff --git a/build/jvm-rules/src/worker-framework/tracing.kt b/build/jvm-rules/src/worker-framework/tracing.kt index ec58c48f5c53..efc79d0e88c3 100644 --- a/build/jvm-rules/src/worker-framework/tracing.kt +++ b/build/jvm-rules/src/worker-framework/tracing.kt @@ -1,24 +1,41 @@ -// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("ReplaceGetOrSet") + package org.jetbrains.bazel.jvm import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.SpanBuilder import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context import io.opentelemetry.semconv.ExceptionAttributes +import kotlinx.coroutines.withContext import java.util.concurrent.CancellationException +import kotlin.coroutines.AbstractCoroutineContextElement +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.coroutineContext -//fun getExceptionAttributes(e: Throwable): Attributes { -// return Attributes.of( -// ExceptionAttributes.EXCEPTION_MESSAGE, e.message ?: "", -// ExceptionAttributes.EXCEPTION_STACKTRACE, e.stackTraceToString() -// ) -//} +class OpenTelemetryContextElement( + @JvmField val context: Context, +) : AbstractCoroutineContextElement(OpenTelemetryContextElement) { + companion object Key : CoroutineContext.Key +} -inline fun SpanBuilder.use(block: (Span) -> T): T { +suspend inline fun Tracer.span(name: String, crossinline block: suspend (Span) -> T): T { + return spanBuilder(name).use(block) +} + +suspend inline fun SpanBuilder.use(crossinline block: suspend (Span) -> T): T { + val telemetryContext = requireNotNull(coroutineContext.get(OpenTelemetryContextElement)) { + "You must set OpenTelemetryContextElement for the root coroutine scope" + }.context + setParent(telemetryContext) val span = startSpan() try { - return block(span) + return withContext(OpenTelemetryContextElement(telemetryContext.with(span))) { + block(span) + } } catch (e: CancellationException) { span.recordException(e, Attributes.of(ExceptionAttributes.EXCEPTION_ESCAPED, true)) diff --git a/lib/.bazelversion b/lib/.bazelversion index 5210382a7a62..51be2ce6c7ed 100644 --- a/lib/.bazelversion +++ b/lib/.bazelversion @@ -1 +1 @@ -8.0.1 \ No newline at end of file +8.1.0rc1 \ No newline at end of file diff --git a/lib/MODULE.bazel b/lib/MODULE.bazel index f94a0505bc3c..ab027221f0b6 100644 --- a/lib/MODULE.bazel +++ b/lib/MODULE.bazel @@ -1,7 +1,7 @@ module(name = "lib") bazel_dep(name = "rules_kotlin", version = "2.0.0-jb.5") -bazel_dep(name = "rules_java", version = "8.7.1") +bazel_dep(name = "rules_java", version = "8.8.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") archive_override(