From 5b4d0e8b163c0cd70cb38d975fb8d780ff09197d Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Tue, 27 Aug 2024 17:06:07 +0200 Subject: [PATCH] frame-based, HTTP/2-only, Netty-based client utilizing native, high-performance non-JDK transport and TLS engine for uploading and downloading compilation resources GitOrigin-RevId: 219eca685356ccf2d8bace08411b1a8463b42f81 --- .idea/libraries/netty.xml | 126 ++ .idea/libraries/netty_tcnative_boringssl.xml | 48 + .../build-scripts/downloader/src/retry.kt | 9 +- .../intellij.platform.buildScripts.iml | 5 +- .../build/CommunityLibraryLicenses.kt | 1588 +++++++++++------ .../intellij/build/LibraryLicense.kt | 6 +- .../jetbrains/intellij/build/concurrency.kt | 52 + .../http2Client/Http2ClientConnection.kt | 137 ++ .../Http2ClientConnectionFactory.kt | 126 ++ .../http2Client/Http2ConnectionProvider.kt | 216 +++ .../Http2StreamJsonInboundHandler.kt | 73 + .../intellij/build/http2Client/download.kt | 118 ++ .../intellij/build/http2Client/netty-util.kt | 147 ++ .../build/impl/ArchivedCompilationContext.kt | 4 +- .../ArchivedCompilationOutputStorage.kt | 58 + .../impl/compilation/CompilationPartsUtil.kt | 274 ++- .../DirectFixedSizeByteBufferPool.kt | 51 +- .../PortableCompilationCacheDownloader.kt | 2 +- .../PortableCompilationCacheUploader.kt | 4 +- .../compilation/ZstdCompressContextPool.kt | 43 + .../build/impl/compilation/download.kt | 199 +-- .../intellij/build/impl/compilation/http.kt | 10 +- .../intellij/build/impl/compilation/upload.kt | 258 +-- .../impl/sbom/SoftwareBillOfMaterialsImpl.kt | 2 - .../build-scripts/tests/nginx-webdav.conf | 22 +- platform/build-scripts/tests/server.crt | 21 + platform/build-scripts/tests/server.key | 29 + .../intellij/build/CompilationCacheTest.kt | 10 +- 28 files changed, 2564 insertions(+), 1074 deletions(-) create mode 100644 .idea/libraries/netty.xml create mode 100644 .idea/libraries/netty_tcnative_boringssl.xml create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/concurrency.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnection.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnectionFactory.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ConnectionProvider.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2StreamJsonInboundHandler.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/download.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/netty-util.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ArchivedCompilationOutputStorage.kt create mode 100644 platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ZstdCompressContextPool.kt create mode 100644 platform/build-scripts/tests/server.crt create mode 100644 platform/build-scripts/tests/server.key diff --git a/.idea/libraries/netty.xml b/.idea/libraries/netty.xml new file mode 100644 index 000000000000..557eba8496ac --- /dev/null +++ b/.idea/libraries/netty.xml @@ -0,0 +1,126 @@ + + + + + + 92836a17468c54ba07fee5f3325f0f148d175a1c6098e9c4fe5228fad2b2fae1 + + + f8a9afa625134827f93c4b86c2698b99290d5c3e8c532da9d7747cfb92e3d01d + + + ede4b23ebe57b7f99f03aeb51771387b6b89b3a1cd05bd3c5f37c37fec5e77f7 + + + baf8b8aaa85b3b0701451ad9ef5ab4169832d04bd19f6828b2ff9354d06b293b + + + fc82eb46c5ff23f26c81948fbdd270d79eee95386a409e07b3941dc8ac841ef0 + + + ae7b0324a91d896f035f00f3b6cb929dccdd3fbebcee5b35282ce75fd09de985 + + + 93ddb37b386a15fce6e2fb533289709885db77818c93a3ceefd2e9ec249dae1c + + + b8ec622fc97a50b5ff0afc46f661f0c278fdb2c9744d2ce53a5173ed5b33b204 + + + 1d63bd25dd301730b23556aba2f13294e89d68ed6903702306a78d807b5f228f + + + 2649883b71d6a7372b9f421aab0b7d1933e45df38b91079e8244b76fc399949f + + + 6ca2870309fe9c8f762995a27618153a3163ad6282e8ace6d4c53102b530360b + + + e540dcb34f348c4894b223f6af18a3d27f77932c85d355ab95c77159b71881af + + + 4185c2ec260672dbd0a60d65b808a5065d3195b1f60924ae6af920bcaa75e703 + + + 47d20282148e7164e74d0a601b18b77ec86df4e4bf3c35245c97488b47f4f547 + + + a56cfbaaa8156b8dd3c4b6efda389ee08e321732a0c2888d669375bc11c736b8 + + + c6d684065f8c5ad1f5eaf87a7a6f1650934f5d0fec36fb61e2726a39f0eb5c37 + + + 9441736ca20c82ebff3ec24b62837e7e944d86d0de679359600f75417f6b581a + + + f9b9107a7d2b4e8816e5da8a6d5fbe229bfa18048572a010e1399a03c5b063f8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/netty_tcnative_boringssl.xml b/.idea/libraries/netty_tcnative_boringssl.xml new file mode 100644 index 000000000000..5a0c3473a698 --- /dev/null +++ b/.idea/libraries/netty_tcnative_boringssl.xml @@ -0,0 +1,48 @@ + + + + + + df215103b6082caceef6b83ed5bbf61d2072688b8b248e9d86cc0bbdb785b5e4 + + + 669a811a193dc1e7c9ef86cb547a4ab92f0f34cce8f9b842b9029bf5cfa07cc5 + + + 407547388ead01c371ae1de7616fa9ce8bc26aa4b180aa5f0452e23ecc02a8f1 + + + 0c18e0f8c70d801f1711ca9fef1ef9bdd5f9b9afb43292f439459ee780d758b6 + + + a34307997449310bcf327c42c46e2db0067c2adf3d66a19fe18e0fd9981fe162 + + + 3cf31a82c0d2c79b48050f02a60d08e9a17db00759ca1f7920cd35d842c7f95e + + + 4f5a5665d3d8c4b2d5ffc40a0c4b07f94399b7d0a4ee01966df0bfc6f49d4524 + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/platform/build-scripts/downloader/src/retry.kt b/platform/build-scripts/downloader/src/retry.kt index 81fee12c8803..cdf7746bf0d4 100644 --- a/platform/build-scripts/downloader/src/retry.kt +++ b/platform/build-scripts/downloader/src/retry.kt @@ -5,9 +5,10 @@ import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import kotlinx.coroutines.delay import org.jetbrains.intellij.build.dependencies.BuildDependenciesDownloader -import java.util.* import java.util.concurrent.TimeUnit import kotlin.math.min +import kotlin.random.Random +import kotlin.random.asJavaRandom suspend fun retryWithExponentialBackOff( attempts: Int = 5, @@ -17,7 +18,6 @@ suspend fun retryWithExponentialBackOff( onException: suspend (attempt: Int, e: Exception) -> Unit = { attempt, e -> defaultExceptionConsumer(attempt, e) }, action: suspend (attempt: Int) -> T ): T { - val random = Random() var effectiveDelay = initialDelayMs val exceptions = mutableListOf() for (attempt in 1..attempts) try { @@ -43,7 +43,7 @@ suspend fun retryWithExponentialBackOff( delay(effectiveDelay) } effectiveDelay = nextDelay( - random, previousDelay = effectiveDelay, + previousDelay = effectiveDelay, backOffLimitMs = backOffLimitMs, backOffFactor = backOffFactor, backOffJitter = backOffJitter, @@ -63,14 +63,13 @@ private fun defaultExceptionConsumer(attempt: Int, e: Exception) { } private fun nextDelay( - random: Random, previousDelay: Long, backOffLimitMs: Long, backOffFactor: Int, backOffJitter: Double, exceptions: List ): Long { - val nextDelay = min(previousDelay * backOffFactor, backOffLimitMs) + (random.nextGaussian() * previousDelay * backOffJitter).toLong() + val nextDelay = min(previousDelay * backOffFactor, backOffLimitMs) + (Random.asJavaRandom().nextGaussian() * previousDelay * backOffJitter).toLong() if (nextDelay > backOffLimitMs) { throw Exception("Back off limit ${backOffLimitMs}ms exceeded, see suppressed exceptions for details").apply { exceptions.forEach(this::addSuppressed) diff --git a/platform/build-scripts/intellij.platform.buildScripts.iml b/platform/build-scripts/intellij.platform.buildScripts.iml index 8f09e1647a00..47f1cd0d5c10 100644 --- a/platform/build-scripts/intellij.platform.buildScripts.iml +++ b/platform/build-scripts/intellij.platform.buildScripts.iml @@ -238,7 +238,8 @@ - - + + + \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt index f647ec2b6eed..64f67adbe46c 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/CommunityLibraryLicenses.kt @@ -13,18 +13,24 @@ object CommunityLibraryLicenses { @JvmStatic @Suppress("SpellCheckingInspection", "NonAsciiCharacters") val LICENSES_LIST: List = java.util.List.of( - LibraryLicense(name = "A fast Java JSON schema validator", libraryName = "json-schema-validator", - url = "https://github.com/networknt/json-schema-validator") + LibraryLicense( + name = "A fast Java JSON schema validator", libraryName = "json-schema-validator", + url = "https://github.com/networknt/json-schema-validator" + ) .apache("https://github.com/networknt/json-schema-validator/blob/master/LICENSE"), LibraryLicense(name = "aalto-xml", libraryName = "aalto-xml", url = "https://github.com/FasterXML/aalto-xml/") .apache("https://github.com/FasterXML/aalto-xml/blob/master/LICENSE"), androidDependency(name = "AAPT Protos", libraryName = "aapt-proto"), - LibraryLicense(name = "AhoCorasickDoubleArrayTrie", libraryName = "com.hankcs:aho-corasick-double-array-trie", - url = "https://github.com/hankcs/AhoCorasickDoubleArrayTrie") + LibraryLicense( + name = "AhoCorasickDoubleArrayTrie", libraryName = "com.hankcs:aho-corasick-double-array-trie", + url = "https://github.com/hankcs/AhoCorasickDoubleArrayTrie" + ) .apache("https://github.com/hankcs/AhoCorasickDoubleArrayTrie/blob/master/README.md#license") .suppliedByPersons("hankcs"), - LibraryLicense(name = "Allure java commons", libraryName = "io.qameta.allure.java.commons", - url = "https://github.com/allure-framework/allure-java") + LibraryLicense( + name = "Allure java commons", libraryName = "io.qameta.allure.java.commons", + url = "https://github.com/allure-framework/allure-java" + ) .apache("https://github.com/allure-framework/allure-java/blob/master/README.md"), LibraryLicense(name = "Amazon Ion Java", libraryName = "ion", url = "https://github.com/amazon-ion/ion-java") .apache("https://github.com/amazon-ion/ion-java/blob/master/LICENSE") @@ -45,17 +51,24 @@ object CommunityLibraryLicenses { LibraryLicense(name = "Android Jimfs library", libraryName = "jimfs", url = "https://github.com/google/jimfs") .apache("https://github.com/google/jimfs/blob/master/LICENSE"), androidDependency(name = "Android Layout Library", libraryName = "layoutlib"), - LibraryLicense(name = "Android libwebp library", libraryName = "libwebp.jar", - url = "https://github.com/webmproject/libwebp", - version = LibraryLicense.CUSTOM_REVISION).newBsd("https://github.com/webmproject/libwebp/blob/main/COPYING"), + LibraryLicense( + name = "Android libwebp library", libraryName = "libwebp.jar", + url = "https://github.com/webmproject/libwebp", + version = LibraryLicense.CUSTOM_REVISION + ).newBsd("https://github.com/webmproject/libwebp/blob/main/COPYING"), androidDependency(name = "Android SDK Common", libraryName = "android.tools.sdk.common"), androidDependency(name = "Android Studio Platform", libraryName = "studio-platform"), - LibraryLicense(name = "ANTLR 4.9 Runtime", libraryName = "antlr4-runtime-4.9", - url = "https://www.antlr.org").newBsd("https://www.antlr.org/license.html") + LibraryLicense( + name = "ANTLR 4.9 Runtime", libraryName = "antlr4-runtime-4.9", + url = "https://www.antlr.org" + ).newBsd("https://www.antlr.org/license.html") .suppliedByPersons("Terence Parr"), - LibraryLicense(name = "ap-validation", libraryName = "ap-validation", - url = "https://github.com/JetBrains/ap-validation").apache( - "https://github.com/JetBrains/ap-validation/blob/master/LICENSE"), + LibraryLicense( + name = "ap-validation", libraryName = "ap-validation", + url = "https://github.com/JetBrains/ap-validation" + ).apache( + "https://github.com/JetBrains/ap-validation/blob/master/LICENSE" + ), LibraryLicense(libraryName = "apache.logging.log4j.to.slf4j", url = "https://ant.apache.org/") .apache("https://logging.apache.org/log4j/log4j-2.2/license.html") @@ -65,20 +78,28 @@ object CommunityLibraryLicenses { .apache("https://ant.apache.org/license.html"), LibraryLicense(name = "Apache Axis", libraryName = "axis-1.4", version = "1.4", url = "https://axis.apache.org/axis/") .apache("https://svn.apache.org/viewvc/axis/axis1/java/trunk/LICENSE?view=markup"), - LibraryLicense(name = "Apache Commons CLI", libraryName = "commons-cli", - url = "https://commons.apache.org/proper/commons-cli/") + LibraryLicense( + name = "Apache Commons CLI", libraryName = "commons-cli", + url = "https://commons.apache.org/proper/commons-cli/" + ) .apache("https://gitbox.apache.org/repos/asf?p=commons-cli.git;a=blob_plain;f=LICENSE.txt;hb=HEAD"), LibraryLicense(name = "Apache Commons Codec", libraryName = "commons-codec", url = "https://commons.apache.org/proper/commons-codec/") .apache("https://github.com/apache/commons-codec/blob/master/LICENSE.txt"), - LibraryLicense(name = "Apache Commons Collections", libraryName = "commons-collections", - url = "https://commons.apache.org/proper/commons-collections/") + LibraryLicense( + name = "Apache Commons Collections", libraryName = "commons-collections", + url = "https://commons.apache.org/proper/commons-collections/" + ) .apache("https://gitbox.apache.org/repos/asf?p=commons-collections.git;a=blob_plain;f=LICENSE.txt;hb=HEAD"), - LibraryLicense(name = "Apache Commons Compress", libraryName = "commons-compress", - url = "https://commons.apache.org/proper/commons-compress/") + LibraryLicense( + name = "Apache Commons Compress", libraryName = "commons-compress", + url = "https://commons.apache.org/proper/commons-compress/" + ) .apache("https://gitbox.apache.org/repos/asf?p=commons-compress.git;a=blob_plain;f=LICENSE.txt;hb=HEAD") .suppliedByOrganizations(Suppliers.APACHE), - LibraryLicense(name = "Apache Commons Discovery", libraryName = "commons-discovery", - url = "https://commons.apache.org/dormant/commons-discovery/") + LibraryLicense( + name = "Apache Commons Discovery", libraryName = "commons-discovery", + url = "https://commons.apache.org/dormant/commons-discovery/" + ) .apache("https://commons.apache.org/dormant/commons-discovery/license.html") .copyrightText("Copyright © 2002-2011 The Apache Software Foundation. All Rights Reserved.") .suppliedByPersons( @@ -86,19 +107,29 @@ object CommunityLibraryLicenses { "Richard Sitze", "Craig R. McClanahan", "Costin Manolache", "Davanum Srinivas", "Rory Winston" ), - LibraryLicense(name = "Apache Commons HTTPClient", libraryName = "http-client-3.1", version = "3.1  (with patch by JetBrains)", - url = "https://hc.apache.org/httpclient-3.x").apache(), - LibraryLicense(name = "Apache Commons Imaging (JetBrains's fork)", libraryName = "commons-imaging", - url = "https://github.com/JetBrains/intellij-deps-commons-imaging") + LibraryLicense( + name = "Apache Commons HTTPClient", libraryName = "http-client-3.1", version = "3.1  (with patch by JetBrains)", + url = "https://hc.apache.org/httpclient-3.x" + ).apache(), + LibraryLicense( + name = "Apache Commons Imaging (JetBrains's fork)", libraryName = "commons-imaging", + url = "https://github.com/JetBrains/intellij-deps-commons-imaging" + ) .apache("https://github.com/JetBrains/intellij-deps-commons-imaging/blob/master/LICENSE.txt") - .forkedFrom(sourceCodeUrl = "https://github.com/apache/commons-imaging", - groupId = "org.apache.commons", artifactId = "commons-imaging", - revision = "fa201df06edefd329610d210d67caba6802b1211"), - LibraryLicense(name = "Apache Commons IO", libraryName = "commons-io", - url = "https://commons.apache.org/proper/commons-io/") + .forkedFrom( + sourceCodeUrl = "https://github.com/apache/commons-imaging", + groupId = "org.apache.commons", artifactId = "commons-imaging", + revision = "fa201df06edefd329610d210d67caba6802b1211" + ), + LibraryLicense( + name = "Apache Commons IO", libraryName = "commons-io", + url = "https://commons.apache.org/proper/commons-io/" + ) .apache("https://gitbox.apache.org/repos/asf?p=commons-io.git;a=blob_plain;f=LICENSE.txt;hb=HEAD"), - LibraryLicense(name = "Apache Commons Lang", libraryName = "commons-lang3", - url = "https://commons.apache.org/proper/commons-lang/") + LibraryLicense( + name = "Apache Commons Lang", libraryName = "commons-lang3", + url = "https://commons.apache.org/proper/commons-lang/" + ) .apache("https://gitbox.apache.org/repos/asf?p=commons-lang.git;a=blob_plain;f=LICENSE.txt;hb=HEAD") .suppliedByPersons( "Daniel Rall", "Robert Burrell Donkin", "James Carman", "Benedikt Ritter", "Rob Tompkins", "Stephen Colebourne", @@ -106,68 +137,92 @@ object CommunityLibraryLicenses { "Oliver Heger", "Paul Benedict", "Duncan Jones", "Loic Guibert" ) .copyrightText("Copyright © 2001-2023 The Apache Software Foundation. All Rights Reserved."), - LibraryLicense(name = "Apache Commons Logging", libraryName = "commons-logging", - url = "https://commons.apache.org/proper/commons-logging/") + LibraryLicense( + name = "Apache Commons Logging", libraryName = "commons-logging", + url = "https://commons.apache.org/proper/commons-logging/" + ) .apache("https://gitbox.apache.org/repos/asf?p=commons-logging.git;a=blob_plain;f=LICENSE.txt;hb=HEAD"), - LibraryLicense(name = "Apache Commons Math", libraryName = "commons-math3", - url = "https://commons.apache.org/proper/commons-math/").apache(), - LibraryLicense(name = "Apache Commons Net", libraryName = "commons-net", - url = "https://commons.apache.org/proper/commons-net/") + LibraryLicense( + name = "Apache Commons Math", libraryName = "commons-math3", + url = "https://commons.apache.org/proper/commons-math/" + ).apache(), + LibraryLicense( + name = "Apache Commons Net", libraryName = "commons-net", + url = "https://commons.apache.org/proper/commons-net/" + ) .apache("https://gitbox.apache.org/repos/asf?p=commons-net.git;a=blob_plain;f=LICENSE.txt;hb=HEAD"), - LibraryLicense(name = "Apache Commons Text", libraryName = "commons-text", - url = "https://github.com/apache/commons-text") + LibraryLicense( + name = "Apache Commons Text", libraryName = "commons-text", + url = "https://github.com/apache/commons-text" + ) .apache("https://github.com/apache/commons-text/blob/master/LICENSE.txt") .copyrightText("Copyright 2014-2024 The Apache Software Foundation"), LibraryLicense(name = "Apache Ivy", libraryName = "org.apache.ivy", url = "https://github.com/apache/ant-ivy") .apache("https://github.com/apache/ant-ivy/blob/master/LICENSE") .copyrightText("Copyright 2007-2019,2022-2023 The Apache Software Foundation") .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "Apache Lucene", - libraryName = "lucene-core", url = "https://lucene.apache.org/java", - additionalLibraryNames = listOf( - "lucene-suggest", - "lucene-memory", - "lucene-sandbox", - "lucene-codecs", - "lucene-highlighter", - "lucene-queryparser", - "lucene-queries", - "lucene-analysis-common", - "org.apache.lucene:lucene-core:2.4.1" - )).apache() + LibraryLicense( + name = "Apache Lucene", + libraryName = "lucene-core", url = "https://lucene.apache.org/java", + additionalLibraryNames = listOf( + "lucene-suggest", + "lucene-memory", + "lucene-sandbox", + "lucene-codecs", + "lucene-highlighter", + "lucene-queryparser", + "lucene-queries", + "lucene-analysis-common", + "org.apache.lucene:lucene-core:2.4.1" + ) + ).apache() .copyrightText("Copyright © 2011-2024 The Apache Software Foundation") .suppliedByOrganizations(Suppliers.APACHE), - LibraryLicense(name = "Apache Tuweni-Toml", libraryName = "tuweni-toml", - url = "https://github.com/apache/incubator-tuweni/tree/main/toml") + LibraryLicense( + name = "Apache Tuweni-Toml", libraryName = "tuweni-toml", + url = "https://github.com/apache/incubator-tuweni/tree/main/toml" + ) .apache("https://github.com/apache/incubator-tuweni/blob/main/LICENSE") .copyrightText("Copyright 2019-2023 The Apache Software Foundation"), - LibraryLicense(name = "AsciiDoc support for Visual Studio Code", attachedTo = "intellij.textmate", version = "3.2.4", - url = "https://github.com/asciidoctor/asciidoctor-vscode") + LibraryLicense( + name = "AsciiDoc support for Visual Studio Code", attachedTo = "intellij.textmate", version = "3.2.4", + url = "https://github.com/asciidoctor/asciidoctor-vscode" + ) .mit("https://github.com/asciidoctor/asciidoctor-vscode/blob/master/README.md"), - LibraryLicense(name = "ASM (JetBrains's fork)", libraryName = "ASM", - url = "https://github.com/JetBrains/intellij-deps-asm") + LibraryLicense( + name = "ASM (JetBrains's fork)", libraryName = "ASM", + url = "https://github.com/JetBrains/intellij-deps-asm" + ) .newBsd("https://github.com/JetBrains/intellij-deps-asm/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS) - .forkedFrom(sourceCodeUrl = "https://gitlab.ow2.org/asm/asm", - mavenRepositoryUrl = "https://repo1.maven.org/maven2", - groupId = "org.ow2.asm", artifactId = "asm", - version = "9.5", - authors = "Guillaume Sauthier, Eric Bruneton, Eugene Kuleshov, Remi Forax"), + .forkedFrom( + sourceCodeUrl = "https://gitlab.ow2.org/asm/asm", + mavenRepositoryUrl = "https://repo1.maven.org/maven2", + groupId = "org.ow2.asm", artifactId = "asm", + version = "9.5", + authors = "Guillaume Sauthier, Eric Bruneton, Eugene Kuleshov, Remi Forax" + ), LibraryLicense(name = "ASM Tools", libraryName = "asm-tools", url = "https://asm.ow2.io") .newBsd("https://asm.ow2.io/license.html"), - LibraryLicense(name = "AssertJ fluent assertions", libraryName = "assertJ", - url = "https://github.com/assertj/assertj-core") + LibraryLicense( + name = "AssertJ fluent assertions", libraryName = "assertJ", + url = "https://github.com/assertj/assertj-core" + ) .apache("https://github.com/assertj/assertj-core/blob/main/LICENSE.txt") .suppliedByPersons( "Pascal Schumacher", "Joel Costigliola", "Stefano Cordio", "Erhard Pointl", "Christian Rösch", "Julien Roy", "Régis Pouiller", "Florent Biville", "Patrick Allain" ), - LibraryLicense(name = "AssertJ Swing", libraryName = "assertj-swing", - url = "https://github.com/assertj/assertj-swing") + LibraryLicense( + name = "AssertJ Swing", libraryName = "assertj-swing", + url = "https://github.com/assertj/assertj-swing" + ) .apache("https://github.com/assertj/assertj-swing/blob/main/licence-header.txt") .suppliedByPersons("Joel Costigliola", "Joel Costigliola", "Christian Rösch", "Alex Ruiz", "Yvonne Wang", "Ansgar Konermann"), - LibraryLicense(name = "Atlassian Commonmark", libraryName = "atlassian.commonmark", - url = "https://github.com/commonmark/commonmark-java") + LibraryLicense( + name = "Atlassian Commonmark", libraryName = "atlassian.commonmark", + url = "https://github.com/commonmark/commonmark-java" + ) .simplifiedBsd("https://github.com/commonmark/commonmark-java/blob/main/LICENSE.txt") .copyrightText("Copyright (c) 2015, Atlassian Pty Ltd") .suppliedByOrganizations("Atlassian Pty Ltd"), @@ -181,11 +236,15 @@ object CommunityLibraryLicenses { .suppliedByOrganizations("The Apache Software Foundation"), LibraryLicense(name = "bifurcan", libraryName = "io.lacuna:bifurcan", url = "https://github.com/lacuna/bifurcan") .mit("https://github.com/lacuna/bifurcan/blob/master/LICENSE"), - LibraryLicense(libraryName = "blockmap", - url = "https://github.com/JetBrains/plugin-blockmap-patches") + LibraryLicense( + libraryName = "blockmap", + url = "https://github.com/JetBrains/plugin-blockmap-patches" + ) .apache("https://github.com/JetBrains/plugin-blockmap-patches/blob/master/LICENSE"), - LibraryLicense(name = "Bodymovin", url = "https://github.com/airbnb/lottie-web", - version = "5.5.10", attachedTo = "intellij.platform.ide.newUiOnboarding") + LibraryLicense( + name = "Bodymovin", url = "https://github.com/airbnb/lottie-web", + version = "5.5.10", attachedTo = "intellij.platform.ide.newUiOnboarding" + ) .mit("https://github.com/airbnb/lottie-web/blob/master/LICENSE.md"), LibraryLicense(libraryName = "bouncy-castle-pgp", url = "https://www.bouncycastle.org") .mit("https://www.bouncycastle.org/license.html") @@ -193,58 +252,84 @@ object CommunityLibraryLicenses { LibraryLicense(libraryName = "bouncy-castle-provider", url = "https://www.bouncycastle.org") .mit("https://www.bouncycastle.org/license.html") .suppliedByOrganizations("The Legion of the Bouncy Castle Inc."), - LibraryLicense(name = "caffeine", libraryName = "caffeine", - url = "https://github.com/ben-manes/caffeine") + LibraryLicense( + name = "caffeine", libraryName = "caffeine", + url = "https://github.com/ben-manes/caffeine" + ) .apache("https://github.com/ben-manes/caffeine/blob/master/LICENSE") .suppliedByPersons("Ben Manes"), LibraryLicense(name = "CGLib", libraryName = "cglib", url = "https://github.com/cglib/cglib/") .apache("https://github.com/cglib/cglib/blob/master/LICENSE") .copyrightText("Copyright (c) The Apache Software Foundation") .suppliedByPersons("cglib project contributors"), - LibraryLicense(name = "classgraph", libraryName = "classgraph", license = "codehaus", - url = "https://github.com/classgraph/classgraph", - licenseUrl = "https://github.com/codehaus/classworlds/blob/master/classworlds/LICENSE.txt"), + LibraryLicense( + name = "classgraph", libraryName = "classgraph", license = "codehaus", + url = "https://github.com/classgraph/classgraph", + licenseUrl = "https://github.com/codehaus/classworlds/blob/master/classworlds/LICENSE.txt" + ), LibraryLicense(name = "Clikt", libraryName = "clikt", url = "https://github.com/ajalt/clikt") .apache("https://github.com/ajalt/clikt/blob/master/LICENSE.txt") .copyrightText("Copyright 2018 AJ Alt") .suppliedByOrganizations("AJ Alt"), - LibraryLicense(name = "CMake For VisualStudio Code", attachedTo = "intellij.textmate", version = "0.0.17", - url = "https://github.com/twxs/vs.language.cmake") + LibraryLicense( + name = "CMake For VisualStudio Code", attachedTo = "intellij.textmate", version = "0.0.17", + url = "https://github.com/twxs/vs.language.cmake" + ) .mit("https://github.com/twxs/vs.language.cmake/blob/master/LICENSE"), - LibraryLicense(name = "Command Line Interface Parser for Java", libraryName = "cli-parser", - url = "https://code.google.com/p/cli-parser/").apache() + LibraryLicense( + name = "Command Line Interface Parser for Java", libraryName = "cli-parser", + url = "https://code.google.com/p/cli-parser/" + ).apache() .copyrightText("Copyright 2012 Sam Pullara"), - LibraryLicense(name = "Common Annotations for the JavaTM Platform API", libraryName = "javax.annotation-api", - url = "https://github.com/javaee/javax.annotation", - license = "CDDL 1.1 / GPL 2.0 + Classpath", licenseUrl = "https://oss.oracle.com/licenses/CDDL+GPL-1.1"), + LibraryLicense( + name = "Common Annotations for the JavaTM Platform API", libraryName = "javax.annotation-api", + url = "https://github.com/javaee/javax.annotation", + license = "CDDL 1.1 / GPL 2.0 + Classpath", licenseUrl = "https://oss.oracle.com/licenses/CDDL+GPL-1.1" + ), // for ui-animation-tooling-internal module library in intellij.android.compose-designer - LibraryLicense(name = "Compose Animation Tooling", libraryName = "ui-animation-tooling-internal", version = "0.1.0-SNAPSHOT", - url = "https://source.android.com/").apache() + LibraryLicense( + name = "Compose Animation Tooling", libraryName = "ui-animation-tooling-internal", version = "0.1.0-SNAPSHOT", + url = "https://source.android.com/" + ).apache() .suppliedByOrganizations(Suppliers.GOOGLE), - LibraryLicense(name = "Compose Multiplatform", libraryName = "jetbrains.compose.foundation.desktop", - url = "https://github.com/JetBrains/compose-multiplatform") + LibraryLicense( + name = "Compose Multiplatform", libraryName = "jetbrains.compose.foundation.desktop", + url = "https://github.com/JetBrains/compose-multiplatform" + ) .apache("https://github.com/JetBrains/compose-multiplatform/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Compose Multiplatform Compiler", libraryName = "jetbrains.compose.compiler.hosted", - url = "https://github.com/JetBrains/compose-multiplatform") + LibraryLicense( + name = "Compose Multiplatform Compiler", libraryName = "jetbrains.compose.compiler.hosted", + url = "https://github.com/JetBrains/compose-multiplatform" + ) .apache("https://github.com/JetBrains/compose-multiplatform/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), // For ADB wireless QR Code generation - LibraryLicense(name = "Core barcode encoding/decoding library", url = "https://github.com/zxing/zxing/tree/master/core", - libraryName = "zxing-core").apache("https://github.com/zxing/zxing/blob/master/LICENSE") + LibraryLicense( + name = "Core barcode encoding/decoding library", url = "https://github.com/zxing/zxing/tree/master/core", + libraryName = "zxing-core" + ).apache("https://github.com/zxing/zxing/blob/master/LICENSE") .suppliedByOrganizations("ZXing Authors"), - LibraryLicense(name = "coverage-report", libraryName = "coverage-report", - url = "https://github.com/JetBrains/coverage-report") + LibraryLicense( + name = "coverage-report", libraryName = "coverage-report", + url = "https://github.com/JetBrains/coverage-report" + ) .apache("https://github.com/JetBrains/coverage-report/blob/master/LICENSE"), - LibraryLicense(name = "coverage.py", attachedTo = "intellij.python", version = "4.2.0", - url = "https://coverage.readthedocs.io/") + LibraryLicense( + name = "coverage.py", attachedTo = "intellij.python", version = "4.2.0", + url = "https://coverage.readthedocs.io/" + ) .apache("https://github.com/nedbat/coveragepy/blob/master/LICENSE.txt"), - LibraryLicense(name = "Cucumber-Core", libraryName = "cucumber-core-1.2", - url = "https://github.com/cucumber/cucumber-jvm/blob/main/LICENSE") + LibraryLicense( + name = "Cucumber-Core", libraryName = "cucumber-core-1.2", + url = "https://github.com/cucumber/cucumber-jvm/blob/main/LICENSE" + ) .mit("https://github.com/cucumber/cucumber-jvm/blob/main/LICENSE") .suppliedByOrganizations("SmartBear Software"), - LibraryLicense(name = "Cucumber-Expressions", libraryName = "cucumber-expressions", - url = "https://github.com/cucumber/cucumber/") + LibraryLicense( + name = "Cucumber-Expressions", libraryName = "cucumber-expressions", + url = "https://github.com/cucumber/cucumber/" + ) .mit("https://github.com/cucumber/cucumber-jvm/blob/main/LICENSE") .suppliedByOrganizations("SmartBear Software"), LibraryLicense(name = "Cucumber-Groovy", libraryName = "cucumber-groovy", url = "https://github.com/cucumber/cucumber-jvm/") @@ -253,23 +338,35 @@ object CommunityLibraryLicenses { LibraryLicense(name = "Cucumber-Java", libraryName = "cucumber-java", url = "https://github.com/cucumber/cucumber-jvm/") .mit("https://github.com/cucumber/cucumber-jvm/blob/main/LICENSE") .suppliedByOrganizations("SmartBear Software"), - LibraryLicense(name = "Dart Analysis Server", attachedTo = "intellij.dart", - url = "https://github.com/dart-lang/eclipse3", version = LibraryLicense.CUSTOM_REVISION).eplV1(), - LibraryLicense(name = "Dart VM Service drivers", attachedTo = "intellij.dart", - url = "https://github.com/dart-lang/vm_service_drivers", - version = LibraryLicense.CUSTOM_REVISION) + LibraryLicense( + name = "Dart Analysis Server", attachedTo = "intellij.dart", + url = "https://github.com/dart-lang/eclipse3", version = LibraryLicense.CUSTOM_REVISION + ).eplV1(), + LibraryLicense( + name = "Dart VM Service drivers", attachedTo = "intellij.dart", + url = "https://github.com/dart-lang/vm_service_drivers", + version = LibraryLicense.CUSTOM_REVISION + ) .newBsd("https://github.com/dart-lang/vm_service_drivers/blob/master/LICENSE"), - LibraryLicense(name = "dbus-java", libraryName = "dbus-java", license = "LGPL", - url = "https://github.com/hypfvieh/dbus-java", - licenseUrl = "https://github.com/hypfvieh/dbus-java/blob/dbus-java-3.0/LICENSE") + LibraryLicense( + name = "dbus-java", libraryName = "dbus-java", license = "LGPL", + url = "https://github.com/hypfvieh/dbus-java", + licenseUrl = "https://github.com/hypfvieh/dbus-java/blob/dbus-java-3.0/LICENSE" + ) .suppliedByPersons("David M. "), - LibraryLicense(name = "docutils", attachedTo = "intellij.python", version = "0.12", - url = "https://docutils.sourceforge.io/").simplifiedBsd(), - LibraryLicense(name = "dotenv-kotlin", libraryName = "io.github.cdimascio.dotenv.kotlin", - url = "https://github.com/cdimascio/dotenv-kotlin") + LibraryLicense( + name = "docutils", attachedTo = "intellij.python", version = "0.12", + url = "https://docutils.sourceforge.io/" + ).simplifiedBsd(), + LibraryLicense( + name = "dotenv-kotlin", libraryName = "io.github.cdimascio.dotenv.kotlin", + url = "https://github.com/cdimascio/dotenv-kotlin" + ) .apache("https://github.com/cdimascio/dotenv-kotlin/blob/master/LICENSE"), - LibraryLicense(name = "Eclipse JDT Core", attachedTo = "intellij.platform.jps.build", version = "4.2.1", license = "CPL 1.0", - url = "https://www.eclipse.org/jdt/core/index.php"), + LibraryLicense( + name = "Eclipse JDT Core", attachedTo = "intellij.platform.jps.build", version = "4.2.1", license = "CPL 1.0", + url = "https://www.eclipse.org/jdt/core/index.php" + ), LibraryLicense(name = "Eclipse Layout Kernel", url = "https://www.eclipse.org/elk/", libraryName = "eclipse-layout-kernel") .eplV1("https://github.com/eclipse/elk/blob/master/LICENSE.md") .suppliedByOrganizations(Suppliers.ECLIPSE), @@ -292,45 +389,64 @@ object CommunityLibraryLicenses { ffmpegLibraryLicense("ffmpeg-macos-aarch64"), ffmpegLibraryLicense("ffmpeg-macos-x64"), ffmpegLibraryLicense("ffmpeg-windows-x64"), - LibraryLicense(name = "FiraCode", attachedTo = "intellij.platform.resources", version = "1.206", license = "OFL", - url = "https://github.com/tonsky/FiraCode", licenseUrl = "https://github.com/tonsky/FiraCode/blob/master/LICENSE"), - LibraryLicense(name = "FreeMarker", attachedTo = "intellij.java.coverage", version = "2.3.30", - url = "https://freemarker.apache.org") + LibraryLicense( + name = "FiraCode", attachedTo = "intellij.platform.resources", version = "1.206", license = "OFL", + url = "https://github.com/tonsky/FiraCode", licenseUrl = "https://github.com/tonsky/FiraCode/blob/master/LICENSE" + ), + LibraryLicense( + name = "FreeMarker", attachedTo = "intellij.java.coverage", version = "2.3.30", + url = "https://freemarker.apache.org" + ) .apache("https://freemarker.apache.org/docs/app_license.html"), - LibraryLicense(name = "gauge-java", libraryName = "com.thoughtworks.gauge:gauge-java", - url = "https://github.com/getgauge/gauge-java/") + LibraryLicense( + name = "gauge-java", libraryName = "com.thoughtworks.gauge:gauge-java", + url = "https://github.com/getgauge/gauge-java/" + ) .apache("https://github.com/getgauge/gauge-java/raw/master/LICENSE.txt"), - LibraryLicense(name = "Gherkin", libraryName = "gherkin", - url = "https://github.com/cucumber/gherkin/tree/main") + LibraryLicense( + name = "Gherkin", libraryName = "gherkin", + url = "https://github.com/cucumber/gherkin/tree/main" + ) .mit("https://github.com/cucumber/gherkin/blob/main/LICENSE") .suppliedByOrganizations("Cucumber Ltd"), - LibraryLicense(name = "Gherkin keywords", attachedTo = "intellij.gherkin", version = "2.12.2", - url = "https://github.com/cucumber/gherkin/tree/main") + LibraryLicense( + name = "Gherkin keywords", attachedTo = "intellij.gherkin", version = "2.12.2", + url = "https://github.com/cucumber/gherkin/tree/main" + ) .mit("https://github.com/cucumber/gherkin/blob/main/LICENSE") .suppliedByOrganizations("Cucumber Ltd"), LibraryLicense(url = "https://github.com/oshi/oshi", libraryName = "github.oshi.core").mit( - "https://github.com/oshi/oshi/blob/master/LICENSE") + "https://github.com/oshi/oshi/blob/master/LICENSE" + ) .suppliedByOrganizations("The OSHI Project Contributors"), LibraryLicense(name = "googlecode.plist.dd", libraryName = "googlecode.plist.dd", url = "https://github.com/3breadt/dd-plist/") .mit("https://github.com/3breadt/dd-plist/blob/master/LICENSE.txt"), LibraryLicense(libraryName = "Gradle", url = "https://gradle.org/", licenseUrl = "https://gradle.org/license") .apache("https://github.com/gradle/gradle/blob/master/LICENSE") .suppliedByOrganizations("Gradle Inc."), - LibraryLicense(name = "GraphQL Java", url = "https://github.com/graphql-java/graphql-java", - attachedTo = "intellij.graphql", version = LibraryLicense.CUSTOM_REVISION) + LibraryLicense( + name = "GraphQL Java", url = "https://github.com/graphql-java/graphql-java", + attachedTo = "intellij.graphql", version = LibraryLicense.CUSTOM_REVISION + ) .mit("https://github.com/graphql-java/graphql-java/blob/master/LICENSE.md"), - LibraryLicense(name = "GraphQL Java Dataloader", libraryName = "graphql.java.dataloader", - url = "https://github.com/graphql-java/java-dataloader") + LibraryLicense( + name = "GraphQL Java Dataloader", libraryName = "graphql.java.dataloader", + url = "https://github.com/graphql-java/java-dataloader" + ) .apache("https://github.com/graphql-java/java-dataloader/blob/master/LICENSE"), - LibraryLicense(name = "Grazie AI", libraryName = "ai.grazie.spell.gec.engine.local", - url = "https://packages.jetbrains.team/maven/p/grazi/grazie-platform-public/", - additionalLibraryNames = listOf("ai.grazie.nlp.langs", - "ai.grazie.nlp.detect", - "ai.grazie.utils.lucene.lt.compatibility", - "ai.grazie.spell.hunspell.en", - "ai.grazie.emb", - "ai.grazie.utils.ki", - "ai.grazie.nlp.encoder.bert.uncased")).apache() + LibraryLicense( + name = "Grazie AI", libraryName = "ai.grazie.spell.gec.engine.local", + url = "https://packages.jetbrains.team/maven/p/grazi/grazie-platform-public/", + additionalLibraryNames = listOf( + "ai.grazie.nlp.langs", + "ai.grazie.nlp.detect", + "ai.grazie.utils.lucene.lt.compatibility", + "ai.grazie.spell.hunspell.en", + "ai.grazie.emb", + "ai.grazie.utils.ki", + "ai.grazie.nlp.encoder.bert.uncased" + ) + ).apache() .suppliedByOrganizations(Suppliers.JETBRAINS), LibraryLicense(name = "Groovy", libraryName = "org.codehaus.groovy:groovy", url = "https://groovy-lang.org/") .apache("https://github.com/apache/groovy/blob/master/LICENSE"), @@ -340,11 +456,15 @@ object CommunityLibraryLicenses { .apache("https://github.com/apache/groovy/blob/master/LICENSE"), LibraryLicense(name = "Groovy JSR-223", libraryName = "org.codehaus.groovy:groovy-jsr223", url = "https://groovy-lang.org/") .apache("https://github.com/apache/groovy/blob/master/LICENSE"), - LibraryLicense(name = "Groovy Templates", libraryName = "org.codehaus.groovy:groovy-templates", - url = "https://groovy-lang.org/") + LibraryLicense( + name = "Groovy Templates", libraryName = "org.codehaus.groovy:groovy-templates", + url = "https://groovy-lang.org/" + ) .apache("https://github.com/apache/groovy/blob/master/LICENSE"), - LibraryLicense(name = "Groovy XML", libraryName = "org.codehaus.groovy:groovy-xml", - url = "https://groovy-lang.org/") + LibraryLicense( + name = "Groovy XML", libraryName = "org.codehaus.groovy:groovy-xml", + url = "https://groovy-lang.org/" + ) .apache("https://github.com/apache/groovy/blob/master/LICENSE"), LibraryLicense(libraryName = "grpc-inprocess", url = "https://grpc.io/") @@ -375,63 +495,91 @@ object CommunityLibraryLicenses { .suppliedByPersons("Joe Walnes", "Nat Pryce", "Steve Freeman"), LibraryLicense(libraryName = "hash4j", url = "https://github.com/dynatrace-oss/hash4j") .apache("https://github.com/dynatrace-oss/hash4j/blob/main/LICENSE"), - LibraryLicense(name = "HashiCorp Syntax", attachedTo = "intellij.textmate", version = "0.6.0", - url = "https://github.com/asciidoctor/asciidoctor-vscode", - license = "MPL-2.0", - licenseUrl = "https://github.com/hashicorp/syntax/blob/main/LICENSE"), - LibraryLicense(name = "HDR Histogram", libraryName = "HdrHistogram", license = "CC0 1.0 Universal", - url = "https://github.com/HdrHistogram/HdrHistogram", - licenseUrl = "https://github.com/HdrHistogram/HdrHistogram/blob/master/LICENSE.txt") + LibraryLicense( + name = "HashiCorp Syntax", attachedTo = "intellij.textmate", version = "0.6.0", + url = "https://github.com/asciidoctor/asciidoctor-vscode", + license = "MPL-2.0", + licenseUrl = "https://github.com/hashicorp/syntax/blob/main/LICENSE" + ), + LibraryLicense( + name = "HDR Histogram", libraryName = "HdrHistogram", license = "CC0 1.0 Universal", + url = "https://github.com/HdrHistogram/HdrHistogram", + licenseUrl = "https://github.com/HdrHistogram/HdrHistogram/blob/master/LICENSE.txt" + ) .suppliedByPersons("Gil Tene"), LibraryLicense(name = "hppc", url = "https://github.com/carrotsearch/hppc", libraryName = "com.carrotsearch:hppc") .apache("https://github.com/carrotsearch/hppc/blob/master/LICENSE.txt") .suppliedByPersons("Stanisław Osiński", "Dawid Weiss", "Bruno Roustant"), - LibraryLicense(name = "htmlparser2", - url = "https://github.com/fb55/htmlparser2", attachedTo = "intellij.vuejs", - version = LibraryLicense.CUSTOM_REVISION) + LibraryLicense( + name = "htmlparser2", + url = "https://github.com/fb55/htmlparser2", attachedTo = "intellij.vuejs", + version = LibraryLicense.CUSTOM_REVISION + ) .mit("https://github.com/fb55/htmlparser2/blob/master/LICENSE"), - LibraryLicense(name = "HttpComponents HttpClient", libraryName = "http-client", - url = "https://hc.apache.org/httpcomponents-client-ga/").apache() + LibraryLicense( + name = "HttpComponents HttpClient", libraryName = "http-client", + url = "https://hc.apache.org/httpcomponents-client-ga/" + ).apache() .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "HttpComponents HttpClient Fluent API", libraryName = "fluent-hc", - url = "https://hc.apache.org/httpcomponents-client-ga/").apache() + LibraryLicense( + name = "HttpComponents HttpClient Fluent API", libraryName = "fluent-hc", + url = "https://hc.apache.org/httpcomponents-client-ga/" + ).apache() .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "ICU4J", libraryName = "icu4j", license = "Unicode", - url = "https://icu.unicode.org/", licenseUrl = "https://www.unicode.org/copyright.html"), + LibraryLicense( + name = "ICU4J", libraryName = "icu4j", license = "Unicode", + url = "https://icu.unicode.org/", licenseUrl = "https://www.unicode.org/copyright.html" + ), LibraryLicense(name = "imgscalr", libraryName = "imgscalr", url = "https://github.com/thebuzzmedia/imgscalr") .apache("https://github.com/rkalla/imgscalr/blob/master/LICENSE"), - LibraryLicense(name = "Inconsolata", attachedTo = "intellij.platform.resources", version = "001.010", license = "OFL", - url = "https://github.com/google/fonts/tree/main/ofl/inconsolata", - licenseUrl = "https://github.com/google/fonts/blob/master/ofl/inconsolata/OFL.txt"), - LibraryLicense(name = "Incremental DOM", attachedTo = "intellij.markdown", version = "0.7.0", - url = "https://github.com/google/incremental-dom") + LibraryLicense( + name = "Inconsolata", attachedTo = "intellij.platform.resources", version = "001.010", license = "OFL", + url = "https://github.com/google/fonts/tree/main/ofl/inconsolata", + licenseUrl = "https://github.com/google/fonts/blob/master/ofl/inconsolata/OFL.txt" + ), + LibraryLicense( + name = "Incremental DOM", attachedTo = "intellij.markdown", version = "0.7.0", + url = "https://github.com/google/incremental-dom" + ) .apache("https://github.com/google/incremental-dom/blob/master/LICENSE"), - LibraryLicense(name = "indriya", libraryName = "tech.units:indriya:1.3", - url = "https://github.com/unitsofmeasurement/indriya", - licenseUrl = "https://github.com/unitsofmeasurement/indriya/blob/master/LICENSE") + LibraryLicense( + name = "indriya", libraryName = "tech.units:indriya:1.3", + url = "https://github.com/unitsofmeasurement/indriya", + licenseUrl = "https://github.com/unitsofmeasurement/indriya/blob/master/LICENSE" + ) .newBsd("https://github.com/unitsofmeasurement/indriya/blob/master/LICENSE") .suppliedByPersons( "Jean-Marie Dautelle", "Werner Keil", "Otávio Gonçalves de Santana", "Martin Desruisseaux", "Thodoris Bais", "Daniel Dias", "Jacob Glickman", "Magesh Kasthuri" ), - LibraryLicense(name = "ini4j (JetBrains's fork)", libraryName = "ini4j", - url = "https://github.com/JetBrains/intellij-deps-ini4j") + LibraryLicense( + name = "ini4j (JetBrains's fork)", libraryName = "ini4j", + url = "https://github.com/JetBrains/intellij-deps-ini4j" + ) .apache("https://github.com/JetBrains/intellij-deps-ini4j/blob/master/LICENSE.txt") - .forkedFrom(sourceCodeUrl = "https://sourceforge.net/projects/ini4j", - mavenRepositoryUrl = "https://repo1.maven.org/maven2", - groupId = "org.ini4j", artifactId = "ini4j", - version = "0.5.4", - authors = "Ivan Szkiba"), - LibraryLicense(name = "intellij-markdown", libraryName = "jetbrains.markdown", - url = "https://github.com/JetBrains/markdown") + .forkedFrom( + sourceCodeUrl = "https://sourceforge.net/projects/ini4j", + mavenRepositoryUrl = "https://repo1.maven.org/maven2", + groupId = "org.ini4j", artifactId = "ini4j", + version = "0.5.4", + authors = "Ivan Szkiba" + ), + LibraryLicense( + name = "intellij-markdown", libraryName = "jetbrains.markdown", + url = "https://github.com/JetBrains/markdown" + ) .apache("https://github.com/JetBrains/markdown/blob/master/LICENSE"), - LibraryLicense(name = "IntelliJ IDEA Code Coverage Agent", libraryName = "intellij-coverage", - url = "https://github.com/jetbrains/intellij-coverage") + LibraryLicense( + name = "IntelliJ IDEA Code Coverage Agent", libraryName = "intellij-coverage", + url = "https://github.com/jetbrains/intellij-coverage" + ) .apache("https://github.com/JetBrains/intellij-coverage/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "IntelliJ IDEA Test Discovery Agent", libraryName = "intellij-test-discovery", - url = "https://github.com/JetBrains/intellij-coverage/tree/master/test-discovery") + LibraryLicense( + name = "IntelliJ IDEA Test Discovery Agent", libraryName = "intellij-test-discovery", + url = "https://github.com/JetBrains/intellij-coverage/tree/master/test-discovery" + ) .apache("https://github.com/JetBrains/intellij-coverage/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), LibraryLicense(name = "ISO RELAX", libraryName = "isorelax", url = "https://sourceforge.net/projects/iso-relax/").mit() @@ -440,29 +588,41 @@ object CommunityLibraryLicenses { LibraryLicense(name = "Jackson", libraryName = "jackson", url = "https://github.com/FasterXML/jackson") .apache("https://github.com/FasterXML/jackson-core/blob/2.14/LICENSE") .suppliedByPersons("Tatu Saloranta", "Christopher Currie", "Paul Brown"), - LibraryLicense(name = "jackson-jr-objects", libraryName = "jackson-jr-objects", - url = "https://github.com/FasterXML/jackson-jr") + LibraryLicense( + name = "jackson-jr-objects", libraryName = "jackson-jr-objects", + url = "https://github.com/FasterXML/jackson-jr" + ) .apache("https://github.com/FasterXML/jackson-jr/blob/2.16/LICENSE") .suppliedByPersons("Tatu Saloranta", "Christopher Currie", "Paul Brown"), - LibraryLicense(name = "Jackson Databind", libraryName = "jackson-databind", - url = "https://github.com/FasterXML/jackson-databind") + LibraryLicense( + name = "Jackson Databind", libraryName = "jackson-databind", + url = "https://github.com/FasterXML/jackson-databind" + ) .apache("https://github.com/FasterXML/jackson-databind/blob/2.16/LICENSE") .suppliedByPersons("Tatu Saloranta", "Christopher Currie", "Paul Brown"), - LibraryLicense(name = "Jackson Dataformat CBOR", libraryName = "jackson-dataformat-cbor", - url = "https://github.com/FasterXML/jackson-dataformats-binary") + LibraryLicense( + name = "Jackson Dataformat CBOR", libraryName = "jackson-dataformat-cbor", + url = "https://github.com/FasterXML/jackson-dataformats-binary" + ) .apache("https://github.com/FasterXML/jackson-dataformats-binary/blob/2.14/pom.xml") .suppliedByPersons("Tatu Saloranta", "Christopher Currie", "Paul Brown"), - LibraryLicense(name = "Jackson Dataformat YAML", libraryName = "jackson-dataformat-yaml", - url = "https://github.com/FasterXML/jackson-dataformats-text") + LibraryLicense( + name = "Jackson Dataformat YAML", libraryName = "jackson-dataformat-yaml", + url = "https://github.com/FasterXML/jackson-dataformats-text" + ) .apache("https://github.com/FasterXML/jackson-dataformats-text/blob/2.16/pom.xml") .suppliedByPersons("Tatu Saloranta", "Christopher Currie", "Paul Brown"), - LibraryLicense(name = "Jackson Module Kotlin", libraryName = "jackson-module-kotlin", - url = "https://github.com/FasterXML/jackson-module-kotlin") + LibraryLicense( + name = "Jackson Module Kotlin", libraryName = "jackson-module-kotlin", + url = "https://github.com/FasterXML/jackson-module-kotlin" + ) .apache("https://github.com/FasterXML/jackson-module-kotlin/blob/2.16/LICENSE") - .suppliedByPersons("Tatu Saloranta", "Christopher Currie", "Paul Brown", "Jayson Minard", - "Drew Stephens", "Vyacheslav Artemyev", "Dmitry Spikhalskiy"), + .suppliedByPersons( + "Tatu Saloranta", "Christopher Currie", "Paul Brown", "Jayson Minard", + "Drew Stephens", "Vyacheslav Artemyev", "Dmitry Spikhalskiy" + ), LibraryLicense(name = "JaCoCo", libraryName = "JaCoCo", url = "https://www.eclemma.org/jacoco/") .suppliedByOrganizations("Mountainminds GmbH & Co. KG and Contributors") @@ -477,30 +637,44 @@ object CommunityLibraryLicenses { "Michael Davey", "Harald Kuhn", ), - LibraryLicense(name = "Jarchivelib", libraryName = "rauschig.jarchivelib", - url = "https://github.com/thrau/jarchivelib") + LibraryLicense( + name = "Jarchivelib", libraryName = "rauschig.jarchivelib", + url = "https://github.com/thrau/jarchivelib" + ) .apache("https://github.com/thrau/jarchivelib/blob/master/LICENSE"), - LibraryLicense(libraryName = "Java Compatibility", license = "GPL 2.0 + Classpath", - url = "https://github.com/JetBrains/intellij-deps-java-compatibility", - licenseUrl = "https://github.com/JetBrains/intellij-deps-java-compatibility/raw/master/LICENSE"), + LibraryLicense( + libraryName = "Java Compatibility", license = "GPL 2.0 + Classpath", + url = "https://github.com/JetBrains/intellij-deps-java-compatibility", + licenseUrl = "https://github.com/JetBrains/intellij-deps-java-compatibility/raw/master/LICENSE" + ), - LibraryLicense(name = "Java Poet", libraryName = "javapoet", - url = "https://github.com/square/javapoet") + LibraryLicense( + name = "Java Poet", libraryName = "javapoet", + url = "https://github.com/square/javapoet" + ) .apache("https://github.com/square/javapoet/blob/master/LICENSE.txt"), - LibraryLicense(name = "Java Server Pages (JSP) for Visual Studio Code", attachedTo = "intellij.textmate", version = "0.0.3", - url = "https://github.com/pthorsson/vscode-jsp") + LibraryLicense( + name = "Java Server Pages (JSP) for Visual Studio Code", attachedTo = "intellij.textmate", version = "0.0.3", + url = "https://github.com/pthorsson/vscode-jsp" + ) .mit("https://github.com/pthorsson/vscode-jsp/blob/master/LICENSE"), - LibraryLicense(name = "Java Simple Serial Connector", libraryName = "io.github.java.native.jssc", - url = "https://github.com/java-native/jssc", license = "LGPL 3.0", - licenseUrl = "https://github.com/java-native/jssc/blob/master/LICENSE.txt"), - LibraryLicense(name = "Java String Similarity", libraryName = "java-string-similarity", - url = "https://github.com/tdebatty/java-string-similarity") + LibraryLicense( + name = "Java Simple Serial Connector", libraryName = "io.github.java.native.jssc", + url = "https://github.com/java-native/jssc", license = "LGPL 3.0", + licenseUrl = "https://github.com/java-native/jssc/blob/master/LICENSE.txt" + ), + LibraryLicense( + name = "Java String Similarity", libraryName = "java-string-similarity", + url = "https://github.com/tdebatty/java-string-similarity" + ) .mit("https://github.com/tdebatty/java-string-similarity/blob/master/LICENSE.md") .suppliedByPersons("Thibault Debatty"), - LibraryLicense(name = "JavaBeans Activation Framework", libraryName = "javax.activation", - url = "https://github.com/javaee/activation", - license = "CDDL 1.1 / GPL 2.0 + Classpath", - licenseUrl = "https://github.com/javaee/activation/blob/master/LICENSE.txt") + LibraryLicense( + name = "JavaBeans Activation Framework", libraryName = "javax.activation", + url = "https://github.com/javaee/activation", + license = "CDDL 1.1 / GPL 2.0 + Classpath", + licenseUrl = "https://github.com/javaee/activation/blob/master/LICENSE.txt" + ) .suppliedByPersons("Bill Shannon"), ffmpegLibraryLicense("javacpp-linux-x64"), ffmpegLibraryLicense("javacpp-macos-aarch64"), @@ -508,111 +682,159 @@ object CommunityLibraryLicenses { ffmpegLibraryLicense("javacpp-windows-x64"), LibraryLicense(name = "javaslang", libraryName = "javaslang", url = "https://javaslang.io/").apache() .suppliedByPersons("Daniel Dietrich"), - LibraryLicense(name = "javawriter", attachedTo = "intellij.android.core", - url = "https://github.com/square/javawriter", - version = LibraryLicense.CUSTOM_REVISION).apache(), - LibraryLicense(name = "javax inject", libraryName = "javax-inject", - url = "https://code.google.com/p/atinject/").apache() + LibraryLicense( + name = "javawriter", attachedTo = "intellij.android.core", + url = "https://github.com/square/javawriter", + version = LibraryLicense.CUSTOM_REVISION + ).apache(), + LibraryLicense( + name = "javax inject", libraryName = "javax-inject", + url = "https://code.google.com/p/atinject/" + ).apache() .suppliedByOrganizations(Suppliers.GOOGLE), - LibraryLicense(name = "JAXB (Java Architecture for XML Binding) API", libraryName = "jaxb-api", - url = "https://github.com/javaee/jaxb-spec", - license = "CDDL 1.1 / GPL 2.0 + Classpath", licenseUrl = "https://oss.oracle.com/licenses/CDDL+GPL-1.1") + LibraryLicense( + name = "JAXB (Java Architecture for XML Binding) API", libraryName = "jaxb-api", + url = "https://github.com/javaee/jaxb-spec", + license = "CDDL 1.1 / GPL 2.0 + Classpath", licenseUrl = "https://oss.oracle.com/licenses/CDDL+GPL-1.1" + ) .suppliedByPersons("Roman Grigoriadi", "Martin Grebac", "Iaroslav Savytskyi"), - LibraryLicense(name = "JAXB (JSR 222) Reference Implementation", libraryName = "jaxb-runtime", - url = "https://github.com/javaee/jaxb-v2", - license = "CDDL 1.1 / GPL 2.0 + Classpath", licenseUrl = "https://oss.oracle.com/licenses/CDDL+GPL-1.1") + LibraryLicense( + name = "JAXB (JSR 222) Reference Implementation", libraryName = "jaxb-runtime", + url = "https://github.com/javaee/jaxb-v2", + license = "CDDL 1.1 / GPL 2.0 + Classpath", licenseUrl = "https://oss.oracle.com/licenses/CDDL+GPL-1.1" + ) .suppliedByOrganizations(Suppliers.ECLIPSE), LibraryLicense(libraryName = "Jaxen", url = "https://github.com/jaxen-xpath/jaxen") .newBsd("https://github.com/jaxen-xpath/jaxen/blob/master/LICENSE.txt"), - LibraryLicense(name = "Jayway JsonPath", libraryName = "jsonpath", - url = "https://github.com/json-path/JsonPath") + LibraryLicense( + name = "Jayway JsonPath", libraryName = "jsonpath", + url = "https://github.com/json-path/JsonPath" + ) .apache("https://github.com/json-path/JsonPath/blob/master/LICENSE"), - LibraryLicense(libraryName = "jb-jdi", license = "GPL 2.0 + Classpath", url = "https://github.com/JetBrains/intellij-deps-jdi", - licenseUrl = "https://github.com/JetBrains/intellij-deps-jdi/raw/master/LICENSE.txt"), - LibraryLicense(name = "JCEF", libraryName = "jcef", license = "BSD 3-Clause", - licenseUrl = "https://bitbucket.org/chromiumembedded/java-cef/src/master/LICENSE.txt", - url = "https://bitbucket.org/chromiumembedded/java-cef") + LibraryLicense( + libraryName = "jb-jdi", license = "GPL 2.0 + Classpath", url = "https://github.com/JetBrains/intellij-deps-jdi", + licenseUrl = "https://github.com/JetBrains/intellij-deps-jdi/raw/master/LICENSE.txt" + ), + LibraryLicense( + name = "JCEF", libraryName = "jcef", license = "BSD 3-Clause", + licenseUrl = "https://bitbucket.org/chromiumembedded/java-cef/src/master/LICENSE.txt", + url = "https://bitbucket.org/chromiumembedded/java-cef" + ) .suppliedByPersons("Marshall A. Greenblatt"), - LibraryLicense(name = "JCIP Annotations", libraryName = "jcip", license = "Creative Commons Attribution License", - url = "https://jcip.net", licenseUrl = "https://creativecommons.org/licenses/by/2.5") + LibraryLicense( + name = "JCIP Annotations", libraryName = "jcip", license = "Creative Commons Attribution License", + url = "https://jcip.net", licenseUrl = "https://creativecommons.org/licenses/by/2.5" + ) .suppliedByPersons("Tim Peierls", "Brian Goetz"), - LibraryLicense(name = "JCodings", libraryName = "joni", transitiveDependency = true, version = "1.0.55", - url = "https://github.com/jruby/jcodings") + LibraryLicense( + name = "JCodings", libraryName = "joni", transitiveDependency = true, version = "1.0.55", + url = "https://github.com/jruby/jcodings" + ) .mit("https://github.com/jruby/jcodings/blob/master/LICENSE.txt"), - LibraryLicense(name = "JDOM (JetBrains's fork)", version = "2", attachedTo = "intellij.platform.util.jdom", - url = "https://github.com/JetBrains/intellij-deps-jdom/", - license = "JDOM License", - licenseUrl = "https://github.com/JetBrains/intellij-deps-jdom/blob/master/LICENSE.txt") - .forkedFrom(sourceCodeUrl = "https://github.com/hunterhacker/jdom", - mavenRepositoryUrl = "https://repo1.maven.org/maven2", - groupId = "org.jdom", artifactId = "jdom2", - version = "2.0.6"), - LibraryLicense(libraryName = "jediterm-core", license = "LGPL 3", - url = "https://github.com/JetBrains/jediterm", - licenseUrl = "https://github.com/JetBrains/jediterm/blob/master/LICENSE-LGPLv3.txt") + LibraryLicense( + name = "JDOM (JetBrains's fork)", version = "2", attachedTo = "intellij.platform.util.jdom", + url = "https://github.com/JetBrains/intellij-deps-jdom/", + license = "JDOM License", + licenseUrl = "https://github.com/JetBrains/intellij-deps-jdom/blob/master/LICENSE.txt" + ) + .forkedFrom( + sourceCodeUrl = "https://github.com/hunterhacker/jdom", + mavenRepositoryUrl = "https://repo1.maven.org/maven2", + groupId = "org.jdom", artifactId = "jdom2", + version = "2.0.6" + ), + LibraryLicense( + libraryName = "jediterm-core", license = "LGPL 3", + url = "https://github.com/JetBrains/jediterm", + licenseUrl = "https://github.com/JetBrains/jediterm/blob/master/LICENSE-LGPLv3.txt" + ) .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(libraryName = "jediterm-ui", license = "LGPL 3", - url = "https://github.com/JetBrains/jediterm", - licenseUrl = "https://github.com/JetBrains/jediterm/blob/master/LICENSE-LGPLv3.txt") + LibraryLicense( + libraryName = "jediterm-ui", license = "LGPL 3", + url = "https://github.com/JetBrains/jediterm", + licenseUrl = "https://github.com/JetBrains/jediterm/blob/master/LICENSE-LGPLv3.txt" + ) .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "jetbrains.kotlinx.metadata.jvm", libraryName = "jetbrains.kotlinx.metadata.jvm", - url = "https://github.com/JetBrains/kotlin") + LibraryLicense( + name = "jetbrains.kotlinx.metadata.jvm", libraryName = "jetbrains.kotlinx.metadata.jvm", + url = "https://github.com/JetBrains/kotlin" + ) .apache("https://github.com/JetBrains/kotlin/blob/master/license/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "JetBrains Annotations", libraryName = "jetbrains-annotations", - url = "https://github.com/JetBrains/java-annotations") + LibraryLicense( + name = "JetBrains Annotations", libraryName = "jetbrains-annotations", + url = "https://github.com/JetBrains/java-annotations" + ) .apache("https://github.com/JetBrains/java-annotations/blob/master/LICENSE.txt"), - LibraryLicense(name = "JetBrains Annotations for Java 5", libraryName = "jetbrains-annotations-java5", - url = "https://github.com/JetBrains/java-annotations") + LibraryLicense( + name = "JetBrains Annotations for Java 5", libraryName = "jetbrains-annotations-java5", + url = "https://github.com/JetBrains/java-annotations" + ) .apache("https://github.com/JetBrains/java-annotations/blob/master/LICENSE.txt"), - LibraryLicense(name = "JetBrains Jewel IDE LaF Bridge", - url = "https://github.com/JetBrains/jewel", - libraryName = "jetbrains.jewel.ide.laf.bridge.241" + LibraryLicense( + name = "JetBrains Jewel IDE LaF Bridge", + url = "https://github.com/JetBrains/jewel", + libraryName = "jetbrains.jewel.ide.laf.bridge.241" ) .apache("https://github.com/JetBrains/jewel/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Jetbrains Jewel IDE LaF Bridge", - url = "https://github.com/JetBrains/jewel", - libraryName= "jetbrains-jewel-ide-laf-bridge" + LibraryLicense( + name = "Jetbrains Jewel IDE LaF Bridge", + url = "https://github.com/JetBrains/jewel", + libraryName = "jetbrains-jewel-ide-laf-bridge" ) .apache("https://github.com/JetBrains/jewel/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Jetbrains Jewel Int UI Standalone", - url = "https://github.com/JetBrains/jewel", - libraryName= "jetbrains-jewel-int-ui-standalone", + LibraryLicense( + name = "Jetbrains Jewel Int UI Standalone", + url = "https://github.com/JetBrains/jewel", + libraryName = "jetbrains-jewel-int-ui-standalone", ) .apache("https://github.com/JetBrains/jewel/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Jetbrains Jewel Markdown IDE LaF Bridge", - url = "https://github.com/JetBrains/jewel", - libraryName= "jetbrains-jewel-markdown-laf-bridge-styling", + LibraryLicense( + name = "Jetbrains Jewel Markdown IDE LaF Bridge", + url = "https://github.com/JetBrains/jewel", + libraryName = "jetbrains-jewel-markdown-laf-bridge-styling", ) .apache("https://github.com/JetBrains/jewel/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "JetBrains Runtime", attachedTo = "intellij.platform.ide.impl", version = "11", - license = "GNU General Public License, version 2, with the Classpath Exception", - url = "https://github.com/JetBrains/JetBrainsRuntime", - licenseUrl = "https://github.com/JetBrains/JetBrainsRuntime/blob/master/LICENSE"), - LibraryLicense(name = "JetBrains Runtime API", libraryName = "jbr-api", - url = "https://github.com/JetBrains/JetBrainsRuntime").apache(), + LibraryLicense( + name = "JetBrains Runtime", attachedTo = "intellij.platform.ide.impl", version = "11", + license = "GNU General Public License, version 2, with the Classpath Exception", + url = "https://github.com/JetBrains/JetBrainsRuntime", + licenseUrl = "https://github.com/JetBrains/JetBrainsRuntime/blob/master/LICENSE" + ), + LibraryLicense( + name = "JetBrains Runtime API", libraryName = "jbr-api", + url = "https://github.com/JetBrains/JetBrainsRuntime" + ).apache(), LibraryLicense(name = "jetCheck", libraryName = "jetCheck", url = "https://github.com/JetBrains/jetCheck") .apache("https://github.com/JetBrains/jetCheck/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "JGit (Settings Sync and SettingsRepo)", libraryName = "jetbrains.intellij.deps.eclipse.jgit", - license = "Eclipse Distribution License 1.0", - licenseUrl = "https://www.eclipse.org/org/documents/edl-v10.php", url = "https://www.eclipse.org/jgit/") + LibraryLicense( + name = "JGit (Settings Sync and SettingsRepo)", libraryName = "jetbrains.intellij.deps.eclipse.jgit", + license = "Eclipse Distribution License 1.0", + licenseUrl = "https://www.eclipse.org/org/documents/edl-v10.php", url = "https://www.eclipse.org/jgit/" + ) .suppliedByOrganizations(Suppliers.ECLIPSE), - LibraryLicense(name = "JGoodies Common", libraryName = "jgoodies-common", - url = "https://www.jgoodies.com/freeware/libraries/looks/").simplifiedBsd(), - LibraryLicense(name = "JGoodies Forms", libraryName = "jgoodies-forms", - url = "https://www.jgoodies.com/freeware/libraries/forms/").simplifiedBsd() + LibraryLicense( + name = "JGoodies Common", libraryName = "jgoodies-common", + url = "https://www.jgoodies.com/freeware/libraries/looks/" + ).simplifiedBsd(), + LibraryLicense( + name = "JGoodies Forms", libraryName = "jgoodies-forms", + url = "https://www.jgoodies.com/freeware/libraries/forms/" + ).simplifiedBsd() .suppliedByOrganizations("JGoodies Software GmbH"), LibraryLicense(name = "Jing", libraryName = "jing", url = "https://relaxng.org/jclark/jing.html") .newBsd("https://opensource.org/license/bsd-3-clause/") .suppliedByOrganizations("Thai Open Source Software Center Ltd"), - LibraryLicense(name = "JNA", libraryName = "jna", license = "LGPL 2.1", - url = "https://github.com/java-native-access/jna", - licenseUrl = "https://github.com/java-native-access/jna/blob/master/LICENSE"), + LibraryLicense( + name = "JNA", libraryName = "jna", license = "LGPL 2.1", + url = "https://github.com/java-native-access/jna", + licenseUrl = "https://github.com/java-native-access/jna/blob/master/LICENSE" + ), LibraryLicense(name = "Joni", libraryName = "joni", url = "https://github.com/jruby/joni") .mit("https://github.com/jruby/joni/blob/master/LICENSE"), LibraryLicense(name = "jps-javac-extension", libraryName = "jps-javac-extension", url = "https://github.com/JetBrains/jps-javac-extension/") @@ -625,16 +847,26 @@ object CommunityLibraryLicenses { LibraryLicense(libraryName = "jsch-agent-proxy", url = "https://github.com/ymnk/jsch-agent-proxy") .newBsd("https://github.com/ymnk/jsch-agent-proxy/blob/master/LICENSE.txt") .suppliedByPersons("Atsuhiko Yamanaka"), - LibraryLicense(name = "JSON", libraryName = "json.jar", license = "JSON License", licenseUrl = "https://www.json.org/license.html", - url = "https://www.json.org/", version = LibraryLicense.CUSTOM_REVISION), - LibraryLicense(name = "JSON in Java", libraryName = "org.json:json", license = "JSON License", - licenseUrl = "https://www.json.org/license.html", url = "https://github.com/stleary/JSON-java"), - LibraryLicense(name = "JSON Schema (schema.json)", attachedTo = "intellij.json", version = "draft-04", - url = "https://json-schema.org/draft-04/schema#").simplifiedBsd(), - LibraryLicense(name = "JSON Schema (schema06.json)", attachedTo = "intellij.json", version = "draft-06", - url = "https://json-schema.org/draft-06/schema#").simplifiedBsd(), - LibraryLicense(name = "JSON Schema (schema07.json)", attachedTo = "intellij.json", version = "draft-07", - url = "https://json-schema.org/draft-07/schema#").simplifiedBsd(), + LibraryLicense( + name = "JSON", libraryName = "json.jar", license = "JSON License", licenseUrl = "https://www.json.org/license.html", + url = "https://www.json.org/", version = LibraryLicense.CUSTOM_REVISION + ), + LibraryLicense( + name = "JSON in Java", libraryName = "org.json:json", license = "JSON License", + licenseUrl = "https://www.json.org/license.html", url = "https://github.com/stleary/JSON-java" + ), + LibraryLicense( + name = "JSON Schema (schema.json)", attachedTo = "intellij.json", version = "draft-04", + url = "https://json-schema.org/draft-04/schema#" + ).simplifiedBsd(), + LibraryLicense( + name = "JSON Schema (schema06.json)", attachedTo = "intellij.json", version = "draft-06", + url = "https://json-schema.org/draft-06/schema#" + ).simplifiedBsd(), + LibraryLicense( + name = "JSON Schema (schema07.json)", attachedTo = "intellij.json", version = "draft-07", + url = "https://json-schema.org/draft-07/schema#" + ).simplifiedBsd(), LibraryLicense(libraryName = "jsoup", url = "https://jsoup.org").mit("https://jsoup.org/license"), LibraryLicense(libraryName = "jsr305", url = "https://github.com/amaembo/jsr-305") .newBsd("https://github.com/amaembo/jsr-305/blob/master/ri/LICENSE") @@ -650,8 +882,10 @@ object CommunityLibraryLicenses { LibraryLicense(name = "JUnit5Launcher", libraryName = "JUnit5Launcher", url = "https://junit.org/junit5/").eplV2(), LibraryLicense(name = "JUnit5Vintage", libraryName = "JUnit5Vintage", url = "https://junit.org/junit5/").eplV2(), LibraryLicense(libraryName = "jzlib", url = "http://www.jcraft.com/jzlib/").newBsd("https://github.com/ymnk/jzlib/raw/master/LICENSE.txt"), - LibraryLicense(name = "Kconfig for the Zephyr Project", url = "https://github.com/trond-snekvik/vscode-kconfig", version = "1.2.0", - attachedTo = "intellij.textmate").mit("https://github.com/trond-snekvik/vscode-kconfig/blob/master/LICENSE"), + LibraryLicense( + name = "Kconfig for the Zephyr Project", url = "https://github.com/trond-snekvik/vscode-kconfig", version = "1.2.0", + attachedTo = "intellij.textmate" + ).mit("https://github.com/trond-snekvik/vscode-kconfig/blob/master/LICENSE"), LibraryLicense(name = "KInference", libraryName = "kinference.core", url = "https://packages.jetbrains.team/maven/p/ki/maven") .apache("https://github.com/JetBrains-Research/kinference/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), @@ -670,48 +904,66 @@ object CommunityLibraryLicenses { LibraryLicense(name = "Kotlin multiplatform / multi-format serialization", libraryName = "kotlinx-serialization-core", url = "https://github.com/Kotlin/kotlinx.serialization") .apache("https://github.com/Kotlin/kotlinx.serialization/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Kotlin multiplatform / multi-format serialization", - libraryName = "kotlinx-serialization-json", - url = "https://github.com/Kotlin/kotlinx.serialization") + LibraryLicense( + name = "Kotlin multiplatform / multi-format serialization", + libraryName = "kotlinx-serialization-json", + url = "https://github.com/Kotlin/kotlinx.serialization" + ) .apache("https://github.com/Kotlin/kotlinx.serialization/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Kotlin multiplatform / multi-format serialization", - libraryName = "kotlinx-serialization-protobuf", - url = "https://github.com/Kotlin/kotlinx.serialization") + LibraryLicense( + name = "Kotlin multiplatform / multi-format serialization", + libraryName = "kotlinx-serialization-protobuf", + url = "https://github.com/Kotlin/kotlinx.serialization" + ) .apache("https://github.com/Kotlin/kotlinx.serialization/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Kotlin multiplatform / multi-format serialization", - libraryName = "kotlinx-serialization-cbor", - url = "https://github.com/Kotlin/kotlinx.serialization") + LibraryLicense( + name = "Kotlin multiplatform / multi-format serialization", + libraryName = "kotlinx-serialization-cbor", + url = "https://github.com/Kotlin/kotlinx.serialization" + ) .apache("https://github.com/Kotlin/kotlinx.serialization/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Kotlin reflection library", - libraryName = "kotlin-reflect", - url = "https://github.com/JetBrains/kotlin") + LibraryLicense( + name = "Kotlin reflection library", + libraryName = "kotlin-reflect", + url = "https://github.com/JetBrains/kotlin" + ) .apache("https://github.com/JetBrains/kotlin/blob/master/license/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Kotlin Standard Library", - libraryName = "kotlin-stdlib", - url = "https://github.com/JetBrains/kotlin") + LibraryLicense( + name = "Kotlin Standard Library", + libraryName = "kotlin-stdlib", + url = "https://github.com/JetBrains/kotlin" + ) .apache("https://github.com/JetBrains/kotlin/blob/master/license/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "kotlinx-datetime-jvm", - libraryName = "kotlinx-datetime-jvm", - url = "https://github.com/Kotlin/kotlinx-datetime") + LibraryLicense( + name = "kotlinx-datetime-jvm", + libraryName = "kotlinx-datetime-jvm", + url = "https://github.com/Kotlin/kotlinx-datetime" + ) .apache("https://github.com/Kotlin/kotlinx-datetime/blob/master/LICENSE.txt") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "kotlinx.html", libraryName = "kotlinx-html-jvm", - url = "https://github.com/Kotlin/kotlinx.html") + LibraryLicense( + name = "kotlinx.html", libraryName = "kotlinx-html-jvm", + url = "https://github.com/Kotlin/kotlinx.html" + ) .apache("https://github.com/Kotlin/kotlinx.html/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Kryo5", libraryName = "Kryo5", - url = "https://github.com/EsotericSoftware/kryo") + LibraryLicense( + name = "Kryo5", libraryName = "Kryo5", + url = "https://github.com/EsotericSoftware/kryo" + ) .newBsd("https://github.com/EsotericSoftware/kryo/blob/master/LICENSE.md") .suppliedByPersons("Nathan Sweet"), - LibraryLicense(libraryName = "ktor-client-auth", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + libraryName = "ktor-client-auth", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), LibraryLicense(libraryName = "ktor-client-cio", url = "https://github.com/ktorio/ktor") @@ -720,154 +972,224 @@ object CommunityLibraryLicenses { LibraryLicense(libraryName = "ktor-client-cio-internal", url = "https://github.com/ktorio/ktor") .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(libraryName = "ktor-client-content-negotiation", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + libraryName = "ktor-client-content-negotiation", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(libraryName = "ktor-client-encoding", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + libraryName = "ktor-client-encoding", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(libraryName = "ktor-client-java", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + libraryName = "ktor-client-java", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(libraryName = "ktor-client-logging", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + libraryName = "ktor-client-logging", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(libraryName = "ktor-serialization-gson", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + libraryName = "ktor-serialization-gson", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(libraryName = "ktor-serialization-kotlinx-json", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + libraryName = "ktor-serialization-kotlinx-json", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "ktor.io TLS", libraryName = "ktor-network-tls", - url = "https://github.com/ktorio/ktor") + LibraryLicense( + name = "ktor.io TLS", libraryName = "ktor-network-tls", + url = "https://github.com/ktorio/ktor" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Ktor Client Core", - libraryName = "ktor-client-core", - url = "https://github.com/ktorio/ktor/tree/main/ktor-client/ktor-client-core") + LibraryLicense( + name = "Ktor Client Core", + libraryName = "ktor-client-core", + url = "https://github.com/ktorio/ktor/tree/main/ktor-client/ktor-client-core" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE").suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "Ktor Client OkHttp", - libraryName = "ktor-client-okhttp", - url = "https://github.com/ktorio/ktor/tree/main/ktor-client/ktor-client-okhttp") + LibraryLicense( + name = "Ktor Client OkHttp", + libraryName = "ktor-client-okhttp", + url = "https://github.com/ktorio/ktor/tree/main/ktor-client/ktor-client-okhttp" + ) .apache("https://github.com/ktorio/ktor/blob/main/LICENSE").suppliedByOrganizations(Suppliers.JETBRAINS), LibraryLicense(name = "kXML2", libraryName = "kxml2", url = "https://sourceforge.net/projects/kxml/").simplifiedBsd(), - LibraryLicense(name = "Language Tool", libraryName = "org.languagetool:languagetool-core", - url = "https://github.com/languagetool-org/languagetool", - license = "LGPL 2.1", - licenseUrl = "https://github.com/languagetool-org/languagetool/blob/master/COPYING.txt") + LibraryLicense( + name = "Language Tool", libraryName = "org.languagetool:languagetool-core", + url = "https://github.com/languagetool-org/languagetool", + license = "LGPL 2.1", + licenseUrl = "https://github.com/languagetool-org/languagetool/blob/master/COPYING.txt" + ) .suppliedByPersons("Daniel Naber", "Marcin Miłkowski"), - LibraryLicense(name = "Language Tool (English)", libraryName = "org.languagetool:language-en", - url = "https://github.com/languagetool-org/languagetool", - license = "LGPL 2.1", - licenseUrl = "https://github.com/languagetool-org/languagetool/blob/master/COPYING.txt") + LibraryLicense( + name = "Language Tool (English)", libraryName = "org.languagetool:language-en", + url = "https://github.com/languagetool-org/languagetool", + license = "LGPL 2.1", + licenseUrl = "https://github.com/languagetool-org/languagetool/blob/master/COPYING.txt" + ) .suppliedByPersons("Daniel Naber", "Marcin Miłkowski"), - LibraryLicense(name = "Log4j", libraryName = "Log4J", - url = "https://www.slf4j.org/legacy.html#log4j-over-slf4j").apache() + LibraryLicense( + name = "Log4j", libraryName = "Log4J", + url = "https://www.slf4j.org/legacy.html#log4j-over-slf4j" + ).apache() .suppliedByOrganizations("QOS.ch Sarl"), - LibraryLicense(name = "lz4-java", libraryName = "lz4-java", - url = "https://github.com/lz4/lz4-java") + LibraryLicense( + name = "lz4-java", libraryName = "lz4-java", + url = "https://github.com/lz4/lz4-java" + ) .apache("https://github.com/lz4/lz4-java/blob/master/LICENSE.txt"), - LibraryLicense(name = "MathJax", attachedTo = "intellij.python", version = "2.6.1", - url = "https://github.com/mathjax/MathJax") + LibraryLicense( + name = "MathJax", attachedTo = "intellij.python", version = "2.6.1", + url = "https://github.com/mathjax/MathJax" + ) .apache("https://github.com/mathjax/MathJax/blob/master/LICENSE"), - LibraryLicense(name = "Maven archetype catalog", libraryName = "apache.maven.archetype.catalog-no-trans:321", - url = "https://maven.apache.org/archetype/archetype-common/index.html") + LibraryLicense( + name = "Maven archetype catalog", libraryName = "apache.maven.archetype.catalog-no-trans:321", + url = "https://maven.apache.org/archetype/archetype-common/index.html" + ) .apache("https://github.com/apache/maven-archetype"), - LibraryLicense(name = "Maven archetype common", libraryName = "apache.maven.archetype.common-no-trans:3.2.1", - url = "https://maven.apache.org/archetype/archetype-common/index.html") + LibraryLicense( + name = "Maven archetype common", libraryName = "apache.maven.archetype.common-no-trans:3.2.1", + url = "https://maven.apache.org/archetype/archetype-common/index.html" + ) .apache("https://github.com/apache/maven-archetype"), - LibraryLicense(name = "Maven core", libraryName = "apache.maven.core:3.8.3", - url = "https://maven.apache.org/ref/3.8.6/maven-core/") + LibraryLicense( + name = "Maven core", libraryName = "apache.maven.core:3.8.3", + url = "https://maven.apache.org/ref/3.8.6/maven-core/" + ) .apache("https://github.com/apache/maven/blob/master/LICENSE"), - LibraryLicense(name = "Maven indexer", libraryName = "jetbrains.idea.maven.indexer.api.rt", - url = "https://maven.apache.org/maven-indexer/indexer-core/index.html") + LibraryLicense( + name = "Maven indexer", libraryName = "jetbrains.idea.maven.indexer.api.rt", + url = "https://maven.apache.org/maven-indexer/indexer-core/index.html" + ) .apache("https://github.com/apache/maven-indexer") .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "Maven Resolver Provider", - url = "https://maven.apache.org/ref/3.6.1/maven-resolver-provider/", libraryName = "maven-resolver-provider", - additionalLibraryNames = listOf("org.apache.maven.resolver:maven-resolver-connector-basic", - "org.apache.maven.resolver:maven-resolver-transport-http", - "org.apache.maven.resolver:maven-resolver-transport-file")).apache() + LibraryLicense( + name = "Maven Resolver Provider", + url = "https://maven.apache.org/ref/3.6.1/maven-resolver-provider/", libraryName = "maven-resolver-provider", + additionalLibraryNames = listOf( + "org.apache.maven.resolver:maven-resolver-connector-basic", + "org.apache.maven.resolver:maven-resolver-transport-http", + "org.apache.maven.resolver:maven-resolver-transport-file" + ) + ).apache() .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "Maven wagon provider api", libraryName = "apache.maven.wagon.provider.api:3.5.2", - url = "https://maven.apache.org/wagon/wagon-provider-api/index.html") + LibraryLicense( + name = "Maven wagon provider api", libraryName = "apache.maven.wagon.provider.api:3.5.2", + url = "https://maven.apache.org/wagon/wagon-provider-api/index.html" + ) .apache("https://github.com/apache/maven-wagon") .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "Maven Wrapper", libraryName = "io.takari.maven.wrapper", - url = "https://github.com/takari/maven-wrapper").apache(), - LibraryLicense(name = "Maven3", attachedTo = "intellij.maven.server.m3.common", - additionalLibraryNames = listOf("org.apache.maven.shared:maven-dependency-tree:1.2", - "org.apache.maven.archetype:archetype-common:2.2"), - version = "3.6.1", url = "https://maven.apache.org/").apache(), - LibraryLicense(name = "MDX for Visual Studio Code", attachedTo = "intellij.textmate", version = "1.8.7", - url = "https://github.com/mdx-js/mdx-analyzer/tree/main/packages/vscode-mdx") + LibraryLicense( + name = "Maven Wrapper", libraryName = "io.takari.maven.wrapper", + url = "https://github.com/takari/maven-wrapper" + ).apache(), + LibraryLicense( + name = "Maven3", attachedTo = "intellij.maven.server.m3.common", + additionalLibraryNames = listOf( + "org.apache.maven.shared:maven-dependency-tree:1.2", + "org.apache.maven.archetype:archetype-common:2.2" + ), + version = "3.6.1", url = "https://maven.apache.org/" + ).apache(), + LibraryLicense( + name = "MDX for Visual Studio Code", attachedTo = "intellij.textmate", version = "1.8.7", + url = "https://github.com/mdx-js/mdx-analyzer/tree/main/packages/vscode-mdx" + ) .mit("https://github.com/mdx-js/mdx-analyzer/blob/main/packages/vscode-mdx/LICENSE"), - LibraryLicense(name = "Memory File System", libraryName = "memoryfilesystem", - url = "https://github.com/marschall/memoryfilesystem") + LibraryLicense( + name = "Memory File System", libraryName = "memoryfilesystem", + url = "https://github.com/marschall/memoryfilesystem" + ) .mit("https://github.com/marschall/memoryfilesystem#faq"), - LibraryLicense(name = "mercurial_prompthooks", attachedTo = "intellij.vcs.hg", version = LibraryLicense.CUSTOM_REVISION, - license = "GPLv2 (used as hg extension called from hg executable)", - url = "https://github.com/willemv/mercurial_prompthooks", - licenseUrl = "https://github.com/willemv/mercurial_prompthooks/blob/master/LICENSE.txt"), - LibraryLicense(name = "microba", attachedTo = "intellij.libraries.microba", version = LibraryLicense.CUSTOM_REVISION, - url = "https://microba.sourceforge.net/").newBsd("https://microba.sourceforge.net/license.txt") + LibraryLicense( + name = "mercurial_prompthooks", attachedTo = "intellij.vcs.hg", version = LibraryLicense.CUSTOM_REVISION, + license = "GPLv2 (used as hg extension called from hg executable)", + url = "https://github.com/willemv/mercurial_prompthooks", + licenseUrl = "https://github.com/willemv/mercurial_prompthooks/blob/master/LICENSE.txt" + ), + LibraryLicense( + name = "microba", attachedTo = "intellij.libraries.microba", version = LibraryLicense.CUSTOM_REVISION, + url = "https://microba.sourceforge.net/" + ).newBsd("https://microba.sourceforge.net/license.txt") .suppliedByPersons("Michael Baranov"), LibraryLicense(name = "MigLayout", libraryName = "miglayout-swing", url = "https://github.com/mikaelgrev/miglayout/") .newBsd("https://github.com/mikaelgrev/miglayout/blob/master/src/site/resources/docs/license.txt") .suppliedByOrganizations("MiG InfoCom AB"), - LibraryLicense(name = "morfologik-fsa", libraryName = "org.carrot2:morfologik-fsa", - url = "https://github.com/morfologik/morfologik-stemming").simplifiedBsd() + LibraryLicense( + name = "morfologik-fsa", libraryName = "org.carrot2:morfologik-fsa", + url = "https://github.com/morfologik/morfologik-stemming" + ).simplifiedBsd() .suppliedByPersons("Dawid Weiss", "Marcin Miłkowski"), - LibraryLicense(name = "morfologik-fsa-builders", libraryName = "org.carrot2:morfologik-fsa-builders", - url = "https://github.com/morfologik/morfologik-stemming").simplifiedBsd() + LibraryLicense( + name = "morfologik-fsa-builders", libraryName = "org.carrot2:morfologik-fsa-builders", + url = "https://github.com/morfologik/morfologik-stemming" + ).simplifiedBsd() .suppliedByPersons("Dawid Weiss", "Marcin Miłkowski"), - LibraryLicense(name = "morfologik-speller", libraryName = "org.carrot2:morfologik-speller", - url = "https://github.com/morfologik/morfologik-stemming").simplifiedBsd() + LibraryLicense( + name = "morfologik-speller", libraryName = "org.carrot2:morfologik-speller", + url = "https://github.com/morfologik/morfologik-stemming" + ).simplifiedBsd() .suppliedByPersons("Dawid Weiss", "Marcin Miłkowski"), - LibraryLicense(name = "morfologik-stemming", libraryName = "org.carrot2:morfologik-stemming", - url = "https://github.com/morfologik/morfologik-stemming").simplifiedBsd() + LibraryLicense( + name = "morfologik-stemming", libraryName = "org.carrot2:morfologik-stemming", + url = "https://github.com/morfologik/morfologik-stemming" + ).simplifiedBsd() .suppliedByPersons("Dawid Weiss", "Marcin Miłkowski"), LibraryLicense(libraryName = "mvstore", url = "https://github.com/h2database/h2database") .eplV1("https://github.com/h2database/h2database/blob/master/LICENSE.txt"), - LibraryLicense(name = "NanoXML", license = "zlib/libpng", version = "2.2.3", - url = "https://central.sonatype.com/artifact/be.cyberelf.nanoxml/nanoxml/2.2.3", - licenseUrl = "https://github.com/saulhidalgoaular/nanoxml/raw/master/LICENSE.txt", - attachedTo = "intellij.platform.util.nanoxml") + LibraryLicense( + name = "NanoXML", license = "zlib/libpng", version = "2.2.3", + url = "https://central.sonatype.com/artifact/be.cyberelf.nanoxml/nanoxml/2.2.3", + licenseUrl = "https://github.com/saulhidalgoaular/nanoxml/raw/master/LICENSE.txt", + attachedTo = "intellij.platform.util.nanoxml" + ) .suppliedByPersons("Marc De Scheemaecker", "Saul Hidalgo"), - LibraryLicense(name = "nest_asyncio", attachedTo = "intellij.python.community.impl", - url = "https://github.com/erdewit/nest_asyncio", license = "BSD 2-Clause License", - licenseUrl = "https://github.com/erdewit/nest_asyncio/blob/master/LICENSE", - version = LibraryLicense.CUSTOM_REVISION), - LibraryLicense(name = "net.loomchild.segment", libraryName = "net.loomchild:segment:2.0.1", - url = "https://github.com/loomchild/segment") + LibraryLicense( + name = "nest_asyncio", attachedTo = "intellij.python.community.impl", + url = "https://github.com/erdewit/nest_asyncio", license = "BSD 2-Clause License", + licenseUrl = "https://github.com/erdewit/nest_asyncio/blob/master/LICENSE", + version = LibraryLicense.CUSTOM_REVISION + ), + LibraryLicense( + name = "net.loomchild.segment", libraryName = "net.loomchild:segment:2.0.1", + url = "https://github.com/loomchild/segment" + ) .mit("https://github.com/loomchild/segment/blob/master/LICENSE.txt") .suppliedByPersons("Jarek Lipski"), - LibraryLicense(name = "netty-buffer", libraryName = "netty-buffer", url = "https://netty.io").apache("https://github.com/netty/netty/blob/4.1/LICENSE.txt") - .suppliedByOrganizations("The Netty project"), - LibraryLicense(name = "netty-codec-http", libraryName = "netty-codec-http", url = "https://netty.io").apache("https://github.com/netty/netty/blob/4.1/LICENSE.txt") - .suppliedByOrganizations("The Netty project"), - LibraryLicense(name = "netty-codec-protobuf", libraryName = "netty-codec-protobuf", url = "https://netty.io").apache("https://github.com/netty/netty/blob/4.1/LICENSE.txt") - .suppliedByOrganizations("The Netty project"), - LibraryLicense(name = "netty-handler-proxy", libraryName = "netty-handler-proxy", url = "https://netty.io").apache("https://github.com/netty/netty/blob/4.1/LICENSE.txt") - .suppliedByOrganizations("The Netty project"), + + netty("netty"), + netty("netty-buffer"), + netty("netty-codec-http"), + netty("netty-codec-protobuf"), + netty("netty-handler-proxy"), + netty("netty-tcnative-boringssl"), + LibraryLicense(libraryName = "ngram-slp", url = "https://github.com/SLP-team/SLP-Core") .mit("https://github.com/SLP-team/SLP-Core/blob/master/LICENSE") .suppliedByOrganizations("SLP-team"), @@ -875,39 +1197,57 @@ object CommunityLibraryLicenses { .suppliedByPersons("Henri Tremblay", "Joe Walnes", "Leonardo Mesquita"), LibraryLicense(name = "OkHttp", libraryName = "okhttp", url = "https://square.github.io/okhttp/") .apache("https://square.github.io/okhttp/#license"), - LibraryLicense(libraryName = "opentelemetry", url = "https://opentelemetry.io/", - licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0") + LibraryLicense( + libraryName = "opentelemetry", url = "https://opentelemetry.io/", + licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0" + ) .suppliedByOrganizations("The OpenTelemetry Authors"), - LibraryLicense(libraryName = "opentelemetry-exporter-otlp", url = "https://opentelemetry.io/", - licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0") + LibraryLicense( + libraryName = "opentelemetry-exporter-otlp", url = "https://opentelemetry.io/", + licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0" + ) .suppliedByOrganizations("The OpenTelemetry Authors"), - LibraryLicense(libraryName = "opentelemetry-exporter-otlp-common", url = "https://opentelemetry.io/", - licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0") + LibraryLicense( + libraryName = "opentelemetry-exporter-otlp-common", url = "https://opentelemetry.io/", + licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0" + ) .suppliedByOrganizations("The OpenTelemetry Authors"), - LibraryLicense(libraryName = "opentelemetry-extension-kotlin", url = "https://opentelemetry.io/", - licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0") + LibraryLicense( + libraryName = "opentelemetry-extension-kotlin", url = "https://opentelemetry.io/", + licenseUrl = "https://github.com/open-telemetry/opentelemetry-java/blob/main/LICENSE", license = "Apache 2.0" + ) .suppliedByOrganizations("The OpenTelemetry Authors"), - LibraryLicense(libraryName = "opentelemetry-semconv", url = "https://opentelemetry.io/", - licenseUrl = "https://github.com/open-telemetry/semantic-conventions-java/blob/main/LICENSE", license = "Apache 2.0") + LibraryLicense( + libraryName = "opentelemetry-semconv", url = "https://opentelemetry.io/", + licenseUrl = "https://github.com/open-telemetry/semantic-conventions-java/blob/main/LICENSE", license = "Apache 2.0" + ) .suppliedByOrganizations("The OpenTelemetry Authors"), LibraryLicense(libraryName = "opentest4j", url = "https://github.com/ota4j-team/opentest4j") .apache("https://github.com/ota4j-team/opentest4j/blob/master/LICENSE"), - LibraryLicense(name = "OverlayScrollbars", attachedTo = "intellij.idea.community.main", - url = "https://kingsora.github.io/OverlayScrollbars", version = "2.1.1") + LibraryLicense( + name = "OverlayScrollbars", attachedTo = "intellij.idea.community.main", + url = "https://kingsora.github.io/OverlayScrollbars", version = "2.1.1" + ) .mit("https://github.com/KingSora/OverlayScrollbars/blob/master/LICENSE"), LibraryLicense(name = "Package Search API-Client", libraryName = "package-search-api-client", url = "https://github.com/JetBrains/package-search-api-models") .apache("https://github.com/JetBrains/package-search-api-models/blob/master/LICENSE") .suppliedByOrganizations("JetBrains Team"), - LibraryLicense(name = "pip", attachedTo = "intellij.python", version = "20.3.4", - url = "https://pip.pypa.io/") + LibraryLicense( + name = "pip", attachedTo = "intellij.python", version = "20.3.4", + url = "https://pip.pypa.io/" + ) .mit("https://github.com/pypa/pip/blob/main/LICENSE.txt"), - LibraryLicense(name = "plexus-archiver", libraryName = "plexus-archiver", - url = "https://github.com/codehaus-plexus/plexus-archiver") + LibraryLicense( + name = "plexus-archiver", libraryName = "plexus-archiver", + url = "https://github.com/codehaus-plexus/plexus-archiver" + ) .apache("https://github.com/codehaus-plexus/plexus-archiver/blob/master/LICENSE") .suppliedByOrganizations("The Codehaus Foundation, Inc."), - LibraryLicense(name = "Plexus Utils", libraryName = "plexus-utils", - url = "https://github.com/codehaus-plexus/plexus-utils") + LibraryLicense( + name = "Plexus Utils", libraryName = "plexus-utils", + url = "https://github.com/codehaus-plexus/plexus-utils" + ) .apache("https://github.com/codehaus-plexus/plexus-utils/blob/master/LICENSE.txt") .suppliedByOrganizations("The Codehaus Foundation, Inc."), LibraryLicense(name = "PLY", attachedTo = "intellij.python", version = "3.7", url = "https://www.dabeaz.com/ply/").newBsd(), @@ -915,85 +1255,131 @@ object CommunityLibraryLicenses { LibraryLicense(libraryName = "pngencoder", url = "https://github.com/pngencoder/pngencoder") .mit("https://github.com/pngencoder/pngencoder/blob/develop/LICENSE"), - LibraryLicense(name = "pockets", attachedTo = "intellij.python", version = "0.9.1", - url = "https://pockets.readthedocs.io/") + LibraryLicense( + name = "pockets", attachedTo = "intellij.python", version = "0.9.1", + url = "https://pockets.readthedocs.io/" + ) .newBsd("https://github.com/RobRuana/pockets/blob/master/LICENSE"), LibraryLicense(name = "Protocol Buffers", libraryName = "protobuf", url = "https://developers.google.com/protocol-buffers") .newBsd("https://github.com/google/protobuf/blob/master/LICENSE"), LibraryLicense(name = "Proxy Vole", libraryName = "proxy-vole", url = "https://github.com/akuhtz/proxy-vole") .apache("https://github.com/akuhtz/proxy-vole/blob/master/LICENSE.md"), - LibraryLicense(name = "pty4j", libraryName = "pty4j", - url = "https://github.com/JetBrains/pty4j") + LibraryLicense( + name = "pty4j", libraryName = "pty4j", + url = "https://github.com/JetBrains/pty4j" + ) .eplV1("https://github.com/JetBrains/pty4j/blob/master/LICENSE"), - LibraryLicense(name = "PureJavaComm", libraryName = "pty4j", transitiveDependency = true, version = "0.0.11.1", - url = "https://github.com/nyholku/purejavacomm") + LibraryLicense( + name = "PureJavaComm", libraryName = "pty4j", transitiveDependency = true, version = "0.0.11.1", + url = "https://github.com/nyholku/purejavacomm" + ) .newBsd("https://github.com/nyholku/purejavacomm/blob/master/LICENSE.txt"), - LibraryLicense(name = "pycodestyle", attachedTo = "intellij.python", version = "2.8.0", - url = "https://pycodestyle.pycqa.org/") + LibraryLicense( + name = "pycodestyle", attachedTo = "intellij.python", version = "2.8.0", + url = "https://pycodestyle.pycqa.org/" + ) .mit("https://github.com/PyCQA/pycodestyle/blob/main/LICENSE"), - LibraryLicense(name = "pyparsing", attachedTo = "intellij.python", version = "1.5.6", - url = "https://github.com/pyparsing/pyparsing/") + LibraryLicense( + name = "pyparsing", attachedTo = "intellij.python", version = "1.5.6", + url = "https://github.com/pyparsing/pyparsing/" + ) .mit("https://github.com/pyparsing/pyparsing/blob/master/LICENSE"), - LibraryLicense(name = "qdox-java-parser", libraryName = "qdox-java-parser", - url = "https://github.com/paul-hammant/qdox") + LibraryLicense( + name = "qdox-java-parser", libraryName = "qdox-java-parser", + url = "https://github.com/paul-hammant/qdox" + ) .apache("https://github.com/paul-hammant/qdox/blob/master/LICENSE.txt"), LibraryLicense(name = "R8 DEX shrinker", libraryName = "jb-r8", url = "https://r8.googlesource.com/r8") .newBsd("https://r8.googlesource.com/r8/+/refs/heads/main/LICENSE") .suppliedByOrganizations(Suppliers.GOOGLE), - LibraryLicense(name = "rd core", libraryName = "rd-core", - url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-core") + LibraryLicense( + name = "rd core", libraryName = "rd-core", + url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-core" + ) .apache("https://github.com/JetBrains/rd/blob/master/LICENSE"), - LibraryLicense(name = "rd framework", libraryName = "rd-framework", - url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-framework") + LibraryLicense( + name = "rd framework", libraryName = "rd-framework", + url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-framework" + ) .apache("https://github.com/JetBrains/rd/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "rd generator", libraryName = "rd-gen", - url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-gen") + LibraryLicense( + name = "rd generator", libraryName = "rd-gen", + url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-gen" + ) .apache("https://github.com/JetBrains/rd/blob/master/LICENSE") .suppliedByOrganizations(Suppliers.JETBRAINS), - LibraryLicense(name = "rd Swing integration", libraryName = "rd-swing", - url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-swing") + LibraryLicense( + name = "rd Swing integration", libraryName = "rd-swing", + url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-swing" + ) .apache("https://github.com/JetBrains/rd/blob/master/LICENSE"), - LibraryLicense(name = "rd text buffers", libraryName = "rd-text", - url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-text") + LibraryLicense( + name = "rd text buffers", libraryName = "rd-text", + url = "https://github.com/JetBrains/rd/tree/master/rd-kt/rd-text" + ) .apache("https://github.com/JetBrains/rd/blob/master/LICENSE"), - LibraryLicense(name = "Reactive Streams", libraryName = "reactivestreams.reactive.streams", - url = "https://github.com/reactive-streams/reactive-streams-jvm") + LibraryLicense( + name = "Reactive Streams", libraryName = "reactivestreams.reactive.streams", + url = "https://github.com/reactive-streams/reactive-streams-jvm" + ) .mit("https://github.com/reactive-streams/reactive-streams-jvm/blob/master/LICENSE"), - LibraryLicense(name = "Relax NG Object Model", libraryName = "rngom-20051226-patched.jar", - url = "https://github.com/kohsuke/rngom", version = LibraryLicense.CUSTOM_REVISION) + LibraryLicense( + name = "Relax NG Object Model", libraryName = "rngom-20051226-patched.jar", + url = "https://github.com/kohsuke/rngom", version = LibraryLicense.CUSTOM_REVISION + ) .mit("https://github.com/kohsuke/rngom/blob/master/licenceheader.txt"), - LibraryLicense(name = "Rhino JavaScript Engine", libraryName = "rhino", license = "MPL 1.1", - url = "https://github.com/mozilla/rhino", licenseUrl = "https://www.mozilla.org/MPL/MPL-1.1.html"), - LibraryLicense(name = "Roboto", attachedTo = "intellij.platform.resources", version = "1.100141", - url = "https://github.com/googlefonts/roboto") + LibraryLicense( + name = "Rhino JavaScript Engine", libraryName = "rhino", license = "MPL 1.1", + url = "https://github.com/mozilla/rhino", licenseUrl = "https://www.mozilla.org/MPL/MPL-1.1.html" + ), + LibraryLicense( + name = "Roboto", attachedTo = "intellij.platform.resources", version = "1.100141", + url = "https://github.com/googlefonts/roboto" + ) .apache("https://github.com/google/roboto/blob/master/LICENSE"), - LibraryLicense(name = "roman", attachedTo = "intellij.python", version = "1.4.0", - url = "https://docutils.sourceforge.io/docutils/utils/roman.py", - license = "Python 2.1.1 license", - licenseUrl = "https://www.python.org/download/releases/2.1.1/license/"), - LibraryLicense(libraryName = "sa-jdwp", license = "GPL 2.0 + Classpath", url = "https://github.com/JetBrains/jdk-sa-jdwp", - licenseUrl = "https://github.com/JetBrains/jdk-sa-jdwp/raw/master/LICENSE.txt"), - LibraryLicense(libraryName = "Saxon-6.5.5", version = "6.5.5", license = "Mozilla Public License", - url = "https://saxon.sourceforge.net/", - licenseUrl = "https://www.mozilla.org/MPL/"), - LibraryLicense(libraryName = "Saxon-9HE", version = "9", license = "Mozilla Public License", url = "https://saxon.sourceforge.net/", - licenseUrl = "https://www.mozilla.org/MPL/"), - LibraryLicense(name = "setuptools", attachedTo = "intellij.python", version = "44.1.1", - url = "https://setuptools.pypa.io/") + LibraryLicense( + name = "roman", attachedTo = "intellij.python", version = "1.4.0", + url = "https://docutils.sourceforge.io/docutils/utils/roman.py", + license = "Python 2.1.1 license", + licenseUrl = "https://www.python.org/download/releases/2.1.1/license/" + ), + LibraryLicense( + libraryName = "sa-jdwp", license = "GPL 2.0 + Classpath", url = "https://github.com/JetBrains/jdk-sa-jdwp", + licenseUrl = "https://github.com/JetBrains/jdk-sa-jdwp/raw/master/LICENSE.txt" + ), + LibraryLicense( + libraryName = "Saxon-6.5.5", version = "6.5.5", license = "Mozilla Public License", + url = "https://saxon.sourceforge.net/", + licenseUrl = "https://www.mozilla.org/MPL/" + ), + LibraryLicense( + libraryName = "Saxon-9HE", version = "9", license = "Mozilla Public License", url = "https://saxon.sourceforge.net/", + licenseUrl = "https://www.mozilla.org/MPL/" + ), + LibraryLicense( + name = "setuptools", attachedTo = "intellij.python", version = "44.1.1", + url = "https://setuptools.pypa.io/" + ) .mit("https://github.com/pypa/setuptools/blob/main/LICENSE"), - LibraryLicense(name = "six.py", attachedTo = "intellij.python", version = "1.9.0", - url = "https://six.readthedocs.io/", - licenseUrl = "https://github.com/benjaminp/six/blob/master/LICENSE") + LibraryLicense( + name = "six.py", attachedTo = "intellij.python", version = "1.9.0", + url = "https://six.readthedocs.io/", + licenseUrl = "https://github.com/benjaminp/six/blob/master/LICENSE" + ) .mit("https://github.com/benjaminp/six/blob/master/LICENSE"), - LibraryLicense(name = "Skiko", libraryName = "jetbrains.skiko.awt.compose", - url = "https://github.com/JetBrains/skiko/") + LibraryLicense( + name = "Skiko", libraryName = "jetbrains.skiko.awt.compose", + url = "https://github.com/JetBrains/skiko/" + ) .apache("https://github.com/JetBrains/skiko/blob/master/LICENSE"), - LibraryLicense(name = "Skiko Runtime", libraryName = "jetbrains.skiko.awt.runtime.all", - url = "https://github.com/JetBrains/skiko/") + LibraryLicense( + name = "Skiko Runtime", libraryName = "jetbrains.skiko.awt.runtime.all", + url = "https://github.com/JetBrains/skiko/" + ) .apache("https://github.com/JetBrains/skiko/blob/master/LICENSE"), LibraryLicense(libraryName = "slf4j-api", url = "https://slf4j.org/") .mit("https://www.slf4j.org/license.html") @@ -1001,23 +1387,35 @@ object CommunityLibraryLicenses { LibraryLicense(libraryName = "slf4j-jdk14", url = "https://slf4j.org/") .mit("https://www.slf4j.org/license.html") .suppliedByOrganizations("QOS.ch Sarl"), - LibraryLicense(name = "SnakeYAML", libraryName = "snakeyaml", - url = "https://bitbucket.org/snakeyaml/snakeyaml/") + LibraryLicense( + name = "SnakeYAML", libraryName = "snakeyaml", + url = "https://bitbucket.org/snakeyaml/snakeyaml/" + ) .apache("https://bitbucket.org/snakeyaml/snakeyaml/src/master/LICENSE.txt") .suppliedByPersons("Andrey Somov", "Alexander Maslov", "Jordan Angold"), - LibraryLicense(name = "snakeyaml-engine", libraryName = "snakeyaml-engine", - url = "https://bitbucket.org/snakeyaml/snakeyaml-engine/") + LibraryLicense( + name = "snakeyaml-engine", libraryName = "snakeyaml-engine", + url = "https://bitbucket.org/snakeyaml/snakeyaml-engine/" + ) .apache("https://bitbucket.org/snakeyaml/snakeyaml-engine/src/master/LICENSE.txt") .suppliedByPersons("Andrey Somov", "Alexander Maslov"), - LibraryLicense(name = "Sonatype Nexus: Indexer", attachedTo = "intellij.maven.server.m3.common", version = "3.0.4", - additionalLibraryNames = listOf("org.sonatype.nexus:nexus-indexer:3.0.4", - "org.sonatype.nexus:nexus-indexer-artifact:1.0.1"), - url = "https://maven.apache.org/maven-indexer/").eplV1(), - LibraryLicense(name = "SourceCodePro", attachedTo = "intellij.platform.resources", version = "2.010", license = "OFL", - url = "https://github.com/adobe-fonts/source-code-pro", - licenseUrl = "https://github.com/adobe-fonts/source-code-pro/blob/master/LICENSE.md"), - LibraryLicense(name = "sphinxcontrib-napoleon", attachedTo = "intellij.python", version = "0.7", - url = "https://sphinxcontrib-napoleon.readthedocs.io/") + LibraryLicense( + name = "Sonatype Nexus: Indexer", attachedTo = "intellij.maven.server.m3.common", version = "3.0.4", + additionalLibraryNames = listOf( + "org.sonatype.nexus:nexus-indexer:3.0.4", + "org.sonatype.nexus:nexus-indexer-artifact:1.0.1" + ), + url = "https://maven.apache.org/maven-indexer/" + ).eplV1(), + LibraryLicense( + name = "SourceCodePro", attachedTo = "intellij.platform.resources", version = "2.010", license = "OFL", + url = "https://github.com/adobe-fonts/source-code-pro", + licenseUrl = "https://github.com/adobe-fonts/source-code-pro/blob/master/LICENSE.md" + ), + LibraryLicense( + name = "sphinxcontrib-napoleon", attachedTo = "intellij.python", version = "0.7", + url = "https://sphinxcontrib-napoleon.readthedocs.io/" + ) .simplifiedBsd("https://github.com/sphinx-contrib/napoleon/blob/master/LICENSE"), LibraryLicense(name = "Squareup Okio", libraryName = "squareup.okio.jvm", url = "https://github.com/square/okio") .apache("https://github.com/square/okio/blob/master/LICENSE.txt") @@ -1025,46 +1423,66 @@ object CommunityLibraryLicenses { LibraryLicense(name = "Squareup Wire", libraryName = "squareup.wire.runtime.jvm", url = "https://github.com/square/wire") .apache("https://github.com/square/wire/blob/master/LICENSE.txt") .suppliedByOrganizations("Square, Inc."), - LibraryLicense(name = "ssh-nio-fs", libraryName = "ssh-nio-fs", - url = "https://github.com/JetBrains/intellij-deps-ssh-nio-fs") + LibraryLicense( + name = "ssh-nio-fs", libraryName = "ssh-nio-fs", + url = "https://github.com/JetBrains/intellij-deps-ssh-nio-fs" + ) .mit("https://github.com/JetBrains/intellij-deps-ssh-nio-fs/blob/master/LICENSE") - .forkedFrom(sourceCodeUrl = "https://github.com/lucastheisen/jsch-nio", - mavenRepositoryUrl = "https://repo1.maven.org/maven2", - groupId = "com.pastdev", artifactId = "jsch-nio", - version = "1.0.14", - authors = "Lucas Theisen"), - LibraryLicense(name = "StreamEx", libraryName = "StreamEx", - url = "https://github.com/amaembo/streamex") + .forkedFrom( + sourceCodeUrl = "https://github.com/lucastheisen/jsch-nio", + mavenRepositoryUrl = "https://repo1.maven.org/maven2", + groupId = "com.pastdev", artifactId = "jsch-nio", + version = "1.0.14", + authors = "Lucas Theisen" + ), + LibraryLicense( + name = "StreamEx", libraryName = "StreamEx", + url = "https://github.com/amaembo/streamex" + ) .apache("https://github.com/amaembo/streamex/blob/master/LICENSE"), - LibraryLicense(name = "swingx", libraryName = "swingx", license = "LGPL 2.1", - url = "https://central.sonatype.com/artifact/org.swinglabs/swingx-core/1.6.2-2", - licenseUrl = "https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html") + LibraryLicense( + name = "swingx", libraryName = "swingx", license = "LGPL 2.1", + url = "https://central.sonatype.com/artifact/org.swinglabs/swingx-core/1.6.2-2", + licenseUrl = "https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html" + ) .suppliedByOrganizations("Sun Microsystems, Inc."), LibraryLicense(libraryName = "TestNG", url = "https://testng.org/") .apache("https://github.com/cbeust/testng/blob/master/LICENSE.txt"), - LibraryLicense(name = "The Erlang LS extension for VSCode", attachedTo = "intellij.textmate", version = "0.0.43", - url = "https://github.com/mblode/vscode-twig-language-2") + LibraryLicense( + name = "The Erlang LS extension for VSCode", attachedTo = "intellij.textmate", version = "0.0.43", + url = "https://github.com/mblode/vscode-twig-language-2" + ) .apache("https://github.com/erlang-ls/vscode/blob/main/LICENSE.md"), LibraryLicense(name = "Thrift", libraryName = "libthrift", url = "https://thrift.apache.org/") .apache("https://github.com/apache/thrift/blob/master/LICENSE"), - LibraryLicense(name = "thriftpy2", attachedTo = "intellij.python", version = "0.4.13", - url = "https://github.com/Thriftpy/thriftpy2/") + LibraryLicense( + name = "thriftpy2", attachedTo = "intellij.python", version = "0.4.13", + url = "https://github.com/Thriftpy/thriftpy2/" + ) .mit("https://github.com/Thriftpy/thriftpy2/blob/master/LICENSE"), // for traceprocessor-proto module library in intellij.android.profilersAndroid - LibraryLicense(name = "Trang", libraryName = "trang-core.jar", - url = "https://relaxng.org/jclark/trang.html", - version = LibraryLicense.CUSTOM_REVISION) + LibraryLicense( + name = "Trang", libraryName = "trang-core.jar", + url = "https://relaxng.org/jclark/trang.html", + version = LibraryLicense.CUSTOM_REVISION + ) .newBsd("https://opensource.org/license/bsd-3-clause/"), - LibraryLicense(name = "Trove4j (JetBrains's fork)", libraryName = "trove", license = "LGPL", - url = "https://github.com/JetBrains/intellij-deps-trove4j", - licenseUrl = "https://github.com/JetBrains/intellij-deps-trove4j/blob/master/LICENSE.txt") + LibraryLicense( + name = "Trove4j (JetBrains's fork)", libraryName = "trove", license = "LGPL", + url = "https://github.com/JetBrains/intellij-deps-trove4j", + licenseUrl = "https://github.com/JetBrains/intellij-deps-trove4j/blob/master/LICENSE.txt" + ) .forkedFrom(sourceCodeUrl = "https://sourceforge.net/p/trove4j/cvs", groupId = "net.sf.trove4j", artifactId = "trove4j"), - LibraryLicense(name = "Typeshed", attachedTo = "intellij.python", version = LibraryLicense.CUSTOM_REVISION, - url = "https://github.com/python/typeshed") + LibraryLicense( + name = "Typeshed", attachedTo = "intellij.python", version = LibraryLicense.CUSTOM_REVISION, + url = "https://github.com/python/typeshed" + ) .apache("https://github.com/python/typeshed/blob/master/LICENSE"), - LibraryLicense(name = "unit-api", libraryName = "javax.measure:unit-api:1.0", - url = "https://github.com/unitsofmeasurement/unit-api") + LibraryLicense( + name = "unit-api", libraryName = "javax.measure:unit-api:1.0", + url = "https://github.com/unitsofmeasurement/unit-api" + ) .newBsd("https://github.com/unitsofmeasurement/unit-api/blob/master/LICENSE") .suppliedByPersons( "Jean-Marie Dautelle", "Werner Keil", "Otávio Gonçalves de Santana", @@ -1073,28 +1491,40 @@ object CommunityLibraryLicenses { "Karen Legrand", "Rajmahendra Hegde", "Mohamed Mahmoud Taman", "Werner Keil", "Mohammed Al-Moayed", "Werner Keil" ), - LibraryLicense(name = "uom-lib-common", libraryName = "tech.uom.lib:uom-lib-common:1.1", - url = "https://github.com/unitsofmeasurement/uom-lib") + LibraryLicense( + name = "uom-lib-common", libraryName = "tech.uom.lib:uom-lib-common:1.1", + url = "https://github.com/unitsofmeasurement/uom-lib" + ) .newBsd("https://github.com/unitsofmeasurement/uom-lib/blob/master/LICENSE") .suppliedByPersons("Jean-Marie Dautelle", "Werner Keil"), LibraryLicense(libraryName = "Velocity", url = "https://velocity.apache.org/") .suppliedByOrganizations(Suppliers.APACHE) .apache("https://gitbox.apache.org/repos/asf?p=velocity-engine.git;a=blob_plain;f=LICENSE;hb=HEAD"), - LibraryLicense(name = "Vim Script language support for Atom", attachedTo = "intellij.textmate", version = "1.2.1", - url = "https://github.com/AlexPl292/language-viml") + LibraryLicense( + name = "Vim Script language support for Atom", attachedTo = "intellij.textmate", version = "1.2.1", + url = "https://github.com/AlexPl292/language-viml" + ) .mit("https://github.com/AlexPl292/language-viml/blob/master/LICENSE.txt"), - LibraryLicense(name = "virtualenv", attachedTo = "intellij.python", version = "20.13.0", - url = "https://virtualenv.pypa.io/") + LibraryLicense( + name = "virtualenv", attachedTo = "intellij.python", version = "20.13.0", + url = "https://virtualenv.pypa.io/" + ) .mit("https://github.com/pypa/virtualenv/blob/main/LICENSE"), - LibraryLicense(name = "Visual Studio Code", attachedTo = "intellij.textmate", version = "1.90.0", - url = "https://github.com/Microsoft/vscode/") + LibraryLicense( + name = "Visual Studio Code", attachedTo = "intellij.textmate", version = "1.90.0", + url = "https://github.com/Microsoft/vscode/" + ) .mit("https://github.com/Microsoft/vscode-react-native/blob/master/LICENSE.txt"), - LibraryLicense(name = "VS Code Twig Language 2", attachedTo = "intellij.textmate", version = "0.9.4", - url = "https://github.com/mblode/vscode-twig-language-2") + LibraryLicense( + name = "VS Code Twig Language 2", attachedTo = "intellij.textmate", version = "0.9.4", + url = "https://github.com/mblode/vscode-twig-language-2" + ) .mit("https://github.com/mblode/vscode-twig-language-2/blob/master/LICENSE.md"), - LibraryLicense(name = "weberknecht", libraryName = "weberknecht-0.1.5.jar", version = "0.1.5", - // originally https://github.com/pelotoncycle/weberknecht - url = "https://github.com/pusher-community/titanium_pusher_android/blob/master/src/de/roderick/weberknecht/") + LibraryLicense( + name = "weberknecht", libraryName = "weberknecht-0.1.5.jar", version = "0.1.5", + // originally https://github.com/pelotoncycle/weberknecht + url = "https://github.com/pusher-community/titanium_pusher_android/blob/master/src/de/roderick/weberknecht/" + ) .apache("https://github.com/pusher-community/titanium_pusher_android/blob/master/src/de/roderick/weberknecht/WebSocket.java"), LibraryLicense(libraryName = "winp", url = "https://github.com/jenkinsci/winp") .mit("https://github.com/jenkinsci/winp/blob/master/LICENSE.txt") @@ -1117,18 +1547,26 @@ object CommunityLibraryLicenses { .apache("https://github.com/xerial/sqlite-jdbc/blob/master/LICENSE") .suppliedByOrganizations("Xerial Project"), - LibraryLicense(name = "xml-apis-ext", libraryName = "xml-apis-ext", - url = "https://xerces.apache.org/xml-commons/components/external/").apache() + LibraryLicense( + name = "xml-apis-ext", libraryName = "xml-apis-ext", + url = "https://xerces.apache.org/xml-commons/components/external/" + ).apache() .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "xml-resolver", libraryName = "xml-resolver", - url = "https://xerces.apache.org/xml-commons/components/resolver/").apache() + LibraryLicense( + name = "xml-resolver", libraryName = "xml-resolver", + url = "https://xerces.apache.org/xml-commons/components/resolver/" + ).apache() .suppliedByOrganizations("The Apache Software Foundation"), - LibraryLicense(name = "XMLBeans", libraryName = "XmlBeans", - url = "https://xmlbeans.apache.org/") + LibraryLicense( + name = "XMLBeans", libraryName = "XmlBeans", + url = "https://xmlbeans.apache.org/" + ) .apache("https://svn.jetbrains.org/idea/Trunk/bundled/WebServices/resources/lib/xmlbeans-2.3.0/xmlbeans.LICENSE") .suppliedByPersons("Cezar Andrei", "Radu Preotiuc", "Radu Preotiuc", "Wing Yew Poon", "Jacob Danner", "POI Team"), - LibraryLicense(name = "XmlRPC", libraryName = "XmlRPC", - url = "https://ws.apache.org/xmlrpc/xmlrpc2/") + LibraryLicense( + name = "XmlRPC", libraryName = "XmlRPC", + url = "https://ws.apache.org/xmlrpc/xmlrpc2/" + ) .apache("https://ws.apache.org/xmlrpc/xmlrpc2/license.html") .suppliedByPersons( "Daniel Rall", "Jon Scott Stevens", "John Wilson", @@ -1136,22 +1574,32 @@ object CommunityLibraryLicenses { "Andrew Evers", "Henri Gomez", "Ryan Hoegg", "Leonard Richarson", "Hannes Wallnoefer" ), - LibraryLicense(name = "XSLT Debugger RMI Stubs", - libraryName = "RMI Stubs", - url = "https://confluence.jetbrains.com/display/CONTEST/XSLT-Debugger", - version = LibraryLicense.CUSTOM_REVISION).apache(), - LibraryLicense(name = "XStream", libraryName = "XStream", - url = "https://x-stream.github.io/") + LibraryLicense( + name = "XSLT Debugger RMI Stubs", + libraryName = "RMI Stubs", + url = "https://confluence.jetbrains.com/display/CONTEST/XSLT-Debugger", + version = LibraryLicense.CUSTOM_REVISION + ).apache(), + LibraryLicense( + name = "XStream", libraryName = "XStream", + url = "https://x-stream.github.io/" + ) .newBsd("https://x-stream.github.io/license.html") .suppliedByOrganizations("XStream Committers"), - LibraryLicense(name = "XZ for Java", libraryName = "xz", license = "Public Domain", - url = "https://tukaani.org/xz/java.html", - licenseUrl = "https://git.tukaani.org/?p=xz-java.git;a=blob;f=COPYING;h=8dd17645c4610c3d5eed9bcdd2699ecfac00406b;hb=refs/heads/master"), - LibraryLicense(name = "zip-signer", libraryName = "zip-signer", - url = "https://github.com/JetBrains/marketplace-zip-signer") + LibraryLicense( + name = "XZ for Java", libraryName = "xz", license = "Public Domain", + url = "https://tukaani.org/xz/java.html", + licenseUrl = "https://git.tukaani.org/?p=xz-java.git;a=blob;f=COPYING;h=8dd17645c4610c3d5eed9bcdd2699ecfac00406b;hb=refs/heads/master" + ), + LibraryLicense( + name = "zip-signer", libraryName = "zip-signer", + url = "https://github.com/JetBrains/marketplace-zip-signer" + ) .apache("https://github.com/JetBrains/marketplace-zip-signer/blob/master/LICENSE"), - LibraryLicense(name = "zstd-jni", libraryName = "zstd-jni", - url = "https://github.com/luben/zstd-jni") + LibraryLicense( + name = "zstd-jni", libraryName = "zstd-jni", + url = "https://github.com/luben/zstd-jni" + ) .simplifiedBsd("https://github.com/luben/zstd-jni/blob/master/LICENSE"), jetbrainsLibrary("change-reminder-prediction-model"), jetbrainsLibrary("cloud-config-client"), @@ -1248,21 +1696,27 @@ object CommunityLibraryLicenses { jetbrainsLibrary("tips-pycharm-community"), jetbrainsLibrary("workspace-model-codegen"), ) - - private fun ffmpegLibraryLicense(libraryName: String): LibraryLicense { - return LibraryLicense( - name = libraryName, - libraryName = libraryName, - url = "https://android.googlesource.com/platform/prebuilts/tools/+/refs/tags/studio-2022.3.1-beta2/common/m2/repository/org/bytedeco", - license = "LGPL v2.1+", - licenseUrl = "https://android.googlesource.com/platform/prebuilts/tools/+/refs/tags/studio-2022.3.1-beta2/common/m2/repository/org/bytedeco/ffmpeg-LICENSE.md" - ).suppliedByOrganizations(Suppliers.GOOGLE) - } - - private fun androidDependency(name: String, libraryName: String? = name, version: String? = null) = - LibraryLicense(name = name, libraryName = libraryName, version = version, - url = "https://source.android.com/") - .apache("https://source.android.com/setup/start/licenses") - .copyrightText("Copyright (C) The Android Open Source Project") - .suppliedByOrganizations(Suppliers.GOOGLE) +} + +private fun ffmpegLibraryLicense(libraryName: String): LibraryLicense { + return LibraryLicense( + name = libraryName, + libraryName = libraryName, + url = "https://android.googlesource.com/platform/prebuilts/tools/+/refs/tags/studio-2022.3.1-beta2/common/m2/repository/org/bytedeco", + license = "LGPL v2.1+", + licenseUrl = "https://android.googlesource.com/platform/prebuilts/tools/+/refs/tags/studio-2022.3.1-beta2/common/m2/repository/org/bytedeco/ffmpeg-LICENSE.md" + ).suppliedByOrganizations(Suppliers.GOOGLE) +} + +private fun androidDependency(name: String, libraryName: String? = name, version: String? = null): LibraryLicense { + return LibraryLicense(name = name, libraryName = libraryName, version = version, url = "https://source.android.com/") + .apache("https://source.android.com/setup/start/licenses") + .copyrightText("Copyright (C) The Android Open Source Project") + .suppliedByOrganizations(Suppliers.GOOGLE) +} + +private fun netty(libraryName: String): LibraryLicense { + return LibraryLicense(name = libraryName, libraryName = libraryName, url = "https://netty.io") + .apache("https://github.com/netty/netty/blob/4.1/LICENSE.txt") + .suppliedByOrganizations("The Netty project") } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/LibraryLicense.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/LibraryLicense.kt index 0b21cb0d0fa0..712ad9d33e32 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/LibraryLicense.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/LibraryLicense.kt @@ -1,4 +1,4 @@ -// Copyright 2000-2022 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. @file:Suppress("ReplaceGetOrSet") package org.jetbrains.intellij.build @@ -110,8 +110,7 @@ data class LibraryLicense( private const val APACHE_LICENSE_URL = "https://www.apache.org/licenses/LICENSE-2.0" private val PREDEFINED_LICENSE_URLS = mapOf("Apache 2.0" to APACHE_LICENSE_URL) - @JvmStatic - val JETBRAINS_OWN = "JetBrains" + const val JETBRAINS_OWN = "JetBrains" /** * Denotes version of a library built from custom revision @@ -123,7 +122,6 @@ data class LibraryLicense( * so there is no way to give a link to their sites. * For other libraries please fill all necessary fields of [LibraryLicense] instead of using this method. */ - @JvmStatic fun jetbrainsLibrary(libraryName: String): LibraryLicense { return LibraryLicense( libraryName = libraryName, diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/concurrency.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/concurrency.kt new file mode 100644 index 000000000000..e43fe819c306 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/concurrency.kt @@ -0,0 +1,52 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build + +import kotlinx.coroutines.channels.produce +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import java.util.concurrent.CancellationException + +internal suspend fun Collection.forEachConcurrent(concurrency: Int = Runtime.getRuntime().availableProcessors(), action: suspend (T) -> Unit) { + coroutineScope { + val itemChannel = produce { + for (item in this@forEachConcurrent) { + send(item) + } + } + + repeat(concurrency) { + launch { + for (item in itemChannel) { + try { + action(item) + } + catch (e: CancellationException) { + if (coroutineContext.isActive) { + // well, we are not canceled, only child + throw IllegalStateException("Unexpected cancellation - action is cancelled itself", e) + } + } + } + } + } + } + + val semaphore = Semaphore(concurrency) + coroutineScope { + for (item in this@forEachConcurrent) { + launch { + semaphore.withPermit { + try { + action(item) + } + catch (e: CancellationException) { + throw e + } + } + } + } + } +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnection.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnection.kt new file mode 100644 index 000000000000..b39fb46f1f51 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnection.kt @@ -0,0 +1,137 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("SSBasedInspection", "ReplacePutWithAssignment", "ReplaceGetOrSet") + +package org.jetbrains.intellij.build.http2Client + +import io.netty.buffer.ByteBufUtil +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http2.Http2Headers +import io.netty.handler.codec.http2.Http2HeadersFrame +import io.netty.handler.codec.http2.Http2StreamChannel +import io.netty.handler.codec.http2.ReadOnlyHttp2Headers +import io.netty.util.AsciiString +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.withContext +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.serializer + +// https://cabulous.medium.com/http-2-and-how-it-works-9f645458e4b2 +internal class Http2ClientConnection internal constructor( + private val scheme: AsciiString, + private val authority: AsciiString, + private val commonHeaders: Array, + @JvmField internal val connection: Http2ConnectionProvider, +) { + suspend inline fun use(block: (Http2ClientConnection) -> T): T { + try { + return block(this) + } + finally { + withContext(NonCancellable) { + close() + } + } + } + + suspend fun close() { + connection.close() + } + + internal suspend inline fun post(path: String, data: String, contentType: AsciiString): T { + return post(path = AsciiString.of(path), data = data, contentType = contentType, deserializer = serializer()) + } + + private suspend fun post(path: AsciiString, data: String, contentType: AsciiString, deserializer: DeserializationStrategy): T { + return connection.stream { stream, result -> + val bufferAllocator = stream.alloc() + stream.pipeline().addLast(Http2StreamJsonInboundHandler(result = result, bufferAllocator = bufferAllocator, deserializer = deserializer)) + + stream.writeHeaders( + headers = ReadOnlyHttp2Headers.clientHeaders( + true, + HttpMethod.POST.asciiName(), + path, + scheme, + authority, + HttpHeaderNames.CONTENT_TYPE, contentType, + HttpHeaderNames.ACCEPT_ENCODING, HttpHeaderValues.GZIP, + *commonHeaders + ), + endStream = false, + ) + stream.writeData(ByteBufUtil.writeUtf8(bufferAllocator, data), endStream = true) + } + } + + suspend fun head(path: CharSequence): HttpResponseStatus { + return connection.stream { stream, result -> + stream.pipeline().addLast(object : InboundHandlerResultTracker(result) { + override fun channelRead0(context: ChannelHandlerContext, frame: Http2HeadersFrame) { + result.complete(HttpResponseStatus.parseLine(frame.headers().status())) + } + }) + + stream.writeHeaders(createHeaders(HttpMethod.HEAD, AsciiString.of(path)), endStream = true) + } + } + + suspend fun getRedirectLocation(path: CharSequence): CharSequence? { + return connection.stream { stream, result -> + stream.pipeline().addLast(object : InboundHandlerResultTracker(result) { + override fun channelRead0(context: ChannelHandlerContext, frame: Http2HeadersFrame) { + result.complete( + when (HttpResponseStatus.parseLine(frame.headers().status())) { + HttpResponseStatus.FOUND, HttpResponseStatus.MOVED_PERMANENTLY, HttpResponseStatus.TEMPORARY_REDIRECT, HttpResponseStatus.SEE_OTHER -> { + frame.headers().get(HttpHeaderNames.LOCATION) + } + else -> { + null + } + } + ) + } + }) + + stream.writeHeaders(createHeaders(HttpMethod.HEAD, AsciiString.of(path)), endStream = true) + } + } + + suspend fun put(path: AsciiString, writer: suspend (stream: Http2StreamChannel) -> Unit) { + connection.stream { stream, result -> + stream.pipeline().addLast(WebDavPutStatusChecker(result)) + + stream.writeHeaders(createHeaders(HttpMethod.PUT, path), endStream = false) + writer(stream) + + // 1. writer must send the last data frame with endStream=true + // 2. stream now has the half-closed state - we listen for server header response with endStream + // 3. our ChannelInboundHandler above checks status and Netty closes the stream (as endStream was sent by both client and server) + } + } + + internal fun createHeaders(method: HttpMethod, path: AsciiString): Http2Headers { + return ReadOnlyHttp2Headers.clientHeaders(true, method.asciiName(), path, scheme, authority, *commonHeaders) + } +} + +private class WebDavPutStatusChecker(private val result: CompletableDeferred) : InboundHandlerResultTracker(result) { + override fun channelRead0(context: ChannelHandlerContext, frame: Http2HeadersFrame) { + if (!frame.isEndStream) { + return + } + + val status = HttpResponseStatus.parseLine(frame.headers().status()) + // WebDAV server returns 204 for existing resources + if (status == HttpResponseStatus.CREATED || status == HttpResponseStatus.NO_CONTENT || status == HttpResponseStatus.OK) { + result.complete(Unit) + } + else { + result.completeExceptionally(IllegalStateException("Unexpected response status: $status")) + } + } +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnectionFactory.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnectionFactory.kt new file mode 100644 index 000000000000..0b5a49817f62 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ClientConnectionFactory.kt @@ -0,0 +1,126 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.http2Client + +import io.netty.bootstrap.Bootstrap +import io.netty.channel.ChannelOption +import io.netty.channel.MultiThreadIoEventLoopGroup +import io.netty.channel.epoll.EpollIoHandler +import io.netty.channel.epoll.EpollSocketChannel +import io.netty.channel.kqueue.KQueueIoHandler +import io.netty.channel.kqueue.KQueueSocketChannel +import io.netty.channel.nio.NioIoHandler +import io.netty.channel.socket.nio.NioSocketChannel +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http2.Http2SecurityUtil +import io.netty.handler.ssl.* +import io.netty.handler.ssl.util.InsecureTrustManagerFactory +import io.netty.util.AsciiString +import io.netty.util.internal.PlatformDependent +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.common.Attributes +import io.opentelemetry.api.trace.Span +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.Runnable +import kotlinx.coroutines.withContext +import java.net.InetSocketAddress +import kotlin.coroutines.CoroutineContext + +private fun nioIoHandlerAndSocketChannel() = NioIoHandler.newFactory() to NioSocketChannel::class.java + +internal class Http2ClientConnectionFactory( + private val bootstrapTemplate: Bootstrap, + private val sslContext: SslContext?, + private val ioDispatcher: CoroutineDispatcher, +) { + suspend inline fun use(block: (Http2ClientConnectionFactory) -> T): T { + try { + return block(this) + } + finally { + withContext(NonCancellable) { + close() + } + } + } + + fun connect(host: String, port: Int = 443): Http2ClientConnection { + return connect(InetSocketAddress.createUnresolved(host, port.let { if (it == -1) 443 else it })) + } + + fun connect(server: InetSocketAddress): Http2ClientConnection { + return Http2ClientConnection( + connection = Http2ConnectionProvider(server = server, bootstrapTemplate = bootstrapTemplate, sslContext = sslContext, ioDispatcher = ioDispatcher), + scheme = AsciiString.of(if (sslContext == null) "http" else "https"), + authority = AsciiString.of(server.hostString + ":" + server.port), + commonHeaders = arrayOf(HttpHeaderNames.USER_AGENT, AsciiString.of("IJ Builder")), + ) + } + + suspend fun close() { + bootstrapTemplate.config().group().shutdownGracefully().joinNonCancellable() + } +} + +internal fun createHttp2ClientSessionFactory(useSsl: Boolean = true, trustAll: Boolean = false): Http2ClientConnectionFactory { + val useNativeTransport = true + val (ioFactory, socketChannel) = try { + when { + useNativeTransport && PlatformDependent.isOsx() -> KQueueIoHandler.newFactory() to KQueueSocketChannel::class.java + useNativeTransport && !PlatformDependent.isWindows() -> EpollIoHandler.newFactory() to EpollSocketChannel::class.java + else -> nioIoHandlerAndSocketChannel() + } + } + catch (e: Throwable) { + Span.current().recordException(e, Attributes.of(AttributeKey.stringKey("warn"), "cannot use native transport, fallback to NIO")) + nioIoHandlerAndSocketChannel() + } + + val group = MultiThreadIoEventLoopGroup(ioFactory) + + val sslContext: SslContext? = if (useSsl) { + val isOpenSslSupported = SslProvider.isAlpnSupported(SslProvider.OPENSSL) + SslContextBuilder.forClient() + .sslProvider(if (isOpenSslSupported) SslProvider.OPENSSL else SslProvider.JDK) + .ciphers(Http2SecurityUtil.CIPHERS, SupportedCipherSuiteFilter.INSTANCE) + .applicationProtocolConfig( + ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + // OpenSSL provider does not support FATAL_ALERT behavior + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2, + ) + ) + .also { + if (trustAll) { + it.trustManager(InsecureTrustManagerFactory.INSTANCE) + } + } + .build() + } + else { + null + } + + // use JDK DNS resolver - custom one is not easy to configure and overkill in our case (just several host names) + val bootstrapTemplate = Bootstrap() + .group(group) + .channel(socketChannel) + .option(ChannelOption.SO_KEEPALIVE, true) + + // asCoroutineDispatcher is an overkill - we don't need schedule (`Delay`) features + val dispatcher = object : CoroutineDispatcher() { + override fun dispatch(context: CoroutineContext, block: Runnable) { + group.execute(block) + } + + override fun toString(): String = group.toString() + } + + return Http2ClientConnectionFactory( + bootstrapTemplate = bootstrapTemplate, + sslContext = sslContext, + ioDispatcher = dispatcher, + ) +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ConnectionProvider.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ConnectionProvider.kt new file mode 100644 index 000000000000..7b76accc8909 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2ConnectionProvider.kt @@ -0,0 +1,216 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("SSBasedInspection") + +package org.jetbrains.intellij.build.http2Client + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBuf +import io.netty.channel.Channel +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInitializer +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.handler.codec.http2.* +import io.netty.handler.ssl.SslContext +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.common.Attributes +import io.opentelemetry.api.trace.Span +import kotlinx.coroutines.* +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.net.InetSocketAddress +import java.nio.channels.ClosedChannelException +import java.util.concurrent.CancellationException +import java.util.concurrent.atomic.AtomicReference +import kotlin.coroutines.coroutineContext +import kotlin.math.min +import kotlin.random.Random +import kotlin.random.asJavaRandom +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +private class ConnectionState( + @JvmField val bootstrap: Http2StreamChannelBootstrap, + @JvmField val channel: Channel, + @JvmField val coroutineScope: CoroutineScope, +) + +private const val MAX_ATTEMPTS = 2 + +// https://stackoverflow.com/questions/55087292/how-to-handle-http-2-goaway-with-java-net-httpclient +// Server can send GOAWAY frame after X streams, that's why we need manager for channel - open a new one in case of such error +internal class Http2ConnectionProvider( + private val server: InetSocketAddress, + private val bootstrapTemplate: Bootstrap, + private val ioDispatcher: CoroutineDispatcher, + sslContext: SslContext?, +) { + private val mutex = Mutex() + private val channelInitializer = Http2ClientFrameInitializer(sslContext, server) + private val connectionRef = AtomicReference() + + private suspend fun getConnection(): ConnectionState { + connectionRef.get()?.takeIf { it.coroutineScope.isActive }?.let { + return it + } + + // not reentrant + mutex.withLock { + return connectionRef.get()?.takeIf { it.coroutineScope.isActive } ?: withContext(ioDispatcher) { openChannel() } + } + } + + // must be called with mutex + private suspend fun openChannel(): ConnectionState { + Span.current().addEvent("open TCP connection", Attributes.of(AttributeKey.stringKey("host"), server.hostString, AttributeKey.longKey("port"), server.port.toLong())) + + val channel = bootstrapTemplate + .clone() + .remoteAddress(server) + .handler(channelInitializer) + .connectAsync() + + val connection = ConnectionState( + bootstrap = Http2StreamChannelBootstrap(channel), + channel = channel, + coroutineScope = CoroutineScope(SupervisorJob() + ioDispatcher), + ) + (channel.pipeline().get("Http2FrameCodec") as Http2FrameCodec).connection().addListener(object : Http2ConnectionAdapter() { + override fun onGoAwayReceived(lastStreamId: Int, errorCode: Long, debugData: ByteBuf?) { + connection.coroutineScope.coroutineContext.cancel() + connectionRef.compareAndSet(connection, null) + } + }) + val old = connectionRef.getAndSet(connection) + require(old == null || !old.coroutineScope.isActive) { + "Old connection must be inactive before opening a new one" + } + return connection + } + + suspend fun close() { + val connection = connectionRef.get() ?: return + try { + connection.coroutineScope.coroutineContext.job.cancelAndJoin() + } + finally { + connection.channel.close().joinNonCancellable() + } + } + + suspend fun stream(block: suspend (streamChannel: Http2StreamChannel, result: CompletableDeferred) -> Unit): T { + var attemptIndex = 0 + var effectiveDelay = 1.seconds.inWholeMilliseconds + val backOffLimitMs = 500.milliseconds.inWholeMilliseconds + val backOffFactor = 2L + val backOffJitter = 0.1 + var suppressedExceptions: MutableList? = null + while (true) { + var currentConnection: ConnectionState? = null + try { + currentConnection = getConnection() + return withContext(currentConnection.coroutineScope.coroutineContext) { + openStreamAndConsume(connectionState = currentConnection, block = block) + } + } + catch (e: Http2Exception) { + handleHttpError(e = e, attemptIndex = attemptIndex, currentConnection = currentConnection) + } + catch (e: CancellationException) { + if (coroutineContext.isActive) { + // task is canceled (due to GoAway or other such reasons), but not parent context - retry (without incrementing attemptIndex) + continue + } + } + catch (e: ClosedChannelException) { + if (attemptIndex >= MAX_ATTEMPTS) { + if (suppressedExceptions != null) { + for (suppressedException in suppressedExceptions) { + e.addSuppressed(suppressedException) + } + } + throw e + } + + if (suppressedExceptions == null) { + suppressedExceptions = ArrayList() + } + suppressedExceptions.add(e) + if (attemptIndex != 0) { + delay(effectiveDelay) + } + + effectiveDelay = min(effectiveDelay * backOffFactor, backOffLimitMs) + (Random.asJavaRandom().nextGaussian() * effectiveDelay * backOffJitter).toLong() + } + + attemptIndex++ + } + } + + private suspend fun handleHttpError(e: Http2Exception, attemptIndex: Int, currentConnection: ConnectionState?) { + when (val error = e.error()) { + Http2Error.REFUSED_STREAM -> { + // result of goaway frame - open a new connection + if (currentConnection != null) { + currentConnection.coroutineScope.cancel() + connectionRef.compareAndSet(currentConnection, null) + Span.current().addEvent("stream refused", Attributes.of(AttributeKey.longKey("streamId"), Http2Exception.streamId(e).toLong())) + } + } + Http2Error.ENHANCE_YOUR_CALM -> { + delay(100L * (attemptIndex + 1)) + } + else -> { + if (attemptIndex >= MAX_ATTEMPTS) { + throw e + } + else { + // log error and continue + Span.current().recordException(e, Attributes.of(AttributeKey.longKey("attemptIndex"), attemptIndex.toLong(), AttributeKey.stringKey("name"), error.name)) + } + } + } + } + + // must be called with ioDispatcher + private suspend fun openStreamAndConsume( + connectionState: ConnectionState, + block: suspend (streamChannel: Http2StreamChannel, result: CompletableDeferred) -> Unit, + ): T { + val streamChannel = connectionState.bootstrap.open().cancellableAwait() + try { + // must be canceled when the parent context is canceled + val result = CompletableDeferred(parent = coroutineContext.job) + block(streamChannel, result) + return result.await() + } + finally { + if (streamChannel.isOpen && connectionState.coroutineScope.isActive) { + withContext(NonCancellable) { + streamChannel.close().joinNonCancellable() + } + } + } + } +} + +private class Http2ClientFrameInitializer( + private val sslContext: SslContext?, + private val server: InetSocketAddress, +) : ChannelInitializer() { + override fun initChannel(channel: Channel) { + val pipeline = channel.pipeline() + sslContext?.let { + pipeline.addFirst(sslContext.newHandler(channel.alloc(), server.hostString, server.port)) + } + + pipeline.addLast("Http2FrameCodec", Http2FrameCodecBuilder.forClient().build()) + // Http2MultiplexHandler requires not-null handlers for child streams - add noop + pipeline.addLast("Http2MultiplexHandler", Http2MultiplexHandler(object : SimpleChannelInboundHandler() { + override fun acceptInboundMessage(message: Any?): Boolean = false + + override fun channelRead0(ctx: ChannelHandlerContext, message: Any?) { + // noop + } + })) + } +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2StreamJsonInboundHandler.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2StreamJsonInboundHandler.kt new file mode 100644 index 000000000000..a6bfa899ef4c --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/Http2StreamJsonInboundHandler.kt @@ -0,0 +1,73 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.http2Client + +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.ByteBufInputStream +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http2.Http2DataFrame +import io.netty.handler.codec.http2.Http2HeadersFrame +import io.netty.handler.codec.http2.Http2StreamFrame +import kotlinx.coroutines.CompletableDeferred +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import java.util.zip.GZIPInputStream + +private val json = Json { + ignoreUnknownKeys = true +} + +internal class Http2StreamJsonInboundHandler( + private val bufferAllocator: ByteBufAllocator, + private val result: CompletableDeferred, + private val deserializer: DeserializationStrategy, +) : InboundHandlerResultTracker(result) { + private var isGzip = false + + private val cumulativeContent = lazy(LazyThreadSafetyMode.NONE) { + val compositeBuffer = bufferAllocator.compositeBuffer(1024) + result.invokeOnCompletion { + compositeBuffer.release() + } + compositeBuffer + } + + override fun channelRead0(context: ChannelHandlerContext, frame: Http2StreamFrame) { + if (frame is Http2DataFrame) { + val frameContent = frame.content() + if (!frame.isEndStream) { + cumulativeContent.value.addComponent(true, frameContent.retain()) + return + } + val data = if (cumulativeContent.isInitialized()) { + cumulativeContent.value.addComponent(true, frameContent.retain()) + cumulativeContent.value + } + else { + frameContent + } + + val response = if (isGzip) { + GZIPInputStream(ByteBufInputStream(data)).use { + json.decodeFromStream(deserializer, it) + } + } + else { + json.decodeFromString(deserializer, data.toString(Charsets.UTF_8)) + } + result.complete(response) + } + else if (frame is Http2HeadersFrame) { + val status = HttpResponseStatus.parseLine(frame.headers().status()) + if (status != HttpResponseStatus.OK) { + result.completeExceptionally(IllegalStateException("Failed with $status")) + return + } + + isGzip = frame.headers().get(HttpHeaderNames.CONTENT_ENCODING) == HttpHeaderValues.GZIP + } + } +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/download.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/download.kt new file mode 100644 index 000000000000..f01bd4f2b898 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/download.kt @@ -0,0 +1,118 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.http2Client + +import com.github.luben.zstd.ZstdDecompressCtx +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http2.Http2DataFrame +import io.netty.handler.codec.http2.Http2HeadersFrame +import io.netty.handler.codec.http2.Http2StreamFrame +import io.netty.util.AsciiString +import kotlinx.coroutines.CompletableDeferred +import java.nio.channels.FileChannel +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import java.security.MessageDigest +import java.util.* + +internal const val MAX_BUFFER_SIZE = 4 * 1014 * 1024 +private val OVERWRITE_OPERATION = EnumSet.of(StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING) + +internal data class DownloadResult(@JvmField var size: Long, @JvmField val digest: MessageDigest) + +internal suspend fun Http2ClientConnection.download(path: String, file: Path, digestFactory: () -> MessageDigest): DownloadResult { + Files.createDirectories(file.parent) + + return connection.stream { stream, result -> + stream.pipeline().addLast( + DownloadHandler( + result = result, + downloadResult = DownloadResult(size = 0, digest = digestFactory()), + file = file, + ), + ) + + stream.writeHeaders(createHeaders(HttpMethod.GET, AsciiString.of(path)), endStream = true) + + result.await() + } +} + +private class DownloadHandler( + private val result: CompletableDeferred, + private val downloadResult: DownloadResult, + private val file: Path, +) : InboundHandlerResultTracker(result) { + private var offset = 0L + private var fileChannel: FileChannel? = null + private var zstdDecompressContext: ZstdDecompressCtx? = null + + override fun acceptInboundMessage(message: Any): Boolean = message is Http2DataFrame || message is Http2HeadersFrame + + override fun handlerAdded(ctx: ChannelHandlerContext?) { + fileChannel = FileChannel.open(file, OVERWRITE_OPERATION) + zstdDecompressContext = ZstdDecompressCtx() + } + + override fun handlerRemoved(context: ChannelHandlerContext) { + try { + fileChannel?.close() + fileChannel = null + } + finally { + zstdDecompressContext?.close() + zstdDecompressContext = null + } + } + + override fun channelRead0(context: ChannelHandlerContext, frame: Http2StreamFrame) { + if (frame is Http2HeadersFrame) { + val status = HttpResponseStatus.parseLine(frame.headers().status()) + if (status != HttpResponseStatus.OK) { + result.completeExceptionally(IllegalStateException("Unexpected response status: $status")) + } + } + else if (frame is Http2DataFrame) { + val content = frame.content() + downloadResult.size += content.readableBytes() + writeChunk(content, context.alloc()) + + if (frame.isEndStream) { + result.complete(downloadResult) + } + } + } + + private fun writeChunk(chunk: ByteBuf, allocator: ByteBufAllocator) { + val sourceBuffer = chunk.nioBuffer() + val zstdDecompressContext = zstdDecompressContext!! + val fileChannel = fileChannel!! + do { + val targetNettyBuffer = allocator.directBuffer((sourceBuffer.remaining() * 4).coerceAtMost(MAX_BUFFER_SIZE)) + try { + val targetBuffer = targetNettyBuffer.nioBuffer(0, targetNettyBuffer.capacity()) + zstdDecompressContext.decompressDirectByteBufferStream(targetBuffer, sourceBuffer) + + targetBuffer.flip() + if (targetBuffer.hasRemaining()) { + targetBuffer.mark() + downloadResult.digest.update(targetBuffer) + targetBuffer.reset() + + do { + offset += fileChannel.write(targetBuffer, offset) + } + while (targetBuffer.hasRemaining()) + } + } + finally { + targetNettyBuffer.release() + } + } + while (sourceBuffer.hasRemaining()) + } +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/netty-util.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/netty-util.kt new file mode 100644 index 000000000000..c6d96978dba2 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/http2Client/netty-util.kt @@ -0,0 +1,147 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +@file:Suppress("SSBasedInspection", "UsePropertyAccessSyntax", "OVERRIDE_DEPRECATION") + +package org.jetbrains.intellij.build.http2Client + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBuf +import io.netty.channel.Channel +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.handler.codec.http2.DefaultHttp2DataFrame +import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame +import io.netty.handler.codec.http2.Http2Headers +import io.netty.handler.codec.http2.Http2StreamChannel +import io.netty.util.concurrent.Future +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine + +internal suspend fun Bootstrap.connectAsync(): Channel = suspendCancellableCoroutine { continuation -> + val future = connect() + + if (future.isDone) { + if (future.isSuccess) { + continuation.resume(future.channel()) + } + else { + continuation.resumeWithException(future.cause()) + } + return@suspendCancellableCoroutine + } + + if (future.isCancellable) { + continuation.invokeOnCancellation { + future.cancel(true) + } + } + + future.addListener { + if (future.isSuccess) { + continuation.resume(future.channel()) + } + else { + continuation.resumeWithException(future.cause()) + } + } +} + +internal suspend fun Future.cancellableAwait(): T { + if (isDone) { + if (isSuccess) { + return getNow() + } + else { + cause()?.let { + throw it + } + } + } + + return suspendCancellableCoroutine { continuation -> + if (isCancellable) { + continuation.invokeOnCancellation { + cancel(true) + } + } + + addListener { + if (isSuccess) { + continuation.resume(get()) + } + else { + continuation.resumeWithException(cause()) + } + } + } +} + +internal suspend fun Future<*>.joinCancellable(cancelFutureOnCancellation: Boolean = true) { + if (isDone) { + cause()?.let { + throw it + } + } + + suspendCancellableCoroutine { continuation -> + if (cancelFutureOnCancellation && isCancellable) { + continuation.invokeOnCancellation { + cancel(true) + } + } + + addListener { + if (isSuccess) { + continuation.resume(Unit) + } + else { + continuation.resumeWithException(cause()) + } + } + } +} + +internal suspend fun Http2StreamChannel.writeHeaders(headers: Http2Headers, endStream: Boolean) { + writeAndFlush(DefaultHttp2HeadersFrame(headers, endStream)).joinCancellable() +} + +internal suspend fun Http2StreamChannel.writeData(data: ByteBuf, endStream: Boolean) { + writeAndFlush(DefaultHttp2DataFrame(data, endStream)).joinCancellable() +} + +internal abstract class InboundHandlerResultTracker( + private val result: CompletableDeferred<*>, +) : SimpleChannelInboundHandler() { + override fun exceptionCaught(context: ChannelHandlerContext, cause: Throwable) { + result.completeExceptionally(cause) + } + + override fun channelInactive(context: ChannelHandlerContext) { + if (!result.isCompleted) { + result.completeExceptionally(IllegalStateException("Stream closed without result ($this)")) + } + } +} + +// not suspendCancellableCoroutine - we must close the channel / event loop group +internal suspend fun Future<*>.joinNonCancellable() { + if (isDone) { + cause()?.let { + throw it + } + } + + // not suspendCancellableCoroutine - we must close the channel + return suspendCoroutine { continuation -> + addListener { future -> + if (future.isSuccess) { + continuation.resume(Unit) + } + else { + continuation.resumeWithException(future.cause()) + } + } + } +} diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ArchivedCompilationContext.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ArchivedCompilationContext.kt index f32b1e5426fc..76b23c254eb7 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ArchivedCompilationContext.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/ArchivedCompilationContext.kt @@ -8,7 +8,7 @@ import org.jetbrains.intellij.build.BuildMessages import org.jetbrains.intellij.build.BuildOptions import org.jetbrains.intellij.build.BuildPaths import org.jetbrains.intellij.build.CompilationContext -import org.jetbrains.intellij.build.impl.compilation.ArchivedCompilationOutputsStorage +import org.jetbrains.intellij.build.impl.compilation.ArchivedCompilationOutputStorage import org.jetbrains.jps.model.module.JpsModule import java.io.File import java.nio.file.Path @@ -17,7 +17,7 @@ import kotlin.io.path.writeLines @ApiStatus.Internal class ArchivedCompilationContext( private val delegate: CompilationContext, - private val storage: ArchivedCompilationOutputsStorage = ArchivedCompilationOutputsStorage(paths = delegate.paths, classesOutputDirectory = delegate.classesOutputDirectory).apply { + private val storage: ArchivedCompilationOutputStorage = ArchivedCompilationOutputStorage(paths = delegate.paths, classesOutputDirectory = delegate.classesOutputDirectory).apply { delegate.options.pathToCompiledClassesArchivesMetadata?.let { this.loadMetadataFile(it) } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ArchivedCompilationOutputStorage.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ArchivedCompilationOutputStorage.kt new file mode 100644 index 000000000000..30818a3671e2 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ArchivedCompilationOutputStorage.kt @@ -0,0 +1,58 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.impl.compilation + +import kotlinx.serialization.json.Json +import org.jetbrains.annotations.ApiStatus +import org.jetbrains.intellij.build.BuildPaths +import org.jetbrains.intellij.build.io.AddDirEntriesMode +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.util.concurrent.ConcurrentHashMap +import kotlin.io.path.invariantSeparatorsPathString + +@ApiStatus.Internal +class ArchivedCompilationOutputStorage( + private val paths: BuildPaths, + private val classesOutputDirectory: Path, + val archivedOutputDirectory: Path = getArchiveStorage(classesOutputDirectory.parent), +) { + private val unarchivedToArchivedMap = ConcurrentHashMap() + + internal fun loadMetadataFile(metadataFile: Path) { + val metadata = Json.decodeFromString(Files.readString(metadataFile)) + for (entry in metadata.files) { + unarchivedToArchivedMap.put(classesOutputDirectory.resolve(entry.key), archivedOutputDirectory.resolve(entry.key).resolve("${entry.value}.jar")) + } + } + + suspend fun getArchived(path: Path): Path { + if (Files.isRegularFile(path) || !path.startsWith(classesOutputDirectory)) { + return path + } + + unarchivedToArchivedMap.get(path)?.let { + return it + } + + val archived = archive(path) + return unarchivedToArchivedMap.putIfAbsent(path, archived) ?: archived + } + + private suspend fun archive(path: Path): Path { + val name = classesOutputDirectory.relativize(path).toString() + + val archive = Files.createTempFile(paths.tempDir, name.replace(File.separator, "_"), ".jar") + Files.deleteIfExists(archive) + val hash = packAndComputeHash(addDirEntriesMode = AddDirEntriesMode.ALL, name = name, archive = archive, directory = path) + + val result = archivedOutputDirectory.resolve(name).resolve("$hash.jar") + Files.createDirectories(result.parent) + Files.move(archive, result, StandardCopyOption.REPLACE_EXISTING) + + return result + } + + internal fun getMapping(): List> = unarchivedToArchivedMap.entries.sortedBy { it.key.invariantSeparatorsPathString } +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/CompilationPartsUtil.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/CompilationPartsUtil.kt index e2bb5ba88e39..58cd650d62b3 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/CompilationPartsUtil.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/CompilationPartsUtil.kt @@ -4,30 +4,29 @@ package org.jetbrains.intellij.build.impl.compilation -import com.intellij.platform.util.coroutines.forEachConcurrent -import com.intellij.platform.util.coroutines.mapConcurrent import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json -import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.VisibleForTesting import org.jetbrains.intellij.build.BuildMessages -import org.jetbrains.intellij.build.BuildPaths import org.jetbrains.intellij.build.CompilationContext +import org.jetbrains.intellij.build.forEachConcurrent +import org.jetbrains.intellij.build.http2Client.createHttp2ClientSessionFactory import org.jetbrains.intellij.build.io.AddDirEntriesMode import org.jetbrains.intellij.build.io.zip import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder import org.jetbrains.intellij.build.telemetry.use -import java.io.File import java.math.BigInteger +import java.net.InetSocketAddress +import java.net.URI import java.nio.ByteBuffer import java.nio.channels.FileChannel import java.nio.file.* @@ -42,14 +41,13 @@ import java.util.concurrent.atomic.AtomicLong import java.util.zip.GZIPOutputStream import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.deleteRecursively -import kotlin.io.path.invariantSeparatorsPathString import kotlin.io.path.listDirectoryEntries -internal val uploadParallelism = Runtime.getRuntime().availableProcessors() -internal val downloadParallelism = uploadParallelism +internal val uploadParallelism = Runtime.getRuntime().availableProcessors().coerceIn(4, 32) +internal val downloadParallelism = (Runtime.getRuntime().availableProcessors() * 2).coerceIn(8, 16) private const val BRANCH_PROPERTY_NAME = "intellij.build.compiled.classes.branch" -private const val SERVER_URL = "intellij.build.compiled.classes.server.url" +private const val SERVER_URL_PROPERTY = "intellij.build.compiled.classes.server.url" private const val UPLOAD_PREFIX = "intellij.build.compiled.classes.upload.prefix" class CompilationCacheUploadConfiguration( @@ -59,9 +57,18 @@ class CompilationCacheUploadConfiguration( branch: String? = null, uploadPredix: String? = null, ) { - val serverUrl: String by lazy { serverUrl ?: normalizeServerUrl() } + val serverUrl: String by lazy(LazyThreadSafetyMode.NONE) { serverUrl ?: getAndNormalizeServerUrlBySystemProperty() } - val uploadPrefix: String by lazy { + private val serverUri: URI by lazy(LazyThreadSafetyMode.NONE) { URI(serverUrl ?: getAndNormalizeServerUrlBySystemProperty()) } + + // even if empty, a final path must always start with `/` (otherwise, error like "client sent invalid :path header") + val serverUrlPathPrefix: String by lazy(LazyThreadSafetyMode.NONE) { serverUri.path } + + val serverAddress: InetSocketAddress by lazy { + InetSocketAddress.createUnresolved(serverUri.host, serverUri.port.let { if (it == -1) 443 else it }) + } + + val uploadUrlPathPrefix: String by lazy { uploadPredix ?: System.getProperty(UPLOAD_PREFIX, "intellij-compile/v2").also { check(!it.isNullOrBlank()) { "$UPLOAD_PREFIX system property should not be blank." @@ -78,11 +85,10 @@ class CompilationCacheUploadConfiguration( } } -private fun normalizeServerUrl(): String { - val serverUrlPropertyName = SERVER_URL - var result = System.getProperty(serverUrlPropertyName)?.trimEnd('/') +private fun getAndNormalizeServerUrlBySystemProperty(): String { + var result = System.getProperty(SERVER_URL_PROPERTY)?.trimEnd('/') check(!result.isNullOrBlank()) { - "Compilation cache archive server url is not defined. Please set $serverUrlPropertyName system property." + "Compilation cache archive server url is not defined. Please set $SERVER_URL_PROPERTY system property." } if (!result.startsWith("http")) { @Suppress("HttpUrlsUsage") @@ -112,10 +118,6 @@ suspend fun packAndUploadToServer(context: CompilationContext, zipDir: Path, con } } -private fun createBufferPool(@Suppress("SameParameterValue") maxPoolSize: Int): DirectFixedSizeByteBufferPool { - return DirectFixedSizeByteBufferPool(bufferSize = MAX_BUFFER_SIZE, maxPoolSize = maxPoolSize) -} - private suspend fun packCompilationResult(zipDir: Path, context: CompilationContext, addDirEntriesMode: AddDirEntriesMode = AddDirEntriesMode.ALL): List { val incremental = context.options.incrementalCompilation if (!incremental) { @@ -163,15 +165,17 @@ private suspend fun packCompilationResult(zipDir: Path, context: CompilationCont } } - spanBuilder("build zip archives").use(Dispatchers.IO) { - items.forEachConcurrent { item -> - item.hash = packAndComputeHash(addDirEntriesMode = addDirEntriesMode, name = item.name, archive = item.archive, directory = item.output) + spanBuilder("build zip archives").use { + for (item in items) { + launch { + item.hash = packAndComputeHash(addDirEntriesMode = addDirEntriesMode, name = item.name, archive = item.archive, directory = item.output) + } } } return items } -private suspend fun packAndComputeHash( +internal suspend fun packAndComputeHash( addDirEntriesMode: AddDirEntriesMode, name: String, archive: Path, @@ -214,7 +218,7 @@ private suspend fun upload( CompilationPartsMetadata( serverUrl = config.serverUrl, branch = config.branch, - prefix = config.uploadPrefix, + prefix = config.uploadUrlPathPrefix, files = items.associateTo(TreeMap()) { item -> item.name to item.hash!! }, @@ -236,68 +240,24 @@ private suspend fun upload( messages.artifactBuilt(gzippedMetadataFile.toString()) } - spanBuilder("upload archives").setAttribute(AttributeKey.stringArrayKey("items"), items.map(PackAndUploadItem::name)).use { - createBufferPool(maxPoolSize = uploadParallelism * 2).use { bufferPool -> - uploadArchives( - reportStatisticValue = messages::reportStatisticValue, - config = config, - metadataJson = metadataJson, - httpClient = httpClient, - items = items, - bufferPool = bufferPool, - ) + val serverAddress = config.serverAddress + createHttp2ClientSessionFactory(trustAll = serverAddress.hostString == "127.0.0.1").use { client -> + client.connect(serverAddress).use { connection -> + spanBuilder("upload archives").setAttribute(AttributeKey.stringArrayKey("items"), items.map(PackAndUploadItem::name)).use { + uploadArchives( + reportStatisticValue = messages::reportStatisticValue, + config = config, + metadataJson = metadataJson, + httpConnection = connection, + items = items, + ) + } } } } -private fun getArchivesStorage(fallbackPersistentCacheRoot: Path): Path { - return (System.getProperty("agent.persistent.cache")?.let { Path.of(it) } ?: fallbackPersistentCacheRoot) - .resolve("idea-compile-parts-v2") -} - -@ApiStatus.Internal -class ArchivedCompilationOutputsStorage( - private val paths: BuildPaths, - private val classesOutputDirectory: Path, - val archivedOutputDirectory: Path = getArchivesStorage(classesOutputDirectory.parent), -) { - private val unarchivedToArchivedMap = ConcurrentHashMap() - - internal fun loadMetadataFile(metadataFile: Path) { - val metadata = Json.decodeFromString(Files.readString(metadataFile)) - for (entry in metadata.files) { - unarchivedToArchivedMap.put(classesOutputDirectory.resolve(entry.key), archivedOutputDirectory.resolve(entry.key).resolve("${entry.value}.jar")) - } - } - - suspend fun getArchived(path: Path): Path { - if (Files.isRegularFile(path) || !path.startsWith(classesOutputDirectory)) { - return path - } - - unarchivedToArchivedMap.get(path)?.let { - return it - } - - val archived = archive(path) - return unarchivedToArchivedMap.putIfAbsent(path, archived) ?: archived - } - - private suspend fun archive(path: Path): Path { - val name = classesOutputDirectory.relativize(path).toString() - - val archive = Files.createTempFile(paths.tempDir, name.replace(File.separator, "_"), ".jar") - Files.deleteIfExists(archive) - val hash = packAndComputeHash(addDirEntriesMode = AddDirEntriesMode.ALL, name = name, archive = archive, directory = path) - - val result = archivedOutputDirectory.resolve(name).resolve("$hash.jar") - Files.createDirectories(result.parent) - Files.move(archive, result, StandardCopyOption.REPLACE_EXISTING) - - return result - } - - internal fun getMapping(): List> = unarchivedToArchivedMap.entries.sortedBy { it.key.invariantSeparatorsPathString } +internal fun getArchiveStorage(fallbackPersistentCacheRoot: Path): Path { + return (System.getProperty("agent.persistent.cache")?.let { Path.of(it) } ?: fallbackPersistentCacheRoot).resolve("idea-compile-parts-v2") } @VisibleForTesting @@ -309,7 +269,7 @@ suspend fun fetchAndUnpackCompiledClasses( saveHash: Boolean, ) { val metadata = Json.decodeFromString(Files.readString(metadataFile)) - val tempDownloadStorage = getArchivesStorage(classOutput.parent) + val tempDownloadStorage = getArchiveStorage(classOutput.parent) val items = metadata.files.mapTo(ArrayList(metadata.files.size)) { entry -> FetchAndUnpackItem( @@ -321,10 +281,9 @@ suspend fun fetchAndUnpackCompiledClasses( } items.sortBy { it.name } - var verifyTime = 0L val upToDate = ConcurrentHashMap.newKeySet() - spanBuilder("check previously unpacked directories").use { span -> - verifyTime += checkPreviouslyUnpackedDirectories( + var verifyTime = spanBuilder("check previously unpacked directories").use { span -> + checkPreviouslyUnpackedDirectories( items = items, span = span, upToDate = upToDate, @@ -336,32 +295,32 @@ suspend fun fetchAndUnpackCompiledClasses( val toUnpack = LinkedHashSet(items.size) val verifyStart = System.nanoTime() - val toDownload = spanBuilder("check previously downloaded archives").use(Dispatchers.IO) { span -> - items - .filter { item -> - if (upToDate.contains(item.name)) { - return@filter false - } - - if (!skipUnpack) { - toUnpack.add(item) - } - true + val toDownload = ConcurrentHashMap.newKeySet() + spanBuilder("check previously downloaded archives").use { span -> + items.filter { item -> + if (upToDate.contains(item.name)) { + return@filter false } - .mapConcurrent { item -> + + if (!skipUnpack) { + toUnpack.add(item) + } + true + } + .forEachConcurrent(Runtime.getRuntime().availableProcessors().coerceAtMost(4)) { item -> val file = item.file when { - Files.notExists(file) -> item - item.hash == computeHash(file) -> null + Files.notExists(file) -> toDownload.add(item) + item.hash == computeHash(file) -> return@forEachConcurrent else -> { span.addEvent("file has unexpected hash, will refetch", Attributes.of(AttributeKey.stringKey("file"), "${item.name}/${item.hash}.jar")) Files.deleteIfExists(file) - item + toDownload.add(item) } } } - }.filterNotNull() - verifyTime += TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - verifyStart) + } + verifyTime += System.nanoTime() - verifyStart // toUnpack is performed as part of download for (item in toDownload) { @@ -398,6 +357,7 @@ suspend fun fetchAndUnpackCompiledClasses( Span.current().addEvent("failed to cleanup outdated archives", Attributes.of(AttributeKey.stringKey("error"), e.message ?: "")) } + reportStatisticValue("compile-parts:verify:time", TimeUnit.NANOSECONDS.toMillis(verifyTime).toString()) reportStatisticValue("compile-parts:cleanup:time", TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - start)).toString()) reportStatisticValue("compile-parts:removed:bytes", bytes.toString()) reportStatisticValue("compile-parts:removed:count", count.toString()) @@ -406,23 +366,17 @@ suspend fun fetchAndUnpackCompiledClasses( spanBuilder("fetch compiled classes archives").use { val start = System.nanoTime() - val prefix = metadata.prefix - val serverUrl = metadata.serverUrl - val downloadedBytes = AtomicLong() val failed: List = if (toDownload.isEmpty()) { emptyList() } else { - val httpClientWithoutFollowingRedirects = httpClient.newBuilder().followRedirects(false).build() - // 4MB block, x2 of thread count - one buffer to source, another one for target - createBufferPool(downloadParallelism * 2).use { bufferPool -> + createHttp2ClientSessionFactory(trustAll = metadata.serverUrl.contains("127.0.0.1")).use { client -> downloadCompilationCache( - serverUrl = serverUrl, - prefix = prefix, + client = client, + serverUrl = metadata.serverUrl, + prefix = metadata.prefix, toDownload = toDownload, - client = httpClientWithoutFollowingRedirects, - bufferPool = bufferPool, downloadedBytes = downloadedBytes, skipUnpack = skipUnpack, saveHash = saveHash, @@ -442,10 +396,12 @@ suspend fun fetchAndUnpackCompiledClasses( } val start = System.nanoTime() - spanBuilder("unpack compiled classes archives").use(Dispatchers.IO) { - toUnpack.forEachConcurrent { item -> - spanBuilder("unpack").setAttribute("name", item.name).use { - unpackArchive(item, saveHash) + spanBuilder("unpack compiled classes archives").use { + for (item in toUnpack) { + launch { + spanBuilder("unpack").setAttribute("name", item.name).use { + unpackArchive(item, saveHash) + } } } } @@ -466,55 +422,57 @@ private suspend fun checkPreviouslyUnpackedDirectories( } val start = System.nanoTime() - withContext(Dispatchers.IO) { - launch { + coroutineScope { + launch(Dispatchers.IO) { spanBuilder("remove stalled directories not present in metadata").setAttribute(AttributeKey.stringArrayKey("keys"), java.util.List.copyOf(metadata.files.keys)).use { removeStalledDirs(metadata, classOutput) } } - items.forEachConcurrent { item -> - val out = item.output - if (Files.notExists(out)) { - span.addEvent("output directory doesn't exist", Attributes.of(AttributeKey.stringKey("name"), item.name, AttributeKey.stringKey("outDir"), out.toString())) - return@forEachConcurrent - } - - val hashFile = out.resolve(".hash") - if (!Files.isRegularFile(hashFile)) { - span.addEvent("no .hash file in output directory", Attributes.of(AttributeKey.stringKey("name"), item.name)) - out.deleteRecursively() - return@forEachConcurrent - } - - try { - val actual = Files.readString(hashFile) - if (actual == item.hash) { - upToDate.add(item.name) + for (item in items) { + launch { + val out = item.output + if (Files.notExists(out)) { + span.addEvent("output directory doesn't exist", Attributes.of(AttributeKey.stringKey("name"), item.name, AttributeKey.stringKey("outDir"), out.toString())) + return@launch } - else { - span.addEvent( - "output directory hash mismatch", - Attributes.of( - AttributeKey.stringKey("name"), item.name, - AttributeKey.stringKey("expected"), item.hash, - AttributeKey.stringKey("actual"), actual, + + val hashFile = out.resolve(".hash") + if (!Files.isRegularFile(hashFile)) { + span.addEvent("no .hash file in output directory", Attributes.of(AttributeKey.stringKey("name"), item.name)) + out.deleteRecursively() + return@launch + } + + try { + val actual = Files.readString(hashFile) + if (actual == item.hash) { + upToDate.add(item.name) + } + else { + span.addEvent( + "output directory hash mismatch", + Attributes.of( + AttributeKey.stringKey("name"), item.name, + AttributeKey.stringKey("expected"), item.hash, + AttributeKey.stringKey("actual"), actual, + ) ) - ) + out.deleteRecursively() + } + } + catch (e: CancellationException) { + throw e + } + catch (e: Throwable) { + span.addEvent("output directory hash calculation failed", Attributes.of(AttributeKey.stringKey("name"), item.name)) + span.recordException(e, Attributes.of(AttributeKey.stringKey("name"), item.name)) out.deleteRecursively() } } - catch (e: CancellationException) { - throw e - } - catch (e: Throwable) { - span.addEvent("output directory hash calculation failed", Attributes.of(AttributeKey.stringKey("name"), item.name)) - span.recordException(e, Attributes.of(AttributeKey.stringKey("name"), item.name)) - out.deleteRecursively() - } } } - return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) + return System.nanoTime() - start } private fun CoroutineScope.removeStalledDirs( @@ -557,7 +515,7 @@ private fun CoroutineScope.removeStalledDirs( private val sharedDigest = MessageDigest.getInstance("SHA-256", java.security.Security.getProvider("SUN")) internal fun sha256() = sharedDigest.clone() as MessageDigest -private fun computeHash(file: Path): String { +internal fun computeHash(file: Path): String { val messageDigest = sha256() FileChannel.open(file, READ_OPERATION).use { channel -> val fileSize = channel.size() @@ -583,7 +541,7 @@ private fun computeHash(file: Path): String { // we cannot change file extension or prefix, so, add suffix internal fun digestToString(digest: MessageDigest): String = BigInteger(1, digest.digest()).toString(36) + "-z" -data class PackAndUploadItem( +internal data class PackAndUploadItem( @JvmField val output: Path, @JvmField val name: String, @JvmField val archive: Path, @@ -604,7 +562,7 @@ internal data class FetchAndUnpackItem( * URL for each part should be constructed like: 
${serverUrl}/${prefix}/${files.key}/${files.value}.jar
*/ @Serializable -private data class CompilationPartsMetadata( +internal data class CompilationPartsMetadata( @JvmField @SerialName("server-url") val serverUrl: String, @JvmField val branch: String, @JvmField val prefix: String, diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/DirectFixedSizeByteBufferPool.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/DirectFixedSizeByteBufferPool.kt index c54d2590a768..d9dd75e643f7 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/DirectFixedSizeByteBufferPool.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/DirectFixedSizeByteBufferPool.kt @@ -1,38 +1,53 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl.compilation -import com.intellij.util.lang.ByteBufferCleaner -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.getOrElse +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import org.jetbrains.intellij.build.io.unmapBuffer import java.nio.ByteBuffer import java.nio.ByteOrder +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicInteger -internal class DirectFixedSizeByteBufferPool(private val bufferSize: Int, maxPoolSize: Int) : AutoCloseable { - private val pool = Channel(capacity = maxPoolSize) +internal class DirectFixedSizeByteBufferPool(private val bufferSize: Int, private val maxPoolSize: Int) : AutoCloseable { + private val pool = ConcurrentLinkedQueue() + private val count = AtomicInteger() + @JvmField + val semaphore: Semaphore = Semaphore(maxPoolSize) - fun allocate(): ByteBuffer { - val result = pool.tryReceive() - return when { - result.isSuccess -> result.getOrThrow() - result.isClosed -> throw IllegalStateException("Pool is closed") - else -> ByteBuffer.allocateDirect(bufferSize) + private fun allocate(): ByteBuffer { + val result = pool.poll() ?: return ByteBuffer.allocateDirect(bufferSize) + count.decrementAndGet() + return result + } + + suspend inline fun withBuffer(task: (buffer: ByteBuffer) -> T): T { + return semaphore.withPermit { + val buffer = allocate() + try { + task(buffer) + } + finally { + release(buffer) + } } } - fun release(buffer: ByteBuffer) { + private fun release(buffer: ByteBuffer) { buffer.clear() buffer.order(ByteOrder.BIG_ENDIAN) - pool.trySend(buffer).getOrElse { - // if the pool is full, we simply discard the buffer - ByteBufferCleaner.unmapBuffer(buffer) + if (count.incrementAndGet() < maxPoolSize) { + pool.offer(buffer) + } + else { + count.decrementAndGet() + unmapBuffer(buffer) } } - // pool is not expected to be used during releaseAll call override fun close() { while (true) { - ByteBufferCleaner.unmapBuffer(pool.tryReceive().getOrNull() ?: break) + unmapBuffer(pool.poll() ?: return) } - pool.close() } } \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheDownloader.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheDownloader.kt index 74d169480403..1fca14d1e66e 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheDownloader.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheDownloader.kt @@ -3,7 +3,6 @@ package org.jetbrains.intellij.build.impl.compilation -import com.intellij.platform.util.coroutines.forEachConcurrent import com.intellij.util.io.Decompressor import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes @@ -12,6 +11,7 @@ import kotlinx.coroutines.* import okhttp3.Request import okio.sink import org.jetbrains.intellij.build.CompilationContext +import org.jetbrains.intellij.build.forEachConcurrent import org.jetbrains.intellij.build.impl.compilation.cache.CommitsHistory import org.jetbrains.intellij.build.impl.compilation.cache.getAllCompilationOutputs import org.jetbrains.intellij.build.impl.compilation.cache.parseSourcesStateFile diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheUploader.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheUploader.kt index fa8a777c6d6d..985ad4b6b13a 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheUploader.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/PortableCompilationCacheUploader.kt @@ -3,7 +3,6 @@ package org.jetbrains.intellij.build.impl.compilation import com.google.gson.stream.JsonReader -import com.intellij.platform.util.coroutines.forEachConcurrent import com.intellij.util.io.Compressor import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes @@ -11,12 +10,14 @@ import io.opentelemetry.api.trace.Span import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import okhttp3.MediaType.Companion.toMediaType import okhttp3.Request import okhttp3.RequestBody import okio.BufferedSink import okio.source import org.jetbrains.intellij.build.BuildMessages import org.jetbrains.intellij.build.CompilationContext +import org.jetbrains.intellij.build.forEachConcurrent import org.jetbrains.intellij.build.impl.compilation.cache.CommitsHistory import org.jetbrains.intellij.build.impl.compilation.cache.getAllCompilationOutputs import org.jetbrains.intellij.build.io.copyFile @@ -37,6 +38,7 @@ import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.deleteRecursively private const val SOURCES_STATE_FILE_NAME = "target_sources_state.json" +private val MEDIA_TYPE_BINARY = "application/octet-stream".toMediaType() internal class PortableCompilationCacheUploader( private val context: CompilationContext, diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ZstdCompressContextPool.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ZstdCompressContextPool.kt new file mode 100644 index 000000000000..829352f7b044 --- /dev/null +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/ZstdCompressContextPool.kt @@ -0,0 +1,43 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.intellij.build.impl.compilation + +import com.github.luben.zstd.ZstdCompressCtx +import java.util.concurrent.ConcurrentLinkedQueue + +// we cannot use Netty Recycler as we must close ZstdCompressCtx after use of pool +internal class ZstdCompressContextPool(private val level: Int = 3) : AutoCloseable { + private val pool = ConcurrentLinkedQueue() + + inline fun withZstd(task: (zstd: ZstdCompressCtx) -> T): T { + val zstd = allocate() + try { + return task(zstd) + } + finally { + zstd.reset() + pool.offer(zstd) + } + } + + private fun allocate(): ZstdCompressCtx { + pool.poll()?.let { + configure(it) + return it + } + + val zstd = ZstdCompressCtx() + configure(zstd) + return zstd + } + + private fun configure(zstd: ZstdCompressCtx) { + zstd.setLevel(level) + //zstd.setLong(64) + } + + override fun close() { + while (true) { + (pool.poll() ?: return).close() + } + } +} \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/download.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/download.kt index 7a6ed1c8a58c..50edf7f73149 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/download.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/download.kt @@ -1,31 +1,26 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build.impl.compilation -import com.github.luben.zstd.ZstdDirectBufferDecompressingStreamNoFinalizer -import com.intellij.platform.util.coroutines.mapConcurrent import com.intellij.util.lang.HashMapZipFile import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes -import io.opentelemetry.api.trace.Span -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response import okio.IOException +import org.jetbrains.intellij.build.forEachConcurrent +import org.jetbrains.intellij.build.http2Client.Http2ClientConnection +import org.jetbrains.intellij.build.http2Client.Http2ClientConnectionFactory +import org.jetbrains.intellij.build.http2Client.download import org.jetbrains.intellij.build.io.INDEX_FILENAME -import org.jetbrains.intellij.build.retryWithExponentialBackOff import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder import org.jetbrains.intellij.build.telemetry.use -import java.net.HttpURLConnection -import java.nio.ByteBuffer +import java.math.BigInteger +import java.net.URI import java.nio.channels.FileChannel import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption -import java.security.MessageDigest import java.util.* import java.util.concurrent.CancellationException +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.atomic.AtomicLong import kotlin.io.path.name @@ -33,46 +28,54 @@ private val OVERWRITE_OPERATION = EnumSet.of(StandardOpenOption.WRITE, StandardO internal suspend fun downloadCompilationCache( serverUrl: String, + client: Http2ClientConnectionFactory, prefix: String, - toDownload: List, - client: OkHttpClient, - bufferPool: DirectFixedSizeByteBufferPool, + toDownload: Collection, downloadedBytes: AtomicLong, skipUnpack: Boolean, saveHash: Boolean, ): List { - var urlWithPrefix = "$serverUrl/$prefix/" + var urlPathWithPrefix = "/$prefix/" // first let's check for initial redirect (mirror selection) - spanBuilder("mirror selection").use { span -> - client.newCall(Request.Builder().url(urlWithPrefix).head().build()).executeAsync().use { response -> - val statusCode = response.code - val locationHeader = response.header("location") - if (locationHeader != null && (statusCode == HttpURLConnection.HTTP_MOVED_TEMP || - statusCode == HttpURLConnection.HTTP_MOVED_PERM || - statusCode == 307 || - statusCode == HttpURLConnection.HTTP_SEE_OTHER)) { - urlWithPrefix = locationHeader - span.addEvent("redirected to mirror", Attributes.of(AttributeKey.stringKey("url"), urlWithPrefix)) + val initialServerUri = URI(serverUrl) + var effectiveServerUri = initialServerUri + var connection: Http2ClientConnection? = client.connect(effectiveServerUri.host, effectiveServerUri.port) + try { + spanBuilder("mirror selection").use { span -> + val newLocation = connection!!.getRedirectLocation(urlPathWithPrefix) + if (newLocation == null) { + span.addEvent("origin server will be used", Attributes.of(AttributeKey.stringKey("url"), urlPathWithPrefix)) } else { - span.addEvent("origin server will be used", Attributes.of(AttributeKey.stringKey("url"), urlWithPrefix)) + effectiveServerUri = URI(newLocation.toString()) + urlPathWithPrefix = effectiveServerUri.path + span.addEvent("redirected to mirror", Attributes.of(AttributeKey.stringKey("url"), urlPathWithPrefix)) } } } + finally { + if (initialServerUri != effectiveServerUri) { + connection?.close() + connection = null + } + } - return withContext(Dispatchers.IO) { - toDownload.mapConcurrent(downloadParallelism) { item -> - val url = "$urlWithPrefix${item.name}/${item.file.fileName}" - spanBuilder("download").setAttribute("name", item.name).setAttribute("url", url).use { + if (connection == null) { + connection = client.connect(effectiveServerUri.host, effectiveServerUri.port) + } + try { + val errors = CopyOnWriteArrayList() + toDownload.forEachConcurrent(downloadParallelism) { item -> + val urlPath = "$urlPathWithPrefix${item.name}/${item.file.fileName}" + spanBuilder("download").setAttribute("name", item.name).setAttribute("urlPath", urlPath).use { span -> try { downloadedBytes.getAndAdd( download( item = item, - url = url, - bufferPool = bufferPool, + urlPath = urlPath, skipUnpack = skipUnpack, saveHash = saveHash, - client = client, + connection = connection, ) ) } @@ -80,43 +83,41 @@ internal suspend fun downloadCompilationCache( throw e } catch (e: Throwable) { - return@use CompilePartDownloadFailedError(item, e) + span.recordException(e) + errors.add(CompilePartDownloadFailedError(item, e)) } - null } } - }.filterNotNull() + return errors + } + finally { + connection.close() + } } private suspend fun download( item: FetchAndUnpackItem, - url: String, - bufferPool: DirectFixedSizeByteBufferPool, + urlPath: String, skipUnpack: Boolean, saveHash: Boolean, - client: OkHttpClient, + connection: Http2ClientConnection, ): Long { - val downloaded = retryWithExponentialBackOff(onException = ::onDownloadException) { - client.newCall(Request.Builder().url(url).build()).executeAsync().useSuccessful { response -> - val digest = sha256() - writeFile(file = item.file, response = response, bufferPool = bufferPool, url = url, digest = digest) - val computedHash = digestToString(digest) - if (computedHash != item.hash) { - throw HashMismatchException("hash mismatch") { span, attempt -> - span.addEvent( - "hash mismatch", - Attributes.of( - AttributeKey.longKey("attemptNumber"), attempt.toLong(), - AttributeKey.stringKey("name"), item.file.name, - AttributeKey.stringKey("expected"), item.hash, - AttributeKey.stringKey("computed"), computedHash, - ) - ) - } - } - response.body.contentLength() - } + val (downloaded, digest) = connection.download(path = urlPath, file = item.file, digestFactory = { sha256() }) + val digestBytes = digest.digest() + val computedHash = BigInteger(1, digestBytes).toString(36) + "-z" + if (computedHash != item.hash) { + println("actualHash : ${computeHash(item.file)}") + println("expectedHash: ${item.hash}") + println("computedHash: $computedHash") + + val spanAttributes = Attributes.of( + AttributeKey.stringKey("name"), item.file.name, + AttributeKey.stringKey("expected"), item.hash, + AttributeKey.stringKey("computed"), computedHash, + ) + throw HashMismatchException("hash mismatch ($spanAttributes)") } + if (!skipUnpack) { spanBuilder("unpack").setAttribute("name", item.name).use { unpackArchive(item, saveHash) @@ -125,23 +126,9 @@ private suspend fun download( return downloaded } -private suspend fun onDownloadException(attempt: Int, e: Exception) { - spanBuilder("Retrying download with exponential back off").use { span -> - if (e is HashMismatchException) { - e.eventLogger.invoke(span, attempt) - } - else { - span.addEvent("Attempt failed", Attributes.of( - AttributeKey.longKey("attemptNumber"), attempt.toLong(), - AttributeKey.stringKey("error"), e.toString() - )) - } - } -} - internal class CompilePartDownloadFailedError(@JvmField val item: FetchAndUnpackItem, cause: Throwable) : RuntimeException(cause) -internal class HashMismatchException(message: String, @JvmField val eventLogger: (Span, Int) -> Unit) : IOException(message) +internal class HashMismatchException(message: String) : IOException(message) internal fun unpackArchive(item: FetchAndUnpackItem, saveHash: Boolean) { HashMapZipFile.load(item.file).use { zipFile -> @@ -170,66 +157,4 @@ internal fun unpackArchive(item: FetchAndUnpackItem, saveHash: Boolean) { // save actual hash Files.writeString(item.output.resolve(".hash"), item.hash) } -} - -private fun writeFile(file: Path, response: Response, bufferPool: DirectFixedSizeByteBufferPool, url: String, digest: MessageDigest) { - Files.createDirectories(file.parent) - FileChannel.open(file, OVERWRITE_OPERATION).use { channel -> - val source = response.body.source() - val sourceBuffer = bufferPool.allocate() - object : ZstdDirectBufferDecompressingStreamNoFinalizer(sourceBuffer) { - public override fun refill(toRefill: ByteBuffer): ByteBuffer { - toRefill.clear() - do { - if (source.read(toRefill) == -1) { - break - } - } - while (!source.exhausted() && toRefill.hasRemaining()) - toRefill.flip() - return toRefill - } - - override fun close() { - try { - super.close() - } - finally { - bufferPool.release(sourceBuffer) - } - } - }.use { decompressor -> - var offset = 0L - val targetBuffer = bufferPool.allocate() - try { - // refill is not called on start - decompressor.refill(sourceBuffer) - do { - do { - // decompressor can consume not the whole source buffer if target buffer size is not enough - decompressor.read(targetBuffer) - targetBuffer.flip() - - targetBuffer.mark() - digest.update(targetBuffer) - targetBuffer.reset() - - do { - offset += channel.write(targetBuffer, offset) - } - while (targetBuffer.hasRemaining()) - targetBuffer.clear() - } - while (sourceBuffer.hasRemaining()) - } - while (!source.exhausted()) - } - catch (e: IOException) { - throw IOException("Cannot unpack $url", e) - } - finally { - bufferPool.release(targetBuffer) - } - } - } } \ No newline at end of file diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/http.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/http.kt index e44322dee8b9..1007f0e72c08 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/http.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/http.kt @@ -4,15 +4,12 @@ package org.jetbrains.intellij.build.impl.compilation import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.* -import okhttp3.MediaType.Companion.toMediaType import okhttp3.internal.closeQuietly import org.jetbrains.intellij.build.NoMoreRetriesException import java.io.IOException import java.util.concurrent.TimeUnit import kotlin.coroutines.resumeWithException -internal val MEDIA_TYPE_BINARY = "application/octet-stream".toMediaType() - internal suspend fun OkHttpClient.head(url: String, authHeader: String): Int { return newCall(Request.Builder().url(url).head().header("Authorization", authHeader).build()).executeAsync().use { response -> if (response.code != 200 && response.code != 404) { @@ -43,6 +40,11 @@ internal val httpClient: OkHttpClient by lazy { .connectTimeout(timeout, unit) .writeTimeout(timeout, unit) .readTimeout(timeout, unit) + .dispatcher(Dispatcher().apply { + // we upload/download to the same host - increase `maxRequestsPerHost` + //maxRequestsPerHost = Runtime.getRuntime().availableProcessors().coerceIn(5, 16) + //... but in the same time it can increase the bill for ALB, so, leave it as is + }) .addInterceptor { chain -> var request = chain.request() if (request.header("User-Agent").isNullOrBlank()) { @@ -83,7 +85,7 @@ internal suspend fun Call.executeAsync(): Response { continuation.invokeOnCancellation { this.cancel() } - this.enqueue(object : Callback { + enqueue(object : Callback { override fun onFailure(call: Call, e: IOException) { continuation.resumeWithException(e) } diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/upload.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/upload.kt index 8f486eb57194..627faac3e6bc 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/upload.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/compilation/upload.kt @@ -4,47 +4,42 @@ package org.jetbrains.intellij.build.impl.compilation import com.github.luben.zstd.Zstd -import com.github.luben.zstd.ZstdDirectBufferCompressingStreamNoFinalizer -import com.intellij.platform.util.coroutines.forEachConcurrent +import com.github.luben.zstd.ZstdCompressCtx +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http2.Http2StreamChannel +import io.netty.util.AsciiString import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.trace.Span -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.decodeFromStream -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody -import okhttp3.RequestBody.Companion.toRequestBody -import okio.BufferedSink -import okio.use +import org.jetbrains.intellij.build.forEachConcurrent +import org.jetbrains.intellij.build.http2Client.Http2ClientConnection +import org.jetbrains.intellij.build.http2Client.MAX_BUFFER_SIZE +import org.jetbrains.intellij.build.http2Client.writeData +import org.jetbrains.intellij.build.io.unmapBuffer import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder import org.jetbrains.intellij.build.telemetry.use -import java.nio.ByteBuffer +import java.nio.MappedByteBuffer import java.nio.channels.FileChannel import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption import java.util.* +import java.util.concurrent.CancellationException import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicLong +import kotlin.math.min -private val MEDIA_TYPE_JSON = "application/json".toMediaType() internal val READ_OPERATION = EnumSet.of(StandardOpenOption.READ) -internal const val MAX_BUFFER_SIZE = 4 * 1014 * 1024 -internal const val ZSTD_LEVEL = 3 internal suspend fun uploadArchives( reportStatisticValue: (key: String, value: String) -> Unit, config: CompilationCacheUploadConfiguration, metadataJson: String, - httpClient: OkHttpClient, + httpConnection: Http2ClientConnection, items: List, - bufferPool: DirectFixedSizeByteBufferPool, ) { val uploadedCount = AtomicInteger() val uploadedBytes = AtomicLong() @@ -56,20 +51,25 @@ internal suspend fun uploadArchives( val alreadyUploaded: Set = try { if (config.checkFiles) { spanBuilder("fetch info about already uploaded files").use { - HashSet(getFoundAndMissingFiles(metadataJson, config.serverUrl, httpClient).found) + getFoundAndMissingFiles(metadataJson = metadataJson, urlPathPrefix = config.serverUrlPathPrefix, connection = httpConnection).found } } else { emptySet() } } + catch (e: CancellationException) { + throw e + } catch (e: Throwable) { Span.current().recordException(e, Attributes.of(AttributeKey.stringKey("message"), "failed to fetch info about already uploaded files, will fallback to HEAD requests")) fallbackToHeads = true emptySet() } - withContext(Dispatchers.IO) { + val sourceBlockSize = MAX_BUFFER_SIZE + val urlPathPrefix = "${config.serverUrlPathPrefix}/${config.uploadUrlPathPrefix}" + ZstdCompressContextPool().use { zstdCompressContextPool -> items.forEachConcurrent(uploadParallelism) { item -> if (alreadyUploaded.contains(item.name)) { reusedCount.getAndIncrement() @@ -77,16 +77,18 @@ internal suspend fun uploadArchives( return@forEachConcurrent } + val urlPath = "$urlPathPrefix/${item.name}/${item.hash!!}.jar" spanBuilder("upload archive").setAttribute("name", item.name).setAttribute("hash", item.hash!!).use { val size = Files.size(item.archive) val isUploaded = uploadFile( - url = "${config.serverUrl}/${config.uploadPrefix}/${item.name}/${item.hash!!}.jar", + urlPath = urlPath, file = item.archive, useHead = fallbackToHeads, span = Span.current(), - httpClient = httpClient, - bufferPool = bufferPool, + httpSession = httpConnection, fileSize = size, + sourceBlockSize = sourceBlockSize, + zstdCompressContextPool = zstdCompressContextPool, ) if (isUploaded) { uploadedCount.getAndIncrement() @@ -125,172 +127,100 @@ internal suspend fun uploadArchives( reportStatisticValue("compile-parts:total:count", (reusedCount.get() + uploadedCount.get()).toString()) } -private suspend fun getFoundAndMissingFiles(metadataJson: String, serverUrl: String, httpClient: OkHttpClient): CheckFilesResponse { - httpClient.newCall(Request.Builder() - .url("$serverUrl/check-files") - .post(metadataJson.toRequestBody(MEDIA_TYPE_JSON)) - .build()).executeAsync().useSuccessful { - return Json.decodeFromStream(it.body.byteStream()) - } +private suspend fun getFoundAndMissingFiles(metadataJson: String, urlPathPrefix: String, connection: Http2ClientConnection): CheckFilesResponse { + return connection.post(path = "$urlPathPrefix/check-files", data = metadataJson, contentType = HttpHeaderValues.APPLICATION_JSON) } // Using ZSTD dictionary doesn't make the difference, even slightly worse (default compression level 3). // That's because in our case, we compress a relatively large archive of class files. private suspend fun uploadFile( - url: String, + urlPath: String, file: Path, useHead: Boolean, span: Span, - httpClient: OkHttpClient, - bufferPool: DirectFixedSizeByteBufferPool, + httpSession: Http2ClientConnection, fileSize: Long, + sourceBlockSize: Int, + zstdCompressContextPool: ZstdCompressContextPool, ): Boolean { if (useHead) { - val request = Request.Builder().url(url).head().build() - val code = httpClient.newCall(request).executeAsync().use { - it.code + val status = httpSession.head(urlPath) + if (status == HttpResponseStatus.OK) { + span.addEvent("already exist on server, nothing to upload", Attributes.of(AttributeKey.stringKey("urlPath"), urlPath)) + return false } - - when { - code == 200 -> { - span.addEvent("already exist on server, nothing to upload", Attributes.of(AttributeKey.stringKey("url"), url)) - return false - } - code != 404 -> { - span.addEvent("responded with unexpected", Attributes.of( - AttributeKey.longKey("code"), code.toLong(), - AttributeKey.stringKey("url"), url, - )) - } + else if (status != HttpResponseStatus.NOT_FOUND) { + span.addEvent( + "responded with unexpected", + Attributes.of( + AttributeKey.stringKey("status"), status.toString(), + AttributeKey.stringKey("urlPath"), urlPath + ), + ) } } - if (Zstd.compressBound(fileSize) <= MAX_BUFFER_SIZE) { - compressSmallFile(file = file, fileSize = fileSize, bufferPool = bufferPool, url = url) - } - else { - val request = Request.Builder() - .url(url) - .put(object : RequestBody() { - override fun contentType() = MEDIA_TYPE_BINARY + require(fileSize > 0) - override fun writeTo(sink: BufferedSink) { - compressFile(file = file, output = sink, bufferPool = bufferPool) - } - }) - .build() - - httpClient.newCall(request).executeAsync().useSuccessful { } + val fileBuffer = FileChannel.open(file, READ_OPERATION).use { channel -> + channel.map(FileChannel.MapMode.READ_ONLY, 0, fileSize) } + try { + zstdCompressContextPool.withZstd { zstd -> + httpSession.put(AsciiString.of(urlPath)) { stream -> + compressAndUpload( + fileSize = fileSize, + fileBuffer = fileBuffer, + sourceBlockSize = sourceBlockSize, + zstd = zstd, + stream = stream, + ) + } + } + } + finally { + unmapBuffer(fileBuffer) + } return true } -private suspend fun compressSmallFile(file: Path, fileSize: Long, bufferPool: DirectFixedSizeByteBufferPool, url: String) { - val targetBuffer = bufferPool.allocate() - try { - var readOffset = 0L - val sourceBuffer = bufferPool.allocate() - try { - FileChannel.open(file, READ_OPERATION).use { input -> - do { - readOffset += input.read(sourceBuffer, readOffset) - } - while (readOffset < fileSize) - } - sourceBuffer.flip() +private suspend fun compressAndUpload( + fileSize: Long, + fileBuffer: MappedByteBuffer, + sourceBlockSize: Int, + zstd: ZstdCompressCtx, + stream: Http2StreamChannel, +) { + var position = 0 + while (true) { + val chunkSize = min(fileSize - position, sourceBlockSize.toLong()).toInt() + val targetSize = Zstd.compressBound(chunkSize.toLong()).toInt() + val targetNettyBuffer = stream.alloc().directBuffer(targetSize) + val targetBuffer = targetNettyBuffer.nioBuffer(0, targetSize) + val compressedSize = zstd.compressDirectByteBuffer( + targetBuffer, // compress into targetBuffer + targetBuffer.position(), // write compressed data starting at offset position() + targetSize, // write no more than target block size bytes + fileBuffer, // read data to compress from fileBuffer + position, // start reading at position() + chunkSize, // read chunk size bytes + ) + assert(compressedSize > 0) + targetNettyBuffer.writerIndex(targetNettyBuffer.writerIndex() + compressedSize) + assert(targetNettyBuffer.readableBytes() == compressedSize) - Zstd.compress(targetBuffer, sourceBuffer, ZSTD_LEVEL, false) - targetBuffer.flip() - } - finally { - bufferPool.release(sourceBuffer) - } + position += chunkSize - val compressedSize = targetBuffer.remaining() - - val request = Request.Builder() - .url(url) - .put(object : RequestBody() { - override fun contentLength() = compressedSize.toLong() - - override fun contentType() = MEDIA_TYPE_BINARY - - override fun writeTo(sink: BufferedSink) { - targetBuffer.mark() - sink.write(targetBuffer) - targetBuffer.reset() - } - }) - .build() - - httpClient.newCall(request).executeAsync().useSuccessful { } - } - finally { - bufferPool.release(targetBuffer) - } -} - -private fun compressFile(file: Path, output: BufferedSink, bufferPool: DirectFixedSizeByteBufferPool) { - val targetBuffer = bufferPool.allocate() - CompilationCacheZstdCompressingStream(targetBuffer = targetBuffer, output = output, bufferPool = bufferPool).use { compressor -> - val sourceBuffer = bufferPool.allocate() - try { - var offset = 0L - FileChannel.open(file, READ_OPERATION).use { input -> - val fileSize = input.size() - while (offset < fileSize) { - val actualBlockSize = (fileSize - offset).toInt() - if (sourceBuffer.remaining() > actualBlockSize) { - sourceBuffer.limit(sourceBuffer.position() + actualBlockSize) - } - - var readOffset = offset - do { - readOffset += input.read(sourceBuffer, readOffset) - } - while (sourceBuffer.hasRemaining()) - - sourceBuffer.flip() - compressor.compress(sourceBuffer) - - sourceBuffer.clear() - offset = readOffset - } - } - } - finally { - bufferPool.release(sourceBuffer) - } - } -} - -private class CompilationCacheZstdCompressingStream( - private val targetBuffer: ByteBuffer, - private val output: BufferedSink, - private val bufferPool: DirectFixedSizeByteBufferPool, -) : ZstdDirectBufferCompressingStreamNoFinalizer(targetBuffer, ZSTD_LEVEL) { - override fun flushBuffer(toFlush: ByteBuffer): ByteBuffer { - toFlush.flip() - while (toFlush.hasRemaining()) { - output.write(toFlush) - } - toFlush.clear() - return toFlush - } - - override fun close() { - try { - super.close() - } - finally { - bufferPool.release(targetBuffer) + val endStream = position >= fileSize + stream.writeData(targetNettyBuffer, endStream) + if (endStream) { + break } } } @Serializable private data class CheckFilesResponse( - @JvmField val found: List = emptyList(), - @JvmField val missing: List = emptyList(), -) \ No newline at end of file + @JvmField val found: HashSet = HashSet(), +) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sbom/SoftwareBillOfMaterialsImpl.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sbom/SoftwareBillOfMaterialsImpl.kt index fbc12f3c1ddc..3c8373a01e68 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sbom/SoftwareBillOfMaterialsImpl.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/sbom/SoftwareBillOfMaterialsImpl.kt @@ -4,10 +4,8 @@ package org.jetbrains.intellij.build.impl.sbom import com.intellij.openapi.util.SystemInfoRt -import com.intellij.platform.util.coroutines.forEachConcurrent import com.intellij.util.io.DigestUtil import com.intellij.util.io.DigestUtil.sha1Hex -import com.intellij.util.io.DigestUtil.updateContentHash import com.intellij.util.io.bytesToHex import com.intellij.util.io.sha256Hex import com.jetbrains.plugin.structure.base.utils.exists diff --git a/platform/build-scripts/tests/nginx-webdav.conf b/platform/build-scripts/tests/nginx-webdav.conf index 84d61463def0..fceaa3f888a4 100644 --- a/platform/build-scripts/tests/nginx-webdav.conf +++ b/platform/build-scripts/tests/nginx-webdav.conf @@ -1,19 +1,35 @@ +daemon off; worker_processes auto; events { - worker_connections 1024; } http { + log_format main '[$time_local] "$request" $status $body_bytes_sent '; + access_log /dev/stdout main; + error_log /dev/stderr; + + gzip on; + gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json application/javascript text/x-js; server { - listen 1900; + listen 127.0.0.1:1900 ssl; + http2 on; + server_name 127.0.0.1; + ssl_certificate ./server.crt; + ssl_certificate_key ./server.key; location / { root /tmp/webdav; client_max_body_size 0; create_full_put_path on; - dav_methods PUT DELETE MKCOL COPY MOVE; + dav_methods PUT; autoindex on; } + + location /check-files { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_pass http://127.0.0.1:8082/; + } } } \ No newline at end of file diff --git a/platform/build-scripts/tests/server.crt b/platform/build-scripts/tests/server.crt new file mode 100644 index 000000000000..2d210eb7e1e2 --- /dev/null +++ b/platform/build-scripts/tests/server.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhjCCAm6gAwIBAgIJAJN8H+8liHX0MA0GCSqGSIb3DQEBCwUAMHgxCzAJBgNV +BAYTAlhYMQwwCgYDVQQIDANOL0ExDDAKBgNVBAcMA04vQTEgMB4GA1UECgwXU2Vs +Zi1zaWduZWQgY2VydGlmaWNhdGUxKzApBgNVBAMMIjEyNy4wLjAuMTogU2VsZi1z +aWduZWQgY2VydGlmaWNhdGUwHhcNMjQwODI2MTEyMzEzWhcNMjYwODI2MTEyMzEz +WjB4MQswCQYDVQQGEwJYWDEMMAoGA1UECAwDTi9BMQwwCgYDVQQHDANOL0ExIDAe +BgNVBAoMF1NlbGYtc2lnbmVkIGNlcnRpZmljYXRlMSswKQYDVQQDDCIxMjcuMC4w +LjE6IFNlbGYtc2lnbmVkIGNlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAvRyBeGto8nsEMkr14NK+g9hIyFdiA+j+m1fQ4D/y+zI+N+7Z +2NgVTK5TJlxoZkmjbfqZ4Dmv9nqL/OQ8LF3Jw+gkInEhX6lb/NX+IbywOSzMpkuu +pIzGN0UWhhfb3oF3YfD1HTSrpEcpOP8VeUGRt6XNNK2XgcrFAuQVoSNQktdCOJuF +VC20dYBT2ngk02uW+6Vs0/q4rMpMGalRTOel1aOLJrlCocITe/iUUOwoTugcCaro +T+7hKofSrNLI+vwN9s8H/YGj2QyssfhBqHOsah080Be5jkRP7TolrIlIY72XDe9b +VA/vfHoJ1ry6n4aHNouHU1BC8Kyy416/wPCZJwIDAQABoxMwETAPBgNVHREECDAG +hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBZ+EbXZyIvUGhSdxqbRpUSUX+gVu4h +QlgVNlk0w7s+SnSzru3MLXYV0KW/nbLmW1EUe2zaAbWHPS85VvsZszYlBtOzqUh3 +6uyEXmGr9bq4laMo7mK6pDGxsI72Xl4X3Vm/gUw547z5+gh4/ggzer+lQrOPgLRJ +vhXwCSD26wB2qx4pgQAV0oGtC8vcm7hSkP2FJH1HveOvivdwDQfeo5+pc8YJvsUG +pZK/2QqZY2QkQ4iv1rreaxWv3p7W2EVkioyjxbgCZBglxTC8RwPwH4Kxhs+8tXzi +E2AUE5ryQCwGbQw6dElJvIFY9TpgPPEeCu8l/JzJxxkkR1TNK58rH8rN +-----END CERTIFICATE----- \ No newline at end of file diff --git a/platform/build-scripts/tests/server.key b/platform/build-scripts/tests/server.key new file mode 100644 index 000000000000..04f53d79ac90 --- /dev/null +++ b/platform/build-scripts/tests/server.key @@ -0,0 +1,29 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC9HIF4a2jyewQy +SvXg0r6D2EjIV2ID6P6bV9DgP/L7Mj437tnY2BVMrlMmXGhmSaNt+pngOa/2eov8 +5DwsXcnD6CQicSFfqVv81f4hvLA5LMymS66kjMY3RRaGF9vegXdh8PUdNKukRyk4 +/xV5QZG3pc00rZeBysUC5BWhI1CS10I4m4VULbR1gFPaeCTTa5b7pWzT+risykwZ +qVFM56XVo4smuUKhwhN7+JRQ7ChO6BwJquhP7uEqh9Ks0sj6/A32zwf9gaPZDKyx ++EGoc6xqHTzQF7mORE/tOiWsiUhjvZcN71tUD+98egnWvLqfhoc2i4dTUELwrLLj +Xr/A8JknAgMBAAECggEALSaEJtsGKHaEbvmEsNPAFrxpzCNIzIQxXadewFukSKMb +RcFqE6Krmy43vf3sExfbxCND38wGHgPuLkfTsggGZxaiofJ+tFc8FiaFUUq6jDwM +9Fs3bCQMIyAEm6lQnlQsy5569ykfHc67odcNKnEkOEOGteAIPz3JQcJxA5Lp5tTC +HmyZOcrbFaDSpn1h/HXsq3OIP98TCT2KYOhFuWy0JQTr9uvZDuq8lqoZ53fMNjj/ +0RwesJdHpeRb7GnWW66GQrOmMCFZkKqWOLUK8tcwCYfo9w5GV9BC8sSUx3RF4f1H +6oO8v3caScOChrznnx7b5KnRkS7O0UUxuTt1+h1HOQKBgQD5fBDSfm+YvMJyAP0Z +bmIdqqchMyW6xlF1tfhpfLkgFj1ufpIE/G8q+ZUVfah+TMhPUAyAc3AznYHCiHVe +suu1QJ7tIuBkRfWZgdLo8XdREhpKnm8xAUXOs43t6/CxmPTao+ylN0WGmUHVgQae +OHcpftNN3zLio0rNm1xOlmdDSwKBgQDCDNA0pxzOvELHHFCMG+N7Sjdy/HK2h1LW +UQ/FvP/qx/iPJ0fyW0LZkLF2Y9nc7FxVqiA/5W2sy6MQ3ZuOzpobVobeb8SC3fBm +Q6TSCIKd69E7vLCb9BMfnYPYqW2WP3fEYp8A3MXMyjDJxZSharNeP7/YSSk85VfE +ruq/ege8FQKBgBnslGrrDHmYk7P4+lPcLoHaq8c9Y1xHI0vR/uAnP61f4j5LFK1D +9eFHUgCLsCh/ngjvznzCghQ697LZLykJ+og5EMqfZyXER0MORHZEMRvRf73lPLSg +5zoVWlgwvjAWLstRYVPBrI3R+w9OevuR7n/3V8mtucHnKey3ih34bv6FAoGABtxH +HCVwWkrDnaB9pIZz4277SOBt+dAM+LDC+v20motZWU5NN99MHL8F1yaulCXzGcA7 +BadJ2lsUt8rt7f2V6zOC7yhKbUoFbsgjcp2EaKrmqdMA93KInoyGFcnfqvkxdcr6 +ziAACj53vRp0J8TK9KESWkYz5AhDsxtwBzb8QQUCgYAawMvTQRDPGwMLtoSBEyDt +j7ytvxzWPdMhf9/d+xT6wtyfVHnc1CSWermfmN3BXTHTBskTeAXVA8l2sjM5lMOG +obekKJy31dNz+4ysu1ZiwspxS22iNLgii6L2Qy7u6I16Kb2ZHhmVMwSVcTABzNxS +//ymT1boXSvNKgTWwe30qg== +-----END PRIVATE KEY----- + diff --git a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/CompilationCacheTest.kt b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/CompilationCacheTest.kt index 0f2a06b65375..dd653c369cd6 100644 --- a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/CompilationCacheTest.kt +++ b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/CompilationCacheTest.kt @@ -1,10 +1,11 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.intellij.build +import com.intellij.testFramework.utils.io.deleteRecursively +import com.intellij.util.SystemProperties import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking import org.jetbrains.intellij.build.impl.compilation.fetchAndUnpackCompiledClasses -import org.jetbrains.intellij.build.io.deleteDir import org.jetbrains.intellij.build.telemetry.TraceManager import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.Assumptions.assumeTrue @@ -25,7 +26,7 @@ class CompilationCacheTest { @Test fun testUnpack() = runBlocking(Dispatchers.Default) { - val metadataFile = Path.of("/Volumes/data/Documents/idea/out/compilation-archive/metadata.json") + val metadataFile = Path.of(SystemProperties.getUserHome(), "projects/idea/out/compilation-archive/metadata.json") assumeTrue(Files.exists(metadataFile)) // do not use Junit TempDir - it is very slow @@ -41,10 +42,7 @@ class CompilationCacheTest { ) } finally { - Files.list(outDir).parallel().use { stream -> - stream.forEach(::deleteDir) - } - Files.delete(outDir) + outDir.deleteRecursively() } } } \ No newline at end of file