IJI-1629 build scripts: Maven Central publication

(cherry picked from commit 665265ed77e22e6ef4c3bb7e88a9c9da26a2037a)

IJ-MR-159792

GitOrigin-RevId: 7f5eddfaabe5eae7a413370500e55cba558738fa

platform/build-scripts/src/org/jetbrains/intellij/build/ProprietaryBuildTools.kt

@@ -5,6 +5,7 @@ import io.opentelemetry.api.common.AttributeKey
 import io.opentelemetry.api.common.Attributes
 import io.opentelemetry.api.trace.Span
 import kotlinx.collections.immutable.PersistentMap
+import kotlinx.collections.immutable.persistentMapOf
 import org.jetbrains.intellij.build.fus.FeatureUsageStatisticsProperties
 import java.nio.file.Path
 
@@ -58,8 +59,12 @@ data class ProprietaryBuildTools(
         )
       }
 
+      override suspend fun signFilesWithGpg(files: List<Path>, context: BuildContext) {
+        signFiles(files, context, persistentMapOf())
+      }
+
       override suspend fun getPresignedLibraryFile(path: String, libName: String, libVersion: String, context: BuildContext): Path? {
-        error("Must be not called if signNativeFileMode equals to ENABLED")
+        error("Must be not called if signNativeFileMode equals to $signNativeFileMode")
       }
 
       override suspend fun commandLineClient(context: BuildContext, os: OsFamily, arch: JvmArchitecture): Path? {

platform/build-scripts/src/org/jetbrains/intellij/build/SignTool.kt

@@ -16,6 +16,8 @@ interface SignTool {
 
   suspend fun signFiles(files: List<Path>, context: BuildContext?, options: PersistentMap<String, String>)
 
+  suspend fun signFilesWithGpg(files: List<Path>, context: BuildContext)
+
   /**
    * Returns `null` if failed to download and error is not fatal.
    */

platform/build-scripts/src/org/jetbrains/intellij/build/impl/Checksums.kt

@@ -0,0 +1,38 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build.impl
+
+import com.intellij.util.io.DigestUtil.sha1
+import com.intellij.util.io.DigestUtil.sha256
+import com.intellij.util.io.bytesToHex
+import org.jetbrains.annotations.ApiStatus
+import java.nio.file.Files
+import java.nio.file.Path
+import java.security.MessageDigest
+
+@ApiStatus.Internal
+class Checksums(val path: Path, vararg algorithms: MessageDigest = arrayOf(sha1(), sha256())) {
+  private val results: Map<String, String>
+  val sha1sum: String get() = results.getValue("SHA-1")
+  val sha256sum: String get() = results.getValue("SHA-256")
+  val sha512sum: String get() = results.getValue("SHA-512")
+  val md5sum: String get() = results.getValue("MD5")
+
+  init {
+    require(algorithms.any())
+    val buffer = ByteArray(512 * 1024)
+    Files.newInputStream(path).use {
+      while (true) {
+        val sz = it.read(buffer)
+        if (sz <= 0) {
+          break
+        }
+        for (algorithm in algorithms) {
+          algorithm.update(buffer, 0, sz)
+        }
+      }
+      results = algorithms.associate {
+        it.algorithm to bytesToHex(it.digest())
+      }
+    }
+  }
+}

platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/IntellijModulesPublication.kt

@@ -97,8 +97,8 @@ class IntellijModulesPublication(
 
   private fun deployModuleArtifact(coordinates: MavenCoordinates) {
     val dir = options.outputDir.resolve(coordinates.directoryPath).toFile()
-    val pom = File(dir, coordinates.getFileName("", "pom"))
-    val jar = File(dir, coordinates.getFileName("", "jar"))
+    val pom = File(dir, coordinates.getFileName(packaging = "pom"))
+    val jar = File(dir, coordinates.getFileName(packaging = "jar"))
     val sources = File(dir, coordinates.getFileName("sources", "jar"))
     if (jar.exists()) {
       deployJar(jar, pom, coordinates)

platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenArtifactsBuilder.kt

@@ -298,7 +298,7 @@ data class MavenCoordinates(
   val directoryPath: String
     get() = "${groupId.replace('.', '/')}/$artifactId/$version"
 
-  fun getFileName(classifier: String, packaging: String): String {
+  fun getFileName(classifier: String = "", packaging: String): String {
     return "$artifactId-$version${if (classifier.isEmpty()) "" else "-$classifier"}.$packaging"
   }
 }
@@ -405,10 +405,10 @@ private suspend fun layoutMavenArtifacts(modulesToPublish: Map<MavenArtifactData
         Files.createDirectories(artifactDir)
 
         generatePomXmlData(artifactData = artifactData,
-                           file = artifactDir.resolve(artifactData.coordinates.getFileName("", "pom")))
+                           file = artifactDir.resolve(artifactData.coordinates.getFileName(packaging = "pom")))
 
         buildJar(
-          targetFile = artifactDir.resolve(artifactData.coordinates.getFileName("", "jar")),
+          targetFile = artifactDir.resolve(artifactData.coordinates.getFileName(packaging = "jar")),
           sources = modulesWithSources.map {
             DirSource(dir = context.getModuleOutputDir(it), excludes = commonModuleExcludes)
           },

platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenCentralPublication.kt

@@ -0,0 +1,207 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build.impl.maven
+
+import com.intellij.util.io.Compressor
+import com.intellij.util.io.DigestUtil.md5
+import com.intellij.util.io.DigestUtil.sha1
+import com.intellij.util.io.DigestUtil.sha256
+import com.intellij.util.io.DigestUtil.sha512
+import com.intellij.util.xml.dom.readXmlAsModel
+import kotlinx.coroutines.CoroutineName
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.launch
+import okhttp3.MultipartBody
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.asRequestBody
+import org.jetbrains.annotations.ApiStatus
+import org.jetbrains.intellij.build.BuildContext
+import org.jetbrains.intellij.build.impl.Checksums
+import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder
+import org.jetbrains.intellij.build.telemetry.use
+import java.nio.file.Path
+import java.util.*
+import kotlin.io.path.*
+
+/**
+ * @param workDir is expected to contain:
+ * * [jar]
+ * * [pom]
+ * * [javadoc] jar
+ * * [sources] jar
+ * * md5, sha1, sha256 and sha512 checksum files (optional, will be verified if present)
+ *
+ * @param type https://central.sonatype.org/publish/publish-portal-api/#uploading-a-deployment-bundle
+ */
+@ApiStatus.Internal
+class MavenCentralPublication(
+  private val context: BuildContext,
+  private val workDir: Path,
+  private val type: Type = Type.USER_MANAGED,
+  private val userName: String? = null,
+  private val token: String? = null,
+  private val dryRun: Boolean = context.options.isInDevelopmentMode,
+  private val sign: Boolean = !dryRun,
+) {
+  private companion object {
+    const val URI_BASE = "https://central.sonatype.com/api/v1/publisher/upload"
+  }
+
+  enum class Type {
+    USER_MANAGED,
+    AUTOMATIC,
+  }
+
+  private lateinit var jar: Path
+  private lateinit var pom: Path
+  private lateinit var javadoc: Path
+  private lateinit var sources: Path
+  private lateinit var coordinates: MavenCoordinates
+
+  private fun requireFile(name: String): Path {
+    val matchingFiles = workDir.listDirectoryEntries(name)
+    return requireNotNull(matchingFiles.singleOrNull()) {
+      "A single $name file is expected to be present in $workDir but found: $matchingFiles"
+    }
+  }
+
+  private fun bootstrap() {
+    pom = requireFile("*.pom")
+    val project = readXmlAsModel(pom)
+    require(project.name == "project") {
+      "$pom doesn't contain <project> root element"
+    }
+    coordinates = MavenCoordinates(
+      groupId = project.getChild("groupId")?.content ?: error("$pom doesn't contain <groupId> element"),
+      artifactId = project.getChild("artifactId")?.content ?: error("$pom doesn't contain <artifactId> element"),
+      version = project.getChild("version")?.content ?: error("$pom doesn't contain <version> element"),
+    )
+    jar = requireFile(coordinates.getFileName(packaging = "jar"))
+    sources = requireFile(coordinates.getFileName(classifier = "sources", packaging = "jar"))
+    javadoc = requireFile(coordinates.getFileName(classifier = "javadoc", packaging = "jar"))
+  }
+
+  suspend fun execute() {
+    bootstrap()
+    sign()
+    generateOrVerifyChecksums()
+    publish(bundle())
+  }
+
+  private val distributionFiles: List<Path> get() = listOf(jar, pom, javadoc, sources)
+
+  private suspend fun sign() {
+    if (sign) {
+      context.proprietaryBuildTools.signTool.signFilesWithGpg(distributionFiles, context)
+    }
+  }
+
+  private fun signatures(): List<Path> {
+    if (!sign) return emptyList()
+    val signatures = workDir.listDirectoryEntries(glob = "*.asc")
+    check(signatures.any()) {
+      "Missing .asc signatures"
+    }
+    return signatures
+  }
+
+  private suspend fun generateOrVerifyChecksums() {
+    coroutineScope {
+      for (file in distributionFiles.asSequence().plus(signatures())) {
+        launch(CoroutineName("checksums for $file")) {
+          val checksums = Checksums(file, sha1(), sha256(), sha512(), md5())
+          generateOrVerifyChecksum(file, extension = "sha1", checksums.sha1sum)
+          generateOrVerifyChecksum(file, extension = "sha256", checksums.sha256sum)
+          generateOrVerifyChecksum(file, extension = "sha512", checksums.sha512sum)
+          generateOrVerifyChecksum(file, extension = "md5", checksums.md5sum)
+        }
+      }
+    }
+  }
+
+  class ChecksumMismatch(message: String) : RuntimeException(message)
+
+  private fun CoroutineScope.generateOrVerifyChecksum(file: Path, extension: String, value: String) {
+    launch(CoroutineName("checksum $extension for $file")) {
+      spanBuilder("checksum").setAttribute("file", "$file").setAttribute("extension", extension).use {
+        val checksumFile = file.resolveSibling("${file.fileName}.$extension")
+        if (checksumFile.exists()) {
+          val suppliedValue = checksumFile.readLines().asSequence()
+            .flatMap { it.splitToSequence(" ") }
+            .firstOrNull()
+          if (suppliedValue != value) {
+            throw ChecksumMismatch("The supplied file $checksumFile content mismatch: '$suppliedValue' != '$value'")
+          }
+        }
+        else {
+          checksumFile.writeText(value)
+        }
+      }
+    }
+  }
+
+  private fun checksums(extension: String): List<Path> {
+    val signatures = workDir.listDirectoryEntries(glob = "*.$extension")
+    check(signatures.any()) {
+      "Missing .$extension checksums"
+    }
+    return signatures
+  }
+
+  private fun checksums(): List<Path> {
+    return checksums("md5") +
+           checksums("sha1") +
+           checksums("sha256") +
+           checksums("sha512")
+  }
+
+  private suspend fun bundle(): Path {
+    return spanBuilder("creating a bundle").use {
+      val bundle = workDir.resolve("bundle.zip")
+      Compressor.Zip(bundle).use {
+        for (file in distributionFiles.asSequence() + signatures() + checksums()) {
+          it.addFile(file.name, file)
+        }
+      }
+      bundle
+    }
+  }
+
+  private suspend fun publish(bundle: Path) {
+    spanBuilder("publishing").setAttribute("bundle", "$bundle").use {
+      if (dryRun) {
+        it.addEvent("skipped in the dryRun mode")
+        return@use
+      }
+      requireNotNull(userName) {
+        "Please specify intellij.build.mavenCentral.userName system property"
+      }
+      requireNotNull(token) {
+        "Please specify intellij.build.mavenCentral.token system property"
+      }
+      val deploymentName = "${coordinates.artifactId}-${coordinates.version}"
+      val uri = "$URI_BASE?name=$deploymentName&publicationType=$type"
+      val base64Auth = Base64.getEncoder().encode("$userName:$token".toByteArray()).toString(Charsets.UTF_8)
+      it.addEvent("Sending request to $uri...")
+      val client = OkHttpClient()
+      val request = Request.Builder()
+        .url(uri)
+        .header("Authorization", "Bearer $base64Auth")
+        .post(
+          MultipartBody.Builder()
+            .setType(MultipartBody.FORM)
+            .addFormDataPart("bundle", bundle.name, bundle.toFile().asRequestBody())
+            .build()
+        ).build()
+      client.newCall(request).execute().use { response ->
+        val statusCode = response.code
+        it.addEvent("Upload status code: $statusCode")
+        it.addEvent("Upload response: ${response.body.string()}")
+        check(statusCode == 201) {
+          "Unable to upload to Central repository, status code: $statusCode, upload response: ${response.body.string()}"
+        }
+      }
+    }
+  }
+}

platform/build-scripts/src/org/jetbrains/intellij/build/impl/sbom/SoftwareBillOfMaterialsImpl.kt

@@ -4,9 +4,7 @@
 package org.jetbrains.intellij.build.impl.sbom
 
 import com.intellij.openapi.util.SystemInfoRt
-import com.intellij.util.io.DigestUtil
 import com.intellij.util.io.DigestUtil.sha1Hex
-import com.intellij.util.io.bytesToHex
 import com.intellij.util.io.sha256Hex
 import com.jetbrains.plugin.structure.base.utils.exists
 import com.jetbrains.plugin.structure.base.utils.outputStream
@@ -187,30 +185,6 @@ class SoftwareBillOfMaterialsImpl(
     }
   }
 
-  class Checksums(@JvmField val path: Path) {
-    val sha1sum: String
-    val sha256sum: String
-
-    init {
-      val buffer = ByteArray(512 * 1024)
-      val digests = Files.newInputStream(path).use {
-        val sha1 = DigestUtil.sha1()
-        val sha256 = DigestUtil.sha256()
-        while (true) {
-          val sz = it.read(buffer)
-          if (sz <= 0) {
-            break
-          }
-          sha1.update(buffer, 0, sz)
-          sha256.update(buffer, 0, sz)
-        }
-        bytesToHex(sha1.digest()) to bytesToHex(sha256.digest())
-      }
-      sha1sum = digests.first
-      sha256sum = digests.second
-    }
-  }
-
   private suspend fun generateFromDistributions(): List<Path> {
     return withContext(Dispatchers.IO) {
       distributions.associateWith { distribution ->
@@ -524,14 +498,14 @@ class SoftwareBillOfMaterialsImpl(
       ?: error("Unknown jar repository ID: ${mavenDescriptor.jarRepositoryId}")
     }
     else null
-    val libraryName = coordinates.getFileName(packaging = mavenDescriptor.packaging, classifier = "")
+    val libraryName = coordinates.getFileName(packaging = mavenDescriptor.packaging)
     val checksums = mavenDescriptor.artifactsVerification.filter {
       Path.of(JpsPathUtil.urlToOsPath(it.url)).name == libraryName
     }
     check(checksums.count() == 1) {
       "Missing checksum for $coordinates: ${checksums.map { it.url }}"
     }
-    val pomName = coordinates.getFileName(packaging = "pom", classifier = "")
+    val pomName = coordinates.getFileName(packaging = "pom")
     return MavenLibrary(
       path = libraryFile,
       coordinates = coordinates,
@@ -820,7 +794,7 @@ class SoftwareBillOfMaterialsImpl(
       val repositoryUrl = checkNotNull(upstream.mavenRepositoryUrl) {
         "Missing Maven repository url for ${upstream.groupId}:${upstream.artifactId}"
       }.removeSuffix("/")
-      val jarName = coordinates.getFileName(packaging = "jar", classifier = "")
+      val jarName = coordinates.getFileName(packaging = "jar")
       setDownloadLocation("$repositoryUrl/${coordinates.directoryPath}/$jarName")
       addExternalRef(coordinates.externalRef(this@spdxPackageUpstream, repositoryUrl))
     }

platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt

@@ -0,0 +1,101 @@
+// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+package org.jetbrains.intellij.build.impl.maven
+
+import com.intellij.testFramework.utils.io.createFile
+import kotlinx.coroutines.runBlocking
+import org.jetbrains.intellij.build.BuildContext
+import org.jetbrains.intellij.build.BuildPaths.Companion.COMMUNITY_ROOT
+import org.jetbrains.intellij.build.IdeaCommunityProperties
+import org.jetbrains.intellij.build.impl.BuildContextImpl
+import org.jetbrains.intellij.build.io.suspendAwareReadZipFile
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Path
+import kotlin.io.path.exists
+import kotlin.io.path.name
+import kotlin.io.path.writeText
+
+class MavenCentralPublicationTest {
+  companion object {
+    val context: BuildContext by lazy {
+      runBlocking {
+        BuildContextImpl.createContext(
+          COMMUNITY_ROOT.communityRoot,
+          IdeaCommunityProperties(COMMUNITY_ROOT.communityRoot),
+          setupTracer = false,
+        )
+      }
+    }
+  }
+
+  @TempDir
+  lateinit var workDir: Path
+  val publication: MavenCentralPublication by lazy { MavenCentralPublication(context, workDir, dryRun = true) }
+  fun createDistributionFiles(): List<Path> {
+    val coordinates = MavenCoordinates("foo", "bar", "1.0")
+    return sequenceOf(
+      "pom" to "",
+      "jar" to "",
+      "jar" to "sources",
+      "jar" to "javadoc",
+    ).map { (packaging, classifier) ->
+      workDir.resolve(coordinates.getFileName(classifier = classifier, packaging = packaging)).createFile().apply {
+        if (packaging == "pom") {
+          writeText(
+            """
+            <project>
+              <groupId>${coordinates.groupId}</groupId>
+              <artifactId>${coordinates.artifactId}</artifactId>
+              <version>${coordinates.version}</version>
+            </project>
+          """.trimIndent()
+          )
+        }
+      }
+    }.toList()
+  }
+
+  @Test
+  fun `should fail upon an empty input directory`() {
+    runBlocking {
+      assertThrows<IllegalArgumentException> {
+        publication.execute()
+      }
+    }
+  }
+
+  @Test
+  fun `should generate the bundle zip`() {
+    runBlocking {
+      val files = createDistributionFiles()
+      publication.execute()
+      val bundle = workDir.resolve("bundle.zip")
+      assert(bundle.exists())
+      val entries = buildList {
+        suspendAwareReadZipFile(bundle) { entry, _ ->
+          add(entry)
+        }
+      }
+      assert(entries.containsAll(files.map { it.name }))
+      assert(entries.containsAll(files.map { "${it.name}.sha1" }))
+      assert(entries.containsAll(files.map { "${it.name}.sha256" }))
+      assert(entries.containsAll(files.map { "${it.name}.sha512" }))
+      assert(entries.containsAll(files.map { "${it.name}.md5" }))
+    }
+  }
+
+  @Test
+  fun `should fail upon an invalid input checksum`() {
+    runBlocking {
+      val files = createDistributionFiles()
+      val malformedChecksum = files.first()
+        .resolveSibling("${files.first().fileName}.sha1")
+        .createFile()
+      malformedChecksum.writeText("not a checksum")
+      assertThrows<MavenCentralPublication.ChecksumMismatch> {
+        publication.execute()
+      }
+    }
+  }
+}

platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/org/jetbrains/intellij/build/impl/sbom/ChecksumsTest.kt

@@ -1,23 +1,33 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.intellij.build.org.jetbrains.intellij.build.impl.sbom
 
+import com.intellij.util.io.DigestUtil.md5
+import com.intellij.util.io.DigestUtil.sha1
+import com.intellij.util.io.DigestUtil.sha256
+import com.intellij.util.io.DigestUtil.sha512
 import com.intellij.util.io.write
-import org.jetbrains.intellij.build.impl.sbom.SoftwareBillOfMaterialsImpl
+import org.jetbrains.intellij.build.impl.Checksums
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import java.nio.file.Path
 
-class SoftwareBillOfMaterialsTest {
+class ChecksumsTest {
   @Test
   fun checksums(@TempDir tempDir: Path) {
     val file = tempDir.resolve("file.txt")
     file.write("test")
-    val checksums = SoftwareBillOfMaterialsImpl.Checksums(file)
+    val checksums = Checksums(file, sha1(), sha256(), sha512(), md5())
     assert(checksums.sha1sum == "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3") {
       "Unexpected SHA1 checksum '${checksums.sha1sum}'"
     }
     assert(checksums.sha256sum == "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08") {
       "Unexpected SHA256 checksum '${checksums.sha256sum}'"
     }
+    assert(checksums.sha512sum == "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff") {
+      "Unexpected SHA512 checksum '${checksums.sha512sum}'"
+    }
+    assert(checksums.md5sum == "098f6bcd4621d373cade4e832627b4f6") {
+      "Unexpected MD5 checksum '${checksums.md5sum}'"
+    }
   }
 }