From f326b2b1a9526e6fe29975bffff2c009510cde0b Mon Sep 17 00:00:00 2001 From: "Dmitriy.Panov" Date: Tue, 25 Mar 2025 19:06:50 +0100 Subject: [PATCH] IJI-1629 deployment ID verification (cherry picked from commit 38d62f0302ed36924afd872984b91506e408c580) (cherry picked from commit 34fcbf839a47086681ba49aa49eec022d5be2bbf) IJ-MR-159792 GitOrigin-RevId: a8077ecce6f3051006b925b702024409fbcc11fc --- .../impl/maven/MavenCentralPublication.kt | 156 +++++++++++++----- .../impl/maven/MavenCentralPublicationTest.kt | 36 +++- 2 files changed, 147 insertions(+), 45 deletions(-) diff --git a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenCentralPublication.kt b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenCentralPublication.kt index ff6f86bf0248..4ea8533555dd 100644 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenCentralPublication.kt +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenCentralPublication.kt @@ -7,22 +7,28 @@ import com.intellij.util.io.DigestUtil.sha1 import com.intellij.util.io.DigestUtil.sha256 import com.intellij.util.io.DigestUtil.sha512 import com.intellij.util.xml.dom.readXmlAsModel -import kotlinx.coroutines.CoroutineName -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch +import io.opentelemetry.api.trace.Span +import kotlinx.coroutines.* +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaType import okhttp3.MultipartBody import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.asRequestBody +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response import org.jetbrains.annotations.ApiStatus +import org.jetbrains.annotations.VisibleForTesting import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.impl.Checksums import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder import org.jetbrains.intellij.build.telemetry.use import java.nio.file.Path import java.util.* +import java.util.concurrent.TimeUnit import kotlin.io.path.* +import kotlin.time.Duration.Companion.minutes /** * @param workDir is expected to contain: @@ -33,22 +39,30 @@ import kotlin.io.path.* * * md5, sha1, sha256 and sha512 checksum files (optional, will be verified if present) * * @param type https://central.sonatype.org/publish/publish-portal-api/#uploading-a-deployment-bundle + * + * See https://youtrack.jetbrains.com/articles/IJPL-A-611 internal article for more details */ @ApiStatus.Internal class MavenCentralPublication( private val context: BuildContext, private val workDir: Path, - private val type: Type = Type.USER_MANAGED, + private val type: PublicationType = PublicationType.AUTOMATIC, private val userName: String? = null, private val token: String? = null, private val dryRun: Boolean = context.options.isInDevelopmentMode, private val sign: Boolean = !dryRun, ) { private companion object { - const val URI_BASE = "https://central.sonatype.com/api/v1/publisher/upload" + /** + * See https://central.sonatype.com/api-doc + */ + const val URI_BASE = "https://central.sonatype.com/api/v1/publisher" + const val UPLOADING_URI_BASE = "$URI_BASE/upload" + const val STATUS_URI_BASE = "$URI_BASE/status" + val JSON = Json { ignoreUnknownKeys = true } } - enum class Type { + enum class PublicationType { USER_MANAGED, AUTOMATIC, } @@ -86,7 +100,8 @@ class MavenCentralPublication( bootstrap() sign() generateOrVerifyChecksums() - publish(bundle()) + val deploymentId = publish(bundle()) + if (deploymentId != null) wait(deploymentId) } private val distributionFiles: List get() = listOf(jar, pom, javadoc, sources) @@ -128,15 +143,17 @@ class MavenCentralPublication( val checksumFile = file.resolveSibling("${file.fileName}.$extension") if (checksumFile.exists()) { val suppliedValue = checksumFile.readLines().asSequence() + // sha256sum command output is a line with checksum, + // a character indicating type ('*' for --binary, ' ' for --text), + // and the supplied file argument .flatMap { it.splitToSequence(" ") } .firstOrNull() if (suppliedValue != value) { throw ChecksumMismatch("The supplied file $checksumFile content mismatch: '$suppliedValue' != '$value'") } } - else { - checksumFile.writeText(value) - } + // a checksum file should contain only a checksum itself + checksumFile.writeText(value) } } } @@ -156,52 +173,117 @@ class MavenCentralPublication( checksums("sha512") } + /** + * https://central.sonatype.org/publish/publish-portal-upload/ + */ private suspend fun bundle(): Path { return spanBuilder("creating a bundle").use { val bundle = workDir.resolve("bundle.zip") Compressor.Zip(bundle).use { for (file in distributionFiles.asSequence() + signatures() + checksums()) { - it.addFile(file.name, file) + it.addFile("${coordinates.directoryPath}/${file.name}", file) } } bundle } } - private suspend fun publish(bundle: Path) { - spanBuilder("publishing").setAttribute("bundle", "$bundle").use { + private suspend fun callSonatype( + uri: String, + builder: suspend (Request.Builder) -> Request.Builder, + action: suspend (Response) -> T, + ): T { + requireNotNull(userName) { + "Please specify intellij.build.mavenCentral.userName system property" + } + requireNotNull(token) { + "Please specify intellij.build.mavenCentral.token system property" + } + val base64Auth = Base64.getEncoder() + .encode("$userName:$token".toByteArray()) + .toString(Charsets.UTF_8) + val span = Span.current() + span.addEvent("Sending request to $uri...") + val client = OkHttpClient() + val request = Request.Builder().url(uri) + .header("Authorization", "Bearer $base64Auth") + .let { builder(it) } + .build() + return client.newCall(request) + .execute() + .use { + span.addEvent("Response status code: ${it.code}") + action(it) + } + } + + private suspend fun publish(bundle: Path): String? { + return spanBuilder("publishing").setAttribute("bundle", "$bundle").use { span -> if (dryRun) { - it.addEvent("skipped in the dryRun mode") - return@use - } - requireNotNull(userName) { - "Please specify intellij.build.mavenCentral.userName system property" - } - requireNotNull(token) { - "Please specify intellij.build.mavenCentral.token system property" + span.addEvent("skipped in the dryRun mode") + return@use null } val deploymentName = "${coordinates.artifactId}-${coordinates.version}" - val uri = "$URI_BASE?name=$deploymentName&publicationType=$type" - val base64Auth = Base64.getEncoder().encode("$userName:$token".toByteArray()).toString(Charsets.UTF_8) - it.addEvent("Sending request to $uri...") - val client = OkHttpClient() - val request = Request.Builder() - .url(uri) - .header("Authorization", "Bearer $base64Auth") - .post( + val uri = "$UPLOADING_URI_BASE?name=$deploymentName&publicationType=$type" + callSonatype(uri, builder = { + it.post( MultipartBody.Builder() .setType(MultipartBody.FORM) .addFormDataPart("bundle", bundle.name, bundle.toFile().asRequestBody()) .build() - ).build() - client.newCall(request).execute().use { response -> - val statusCode = response.code - it.addEvent("Upload status code: $statusCode") - it.addEvent("Upload response: ${response.body.string()}") - check(statusCode == 201) { - "Unable to upload to Central repository, status code: $statusCode, upload response: ${response.body.string()}" + ) + }, action = { + val deploymentId = it.body.string() + check(it.code == 201) { + "Unable to upload to Central repository, status code: ${it.code}, upload response: $deploymentId" + } + span.addEvent("Deployment ID: $deploymentId") + deploymentId + }) + } + } + + /** + * @param deploymentId see https://central.sonatype.org/publish/publish-portal-api/#uploading-a-deployment-bundle + */ + private suspend fun wait(deploymentId: String) { + spanBuilder("waiting").setAttribute("deploymentId", deploymentId).use { span -> + withTimeout(30.minutes) { + while (true) { + val deploymentState = callSonatype("$STATUS_URI_BASE?id=$deploymentId", builder = { + it.post("{}".toRequestBody("application/json".toMediaType())) + }, action = { + val response = it.body.string() + span.addEvent(response) + parseDeploymentState(response) + }) + when { + deploymentState == DeploymentState.FAILED -> error("$deploymentId status is $deploymentState") + deploymentState == DeploymentState.VALIDATED && type == PublicationType.USER_MANAGED || + deploymentState == DeploymentState.PUBLISHED && type == PublicationType.AUTOMATIC -> break + else -> delay(TimeUnit.SECONDS.toMillis(15)) + } } } } } + + @VisibleForTesting + @Suppress("unused") + enum class DeploymentState { + PENDING, + VALIDATING, + VALIDATED, + PUBLISHING, + PUBLISHED, + FAILED, + } + + @Serializable + private class StatusResponse(val deploymentState: DeploymentState) + + @VisibleForTesting + fun parseDeploymentState(response: String): DeploymentState { + return JSON.decodeFromString(response).deploymentState + } } \ No newline at end of file diff --git a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt index d1c3b57820a0..8e173903de03 100644 --- a/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt +++ b/platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt @@ -7,7 +7,9 @@ import org.jetbrains.intellij.build.BuildContext import org.jetbrains.intellij.build.BuildPaths.Companion.COMMUNITY_ROOT import org.jetbrains.intellij.build.IdeaCommunityProperties import org.jetbrains.intellij.build.impl.BuildContextImpl +import org.jetbrains.intellij.build.impl.maven.MavenCentralPublication.DeploymentState import org.jetbrains.intellij.build.io.suspendAwareReadZipFile +import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir @@ -32,8 +34,8 @@ class MavenCentralPublicationTest { @TempDir lateinit var workDir: Path val publication: MavenCentralPublication by lazy { MavenCentralPublication(context, workDir, dryRun = true) } + val coordinates = MavenCoordinates("foo", "bar", "1.0") fun createDistributionFiles(): List { - val coordinates = MavenCoordinates("foo", "bar", "1.0") return sequenceOf( "pom" to "", "jar" to "", @@ -66,9 +68,9 @@ class MavenCentralPublicationTest { } @Test - fun `should generate the bundle zip`() { + fun `should generate a bundle zip`() { runBlocking { - val files = createDistributionFiles() + val files = createDistributionFiles().map { "${coordinates.directoryPath}/${it.name}" } publication.execute() val bundle = workDir.resolve("bundle.zip") assert(bundle.exists()) @@ -77,11 +79,11 @@ class MavenCentralPublicationTest { add(entry) } } - assert(entries.containsAll(files.map { it.name })) - assert(entries.containsAll(files.map { "${it.name}.sha1" })) - assert(entries.containsAll(files.map { "${it.name}.sha256" })) - assert(entries.containsAll(files.map { "${it.name}.sha512" })) - assert(entries.containsAll(files.map { "${it.name}.md5" })) + assert(entries.containsAll(files)) + assert(entries.containsAll(files.map { "$it.sha1" })) + assert(entries.containsAll(files.map { "$it.sha256" })) + assert(entries.containsAll(files.map { "$it.sha512" })) + assert(entries.containsAll(files.map { "$it.md5" })) } } @@ -98,4 +100,22 @@ class MavenCentralPublicationTest { } } } + + @Test + fun `deployment state parsing test`() { + Assertions.assertEquals( + DeploymentState.PUBLISHED, publication.parseDeploymentState( + """ + { + "deploymentId": "28570f16-da32-4c14-bd2e-c1acc0782365", + "deploymentName": "central-bundle.zip", + "deploymentState": "PUBLISHED", + "purls": [ + "pkg:maven/com.sonatype.central.example/example_java_project@0.0.7" + ] + } + """.trimIndent() + ) + ) + } } \ No newline at end of file