IJI-2358 an option to validate the generated Maven artifacts

before Sonatype publication, for example

(cherry picked from commit 51501f216e1ae409d39b215b1201bbaef4ec6e19)

IJ-MR-159792

GitOrigin-RevId: dad773ac09311ecd4003e7059edff3a4c88d5fa1

platform/build-scripts/src/org/jetbrains/intellij/build/MavenArtifactsProperties.kt

@@ -5,6 +5,7 @@ import kotlinx.collections.immutable.PersistentList
 import kotlinx.collections.immutable.persistentListOf
 import org.apache.maven.model.Model
 import org.jetbrains.annotations.ApiStatus
+import org.jetbrains.intellij.build.impl.maven.GeneratedMavenArtifacts
 import org.jetbrains.intellij.build.impl.maven.MavenCoordinates
 import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.jps.util.JpsPathUtil
@@ -61,5 +62,5 @@ class MavenArtifactsProperties {
   var isJavadocJarRequired: (JpsModule) -> Boolean = { false }
 
   @ApiStatus.Internal
-  var validate: (Collection<Pair<JpsModule, MavenCoordinates>>) -> Unit = {}
+  var validate: (BuildContext, Collection<GeneratedMavenArtifacts>) -> Unit = { _, _ -> }
 }

platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildTasksImpl.kt

@@ -498,7 +498,7 @@ private fun CoroutineScope.createMavenArtifactJob(context: BuildContext, distrib
     }
 
     val mavenArtifactsBuilder = MavenArtifactsBuilder(context)
-    val builtArtifacts = mutableSetOf<MavenArtifactData>()
+    val builtArtifacts = mutableMapOf<MavenArtifactData, List<Path>>()
     @Suppress("UsePropertyAccessSyntax")
     if (!platformModules.isEmpty()) {
       mavenArtifactsBuilder.generateMavenArtifacts(
@@ -523,7 +523,7 @@ private fun CoroutineScope.createMavenArtifactJob(context: BuildContext, distrib
         outputDir = "proprietary-maven-artifacts"
       )
     }
-    mavenArtifacts.validate(builtArtifacts.map { it.module to it.coordinates })
+    mavenArtifactsBuilder.validate(builtArtifacts)
   }
 }
 

platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenArtifactsBuilder.kt

@@ -5,23 +5,28 @@ import com.intellij.util.text.NameUtilCore
 import io.opentelemetry.api.common.AttributeKey
 import io.opentelemetry.api.common.Attributes
 import io.opentelemetry.api.trace.Span
+import kotlinx.coroutines.CoroutineName
+import kotlinx.coroutines.async
 import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.launch
 import org.apache.maven.model.Dependency
 import org.apache.maven.model.Developer
 import org.apache.maven.model.Exclusion
 import org.apache.maven.model.Model
 import org.apache.maven.model.Organization
 import org.apache.maven.model.io.xpp3.MavenXpp3Writer
+import org.jetbrains.annotations.ApiStatus
 import org.jetbrains.intellij.build.BuildContext
 import org.jetbrains.intellij.build.DirSource
 import org.jetbrains.intellij.build.buildJar
 import org.jetbrains.intellij.build.impl.commonModuleExcludes
-import org.jetbrains.intellij.build.impl.createModuleSourcesNamesFilter
 import org.jetbrains.intellij.build.impl.getLibraryFilename
 import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder
 import org.jetbrains.intellij.build.telemetry.use
-import org.jetbrains.jps.model.java.*
+import org.jetbrains.jps.model.java.JavaResourceRootType
+import org.jetbrains.jps.model.java.JavaSourceRootType
+import org.jetbrains.jps.model.java.JpsJavaClasspathKind
+import org.jetbrains.jps.model.java.JpsJavaDependencyScope
+import org.jetbrains.jps.model.java.JpsJavaExtensionService
 import org.jetbrains.jps.model.library.JpsLibrary
 import org.jetbrains.jps.model.library.JpsMavenRepositoryLibraryDescriptor
 import org.jetbrains.jps.model.library.JpsRepositoryLibraryType
@@ -31,7 +36,7 @@ import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.jps.model.module.JpsModuleDependency
 import java.nio.file.Files
 import java.nio.file.Path
-import java.util.*
+import java.util.Locale
 import java.util.function.BiConsumer
 
 /**
@@ -128,13 +133,15 @@ open class MavenArtifactsBuilder(protected val context: BuildContext) {
     moduleNamesToPublish: Collection<String>,
     moduleNamesToSquashAndPublish: List<String> = emptyList(),
     outputDir: String,
+    validate: Boolean = false,
   ) {
     generateMavenArtifacts(
       moduleNamesToPublish,
       moduleNamesToSquashAndPublish,
       outputDir,
       ignoreNonMavenizable = false,
-      builtArtifacts = mutableSetOf(),
+      builtArtifacts = mutableMapOf(),
+      validate = validate,
     )
   }
 
@@ -146,7 +153,8 @@ open class MavenArtifactsBuilder(protected val context: BuildContext) {
     moduleNamesToSquashAndPublish: List<String> = emptyList(),
     outputDir: String,
     ignoreNonMavenizable: Boolean = false,
-    builtArtifacts: MutableSet<MavenArtifactData>,
+    builtArtifacts: MutableMap<MavenArtifactData, List<Path>>,
+    validate: Boolean = false,
   ) {
     val artifactsToBuild = HashMap<MavenArtifactData, List<JpsModule>>()
 
@@ -171,14 +179,16 @@ open class MavenArtifactsBuilder(protected val context: BuildContext) {
       val coordinates = generateMavenCoordinatesSquashed(module.name, context.buildNumber)
       artifactsToBuild[MavenArtifactData(module, coordinates, dependencies)] = modules.toList()
     }
-    artifactsToBuild -= builtArtifacts
-    spanBuilder("layout maven artifacts")
+    artifactsToBuild -= builtArtifacts.keys
+    builtArtifacts += spanBuilder("layout maven artifacts")
       .setAttribute(AttributeKey.stringArrayKey("modules"), artifactsToBuild.entries.map { entry ->
         "  [${entry.value.joinToString(separator = ",") { it.name }}] -> ${entry.key.coordinates}"
       }).use {
         layoutMavenArtifacts(artifactsToBuild, context.paths.artifactDir.resolve(outputDir), context)
       }
-    builtArtifacts += artifactsToBuild.keys
+    if (validate) {
+      validate(builtArtifacts)
+    }
   }
 
   private fun generateMavenArtifactData(moduleNames: Collection<String>, ignoreNonMavenizable: Boolean): Map<JpsModule, MavenArtifactData> {
@@ -289,6 +299,12 @@ open class MavenArtifactsBuilder(protected val context: BuildContext) {
   protected open fun generateMavenCoordinatesForModule(module: JpsModule): MavenCoordinates {
     return generateMavenCoordinates(module.name, context.buildNumber)
   }
+
+  internal fun validate(builtArtifacts: Map<MavenArtifactData, List<Path>>) {
+    context.productProperties.mavenArtifacts.validate(context, builtArtifacts.map { (data, files) ->
+      GeneratedMavenArtifacts(data.module, data.coordinates, files)
+    })
+  }
 }
 
 internal enum class DependencyScope {
@@ -418,13 +434,15 @@ private fun splitByCamelHumpsMergingNumbers(s: String): List<String> {
  */
 private val COMMON_GROUP_NAMES: Set<String> = setOf("platform", "vcs", "tools", "clouds")
 
-private suspend fun layoutMavenArtifacts(modulesToPublish: Map<MavenArtifactData, List<JpsModule>>,
-                                         outputDir: Path,
-                                         context: BuildContext) {
-  val publishSourceFilter = context.productProperties.mavenArtifacts.publishSourcesFilter
-  coroutineScope {
-    for ((artifactData, modules) in modulesToPublish.entries) {
-      launch {
+private suspend fun layoutMavenArtifacts(
+  modulesToPublish: Map<MavenArtifactData, List<JpsModule>>,
+  outputDir: Path,
+  context: BuildContext,
+): Map<MavenArtifactData, List<Path>> {
+  return coroutineScope {
+    modulesToPublish.entries.map { (artifactData, modules) ->
+      async(CoroutineName("layout maven artifact ${artifactData.coordinates}")) {
+        val artifacts = mutableListOf<Path>()
         val modulesWithSources = modules.filter {
           it.getSourceRoots(JavaSourceRootType.SOURCE).any() || it.getSourceRoots(JavaResourceRootType.RESOURCE).any()
         }
@@ -432,24 +450,29 @@ private suspend fun layoutMavenArtifacts(modulesToPublish: Map<MavenArtifactData
         val dirPath = artifactData.coordinates.directoryPath
         val artifactDir = outputDir.resolve(dirPath)
         Files.createDirectories(artifactDir)
-
+        val pom = artifactDir.resolve(artifactData.coordinates.getFileName(packaging = "pom"))
         generatePomXmlData(
           context = context,
           artifactData = artifactData,
-          file = artifactDir.resolve(artifactData.coordinates.getFileName(packaging = "pom")),
+          file = pom,
         )
-
+        artifacts.add(pom)
+        val jar = artifactDir.resolve(artifactData.coordinates.getFileName(packaging = "jar"))
         buildJar(
-          targetFile = artifactDir.resolve(artifactData.coordinates.getFileName(packaging = "jar")),
+          targetFile = jar,
           sources = modulesWithSources.map {
             DirSource(dir = context.getModuleOutputDir(it), excludes = commonModuleExcludes)
           },
         )
+        artifacts.add(jar)
 
-        val publishSourcesForModules = modules.filter { publishSourceFilter(it, context) }
+        val publishSourcesForModules = modules.filter {
+          context.productProperties.mavenArtifacts.publishSourcesFilter(it, context)
+        }
         if (!publishSourcesForModules.isEmpty() && !modulesWithSources.isEmpty()) {
+          val sources = artifactDir.resolve(artifactData.coordinates.getFileName("sources", "jar"))
           buildJar(
-            targetFile = artifactDir.resolve(artifactData.coordinates.getFileName("sources", "jar")),
+            targetFile = sources,
             sources = publishSourcesForModules.flatMap { module ->
               module.getSourceRoots(JavaSourceRootType.SOURCE).asSequence().map {
                 DirSource(dir = it.path, prefix = it.properties.packagePrefix.replace('.', '/'), excludes = commonModuleExcludes)
@@ -460,19 +483,30 @@ private suspend fun layoutMavenArtifacts(modulesToPublish: Map<MavenArtifactData
             },
             compress = true,
           )
+          artifacts.add(sources)
         }
         if (context.productProperties.mavenArtifacts.isJavadocJarRequired(artifactData.module)) {
           check(modulesWithSources.any()) {
             "No modules with sources found in $modules, a documentation cannot be generated"
           }
           val docsFolder = Dokka(context).generateDocumentation(modules = modulesWithSources)
+          val javadoc = artifactDir.resolve(artifactData.coordinates.getFileName("javadoc", "jar"))
           buildJar(
-            targetFile = artifactDir.resolve(artifactData.coordinates.getFileName("javadoc", "jar")),
+            targetFile = javadoc,
             sources = listOf(DirSource(docsFolder)),
             compress = true,
           )
+          artifacts.add(javadoc)
         }
+        artifactData to artifacts
       }
-    }
+    }.associate { it.await() }
   }
 }
+
+@ApiStatus.Internal
+data class GeneratedMavenArtifacts(
+  val module: JpsModule,
+  val coordinates: MavenCoordinates,
+  val files: List<Path>,
+)

platform/build-scripts/src/org/jetbrains/intellij/build/impl/maven/MavenCentralPublication.kt

@@ -6,9 +6,13 @@ import com.intellij.util.io.DigestUtil.md5
 import com.intellij.util.io.DigestUtil.sha1
 import com.intellij.util.io.DigestUtil.sha256
 import com.intellij.util.io.DigestUtil.sha512
-import com.intellij.util.xml.dom.readXmlAsModel
 import io.opentelemetry.api.trace.Span
-import kotlinx.coroutines.*
+import kotlinx.coroutines.CoroutineName
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withTimeout
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
 import okhttp3.MediaType.Companion.toMediaType
@@ -18,6 +22,7 @@ import okhttp3.Request
 import okhttp3.RequestBody.Companion.asRequestBody
 import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
+import org.apache.maven.model.io.xpp3.MavenXpp3Reader
 import org.jetbrains.annotations.ApiStatus
 import org.jetbrains.annotations.VisibleForTesting
 import org.jetbrains.intellij.build.BuildContext
@@ -26,7 +31,7 @@ import org.jetbrains.intellij.build.impl.Checksums
 import org.jetbrains.intellij.build.telemetry.TraceManager.spanBuilder
 import org.jetbrains.intellij.build.telemetry.use
 import java.nio.file.Path
-import java.util.*
+import java.util.Base64
 import java.util.concurrent.TimeUnit
 import kotlin.io.path.ExperimentalPathApi
 import kotlin.io.path.PathWalkOption
@@ -62,14 +67,47 @@ class MavenCentralPublication(
   private val dryRun: Boolean = context.options.isInDevelopmentMode,
   private val sign: Boolean = !dryRun,
 ) {
-  private companion object {
+  companion object {
     /**
      * See https://central.sonatype.com/api-doc
      */
-    const val URI_BASE = "https://central.sonatype.com/api/v1/publisher"
-    const val UPLOADING_URI_BASE = "$URI_BASE/upload"
-    const val STATUS_URI_BASE = "$URI_BASE/status"
-    val JSON = Json { ignoreUnknownKeys = true }
+    private const val URI_BASE = "https://central.sonatype.com/api/v1/publisher"
+    private const val UPLOADING_URI_BASE = "$URI_BASE/upload"
+    private const val STATUS_URI_BASE = "$URI_BASE/status"
+    private val JSON = Json { ignoreUnknownKeys = true }
+
+    /**
+     * See https://central.sonatype.org/publish/requirements/#required-pom-metadata
+     */
+    fun loadAndValidatePomXml(pom: Path): MavenCoordinates {
+      val pomModel = pom.inputStream().bufferedReader().use {
+        MavenXpp3Reader().read(it, true)
+      }
+      val coordinates = MavenCoordinates(
+        groupId = pomModel.groupId ?: error("$pom doesn't contain <groupId>"),
+        artifactId = pomModel.artifactId ?: error("$pom doesn't contain <artifactId>"),
+        version = pomModel.version ?: error("$pom doesn't contain <version>"),
+      )
+      check(!pomModel.name.isNullOrBlank()) {
+        "$pom doesn't contain <name>"
+      }
+      check(!pomModel.description.isNullOrBlank()) {
+        "$pom doesn't contain <description>"
+      }
+      check(!pomModel.url.isNullOrBlank()) {
+        "$pom doesn't contain <url>"
+      }
+      check(pomModel.licenses.any()) {
+        "$pom doesn't contain <licenses>"
+      }
+      check(pomModel.developers.any()) {
+        "$pom doesn't contain <developers>"
+      }
+      check(pomModel.scm != null) {
+        "$pom doesn't contain <scm>"
+      }
+      return coordinates
+    }
   }
 
   enum class PublishingType {
@@ -142,15 +180,7 @@ class MavenCentralPublication(
 
   private val artifacts: List<MavenArtifacts> by lazy {
     files(extension = "pom").map { pom ->
-      val project = readXmlAsModel(pom)
-      check(project.name == "project") {
-        "$pom doesn't contain <project> root element"
-      }
-      val coordinates = MavenCoordinates(
-        groupId = project.getChild("groupId")?.content ?: error("$pom doesn't contain <groupId> element"),
-        artifactId = project.getChild("artifactId")?.content ?: error("$pom doesn't contain <artifactId> element"),
-        version = project.getChild("version")?.content ?: error("$pom doesn't contain <version> element"),
-      )
+      val coordinates = loadAndValidatePomXml(pom)
       val jar = file(coordinates.getFileName(packaging = "jar"))
       val sources = file(coordinates.getFileName(classifier = "sources", packaging = "jar"))
       val javadoc = file(coordinates.getFileName(classifier = "javadoc", packaging = "jar"))

platform/build-scripts/tests/intellij.platform.buildScripts.tests.iml

@@ -26,5 +26,6 @@
     <orderEntry type="library" scope="TEST" name="kotlinx-coroutines-core" level="project" />
     <orderEntry type="module" module-name="intellij.platform.util.zip" scope="TEST" />
     <orderEntry type="module" module-name="intellij.platform.util.jdom" scope="TEST" />
+    <orderEntry type="library" scope="TEST" name="maven-resolver-provider" level="project" />
   </component>
 </module>

platform/build-scripts/tests/testSrc/org/jetbrains/intellij/build/impl/maven/MavenCentralPublicationTest.kt

@@ -4,6 +4,11 @@ package org.jetbrains.intellij.build.impl.maven
 import com.intellij.testFramework.assertions.Assertions.assertThat
 import com.intellij.testFramework.utils.io.createFile
 import kotlinx.coroutines.runBlocking
+import org.apache.maven.model.Developer
+import org.apache.maven.model.License
+import org.apache.maven.model.Model
+import org.apache.maven.model.Scm
+import org.apache.maven.model.io.xpp3.MavenXpp3Writer
 import org.jetbrains.intellij.build.BuildContext
 import org.jetbrains.intellij.build.BuildPaths.Companion.COMMUNITY_ROOT
 import org.jetbrains.intellij.build.IdeaCommunityProperties
@@ -14,8 +19,8 @@ import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Files
 import java.nio.file.Path
-import kotlin.io.path.exists
 import kotlin.io.path.writeText
 
 class MavenCentralPublicationTest {
@@ -52,21 +57,29 @@ class MavenCentralPublicationTest {
         val zipPath = "${coordinates.directoryPath}/$name"
         val file = workDir.resolve(if (flatLayout) name else zipPath).createFile()
         if (packaging == "pom") {
-          file.writeText(
-            """
-              <project>
-                <groupId>${coordinates.groupId}</groupId>
-                <artifactId>${coordinates.artifactId}</artifactId>
-                <version>${coordinates.version}</version>
-              </project>
-            """.trimIndent()
-          )
+          writePom(coordinates, file)
         }
         Result(file, zipPath = zipPath)
       }
     }.toList()
   }
 
+  private fun writePom(coordinates: MavenCoordinates, file: Path) {
+    val pom = Model()
+    pom.groupId = coordinates.groupId
+    pom.artifactId = coordinates.artifactId
+    pom.version = coordinates.version
+    pom.name = coordinates.artifactId
+    pom.description = coordinates.artifactId
+    pom.url = "https://github.com/JetBrains/intellij-community"
+    pom.addDeveloper(Developer())
+    pom.scm = Scm()
+    pom.addLicense(License())
+    Files.newBufferedWriter(file).use {
+      MavenXpp3Writer().write(it, pom)
+    }
+  }
+
   @Test
   fun `should fail upon an empty input directory`() {
     runBlocking {
@@ -76,10 +89,9 @@ class MavenCentralPublicationTest {
     }
   }
 
-  @Test
-  fun `should generate a bundle zip for artifacts`() {
+  private fun `should generate a bundle zip for artifacts`(flatLayout: Boolean) {
     runBlocking {
-      val files = createDistributionFiles().map { it.zipPath }
+      val files = createDistributionFiles(flatLayout = flatLayout).map { it.zipPath }
       publication.execute()
       val bundle = workDir.resolve("bundle.zip")
       assertThat(bundle).exists()
@@ -100,28 +112,15 @@ class MavenCentralPublicationTest {
     }
   }
 
+
+  @Test
+  fun `should generate a bundle zip for artifacts`() {
+    `should generate a bundle zip for artifacts`(flatLayout = false)
+  }
+
   @Test
   fun `should generate a bundle zip for flat artifacts layout`() {
-    runBlocking {
-      val files = createDistributionFiles(flatLayout = true).map { it.zipPath }
-      publication.execute()
-      val bundle = workDir.resolve("bundle.zip")
-      assert(bundle.exists())
-      val entries = buildList {
-        suspendAwareReadZipFile(bundle) { entry, _ ->
-          add(entry)
-        }
-      }.sorted()
-      Assertions.assertEquals(
-        files.asSequence()
-          .plus(files.asSequence().map { "$it.sha1" })
-          .plus(files.asSequence().map { "$it.sha256" })
-          .plus(files.asSequence().map { "$it.sha512" })
-          .plus(files.asSequence().map { "$it.md5" })
-          .sorted().toList(),
-        entries,
-      )
-    }
+    `should generate a bundle zip for artifacts`(flatLayout = true)
   }
 
   @Test