shared-indexes - rework metadata - add vcsCommitId for project

GitOrigin-RevId: f27ed95dc414626149878d85a2a2ff4d43bf51c2

java/idea-ui/src/com/intellij/internal/DumpJdkIndexStarter.kt

@@ -118,7 +118,7 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") {
     LOG.info("JDK size          = ${StringUtil.formatFileSize(jdkHome.totalSize())}")
     LOG.info("JDK hash          = ${hash}")
 
-    packIndexes(indexKind, indexName, hash, indexZip, infraVersion, outputDir, aliases = aliases)
+    packIndexes(indexKind, indexName, indexZip, infraVersion, outputDir, SharedIndexMetadataInfo(sourcesHash = hash, aliases = aliases))
     exitProcess(0)
   }
 }

java/idea-ui/src/com/intellij/internal/DumpProjectIndexesStarter.kt

@@ -35,10 +35,11 @@ class DumpProjectIndexesStarter : IndexesStarterBase("dump-project-index") {
     println("")
     println("")
 
+    val indexKind = "project"
     val tempDir = args.argFile(tempKey).recreateDir()
     val outputDir = args.argFile(outputKey).apply { mkdirs() }
     val projectDir = args.argFile(projectHome)
-    val hash = args.arg(revisionHash)
+    val vcsCommitId = args.arg(revisionHash)
 
     LOG.info("Opening project from $projectDir")
     val project = runAndCatchNotNull("Opening Project") {
@@ -46,7 +47,7 @@ class DumpProjectIndexesStarter : IndexesStarterBase("dump-project-index") {
     }
 
     val chunkName = FileUtil.sanitizeFileName(project.name, true)
-    val indexZip = File(tempDir, "project-$hash-$chunkName.ijx")
+    val indexZip = File(tempDir, "$indexKind-${FileUtil.sanitizeFileName(project.name)}-$vcsCommitId-$chunkName.ijx")
 
     LOG.info("Generating indexes...")
     val indexingStartTime = System.currentTimeMillis()
@@ -57,9 +58,9 @@ class DumpProjectIndexesStarter : IndexesStarterBase("dump-project-index") {
     val indexingTime = Longs.max(0L, System.currentTimeMillis() - indexingStartTime)
     LOG.info("Indexes build completed in ${StringUtil.formatDuration(indexingTime)}")
     LOG.info("size          = ${StringUtil.formatFileSize(indexZip.totalSize())}")
-    LOG.info("hash          = ${hash}")
+    LOG.info("commitId      = ${vcsCommitId}")
 
-    packIndexes("project", chunkName, hash, indexZip, infraVersion, outputDir)
+    packIndexes(indexKind, chunkName, indexZip, infraVersion, outputDir, SharedIndexMetadataInfo(vcsCommitId = vcsCommitId))
     exitProcess(0)
   }
 }

java/idea-ui/src/com/intellij/internal/IndexesStarterBase.kt

@@ -60,11 +60,10 @@ abstract class IndexesStarterBase(
 
   protected fun packIndexes(indexKind: String,
                             indexName: String,
-                            hash: String,
                             indexZip: File,
                             infraVersion: IndexInfrastructureVersion,
                             outputDir: File,
-                            aliases: Collection<String> = setOf()) {
+                            addon: SharedIndexMetadataInfo) {
     LOG.info("Packing the indexes to XZ...")
     val indexZipXZ = File(indexZip.path + ".xz")
     xz(indexZip, indexZipXZ)
@@ -74,7 +73,7 @@ abstract class IndexesStarterBase(
     LOG.info("Generated index in $indexZip")
 
     val indexMetadata = runAndCatchNotNull("extract JSON metadata from $indexZip") {
-      SharedIndexMetadata.writeIndexMetadata(indexName, indexKind, hash, infraVersion, aliases)
+      SharedIndexMetadata.writeIndexMetadata(indexName, indexKind, infraVersion, addon)
     }
 
     // we generate production layout here:
@@ -91,7 +90,8 @@ abstract class IndexesStarterBase(
     //       | <entry>.sha256 // SHA-256 hash of the .ijx.xz entry
     //
 
-    val indexDir = (outputDir / indexKind / hash).apply { mkdirs() }
+    val targetPathHash = addon.sourcesHash ?: addon.vcsCommitId ?: error("No source hash is specified")
+    val indexDir = (outputDir / indexKind / targetPathHash).apply { mkdirs() }
     fun indexFile(nameSuffix: String) = indexDir / "${indexName}-${infraVersion.weakVersionHash}$nameSuffix"
 
     FileUtil.copy(indexZipXZ, indexFile(".ijx.xz"))

java/idea-ui/src/com/intellij/internal/SharedIndexMetadata.kt

@@ -10,6 +10,13 @@ import com.intellij.util.indexing.IndexInfrastructureVersion
 import java.util.*
 import kotlin.collections.HashMap
 
+data class SharedIndexMetadataInfo(
+  val aliases: List<String> = listOf(),
+  val sourcesHash: String? = null,
+  val vcsCommitId: String? = null
+  //TODO: possibly a commits map?
+)
+
 object SharedIndexMetadata {
   private const val METADATA_VERSION = "2"
 
@@ -32,9 +39,8 @@ object SharedIndexMetadata {
 
   fun writeIndexMetadata(indexName: String,
                          indexKind: String,
-                         sourcesHash: String,
                          infrastructureVersion: IndexInfrastructureVersion,
-                         aliases: Collection<String> = setOf()): ByteArray {
+                         addon: SharedIndexMetadataInfo): ByteArray {
     try {
       val om = ObjectMapper()
 
@@ -45,11 +51,21 @@ object SharedIndexMetadata {
       root.putObject("sources").also { sources ->
         sources.put("os", IndexInfrastructureVersion.Os.getOs().osName)
         sources.put("os_name", SystemInfo.getOsNameAndVersion())
-        sources.put("hash", sourcesHash)
         sources.put("kind", indexKind)
         sources.put("name", indexName)
-        sources.putArray("aliases").let { aliasesNode ->
-          aliases.map { it.toLowerCase().trim() }.toSortedSet().forEach { aliasesNode.add(it) }
+
+        addon.sourcesHash?.let {
+          sources.put("hash", it)
+        }
+
+        addon.vcsCommitId?.let {
+          sources.put("vcs_commit_id", it)
+        }
+
+        addon.aliases.let { aliases ->
+          sources.putArray("aliases").let { aliasesNode ->
+            aliases.map { it.toLowerCase().trim() }.toSortedSet().forEach { aliasesNode.add(it) }
+          }
         }
       }
 

java/java-tests/testSrc/com/intellij/internal/SharedIndexMetadataTest.kt

@@ -16,7 +16,7 @@ class SharedIndexMetadataTest : BasePlatformTestCase() {
   @Test
   fun testAliasesAreIncluded() {
     val selfVersion = getIdeVersion()
-    val data = writeIndexMetadata("mock1", "jdk", "123", selfVersion, aliases = setOf("jonnyzzz", "intellij", "jdk"))
+    val data = writeIndexMetadata("mock1", "jdk2", selfVersion, SharedIndexMetadataInfo(sourcesHash = "123", aliases = listOf("jonnyzzz", "intellij", "jdk")))
 
     val om = ObjectMapper()
     val root = om.readTree(data) as ObjectNode
@@ -29,7 +29,7 @@ class SharedIndexMetadataTest : BasePlatformTestCase() {
   @Test
   fun testSourcesAreIncluded() {
     val selfVersion = getIdeVersion()
-    val data = writeIndexMetadata("mock1", "jdk2", "123", selfVersion)
+    val data = writeIndexMetadata("mock1", "jdk2", selfVersion, SharedIndexMetadataInfo(sourcesHash = "123"))
 
     val om = ObjectMapper()
     val root = om.readTree(data) as ObjectNode
@@ -78,7 +78,7 @@ class SharedIndexMetadataTest : BasePlatformTestCase() {
   fun testSelfSerializationIsStable() {
     val (a,b) = List(2) {
       val selfVersion = getIdeVersion()
-      writeIndexMetadata("mock1", "jdk", "123", selfVersion)
+      writeIndexMetadata("mock1", "jdk", selfVersion, SharedIndexMetadataInfo(sourcesHash = "123"))
     }
 
     Assert.assertArrayEquals(a, b)
@@ -89,7 +89,7 @@ class SharedIndexMetadataTest : BasePlatformTestCase() {
     val version = getIdeVersion()
     Assert.assertEquals(version, version.copy())
 
-    val json = writeIndexMetadata("mock1", "jdk", "123", version.copy().tuneSelfVersion())
+    val json = writeIndexMetadata("mock1", "jdk", version.copy().tuneSelfVersion(), SharedIndexMetadataInfo(sourcesHash = "123"))
     val om = ObjectMapper()
 
     val selfVersion = version.copy()