Reapply "IJPL-159596 reduce jar cache metadata size - hash per item is ok to"

This reverts commit 50ad9f9e2a7a173c4addd7bbec337f26040e2c76.

GitOrigin-RevId: ea123591a5ef7c30f5d114c29964cd5d6d9ef2d7

platform/build-scripts/src/org/jetbrains/intellij/build/SourceAndCacheStrategy.kt

@@ -45,22 +45,20 @@ internal fun createSourceAndCacheStrategyList(sources: List<Source>, productionC
         is InMemoryContentSource -> InMemorySourceAndCacheStrategy(source)
         is FileSource -> FileSourceCacheStrategy(source)
         is ZipSource -> {
-          if (!source.file.startsWith(MAVEN_REPO)) {
-            NonMavenJarSourceAndCacheStrategy(source)
+          if (source.file.startsWith(MAVEN_REPO)) {
+            MavenJarSourceAndCacheStrategy(source)
           }
           else {
-            MavenJarSourceAndCacheStrategy(source)
+            NonMavenJarSourceAndCacheStrategy(source)
           }
         }
         is LazySource -> LazySourceAndCacheStrategy(source)
       }
     }
-    .sortedBy { it.name }
 }
 
 internal sealed interface SourceAndCacheStrategy {
   val source: Source
-  val name: String
 
   /**
    * The [updateAssetDigest] must be called prior to invoking this method.
@@ -73,38 +71,36 @@ internal sealed interface SourceAndCacheStrategy {
 }
 
 private class MavenJarSourceAndCacheStrategy(override val source: ZipSource) : SourceAndCacheStrategy {
-  override val name = MAVEN_REPO.relativize(source.file).invariantSeparatorsPathString
+  private var hash = 0L
 
-  override fun getHash() = Hashing.komihash5_0().hashCharsToLong(name)
+  override fun getHash() = hash
 
-  override fun getSize(): Long = Files.size(source.file)
+  override fun getSize() = Files.size(source.file)
 
   override fun updateAssetDigest(digest: HashStream64) {
-    // path includes version - that's enough
+    val relativePath = MAVEN_REPO.relativize(source.file).invariantSeparatorsPathString
+    hash = Hashing.komihash5_0().hashCharsToLong(relativePath)
+    digest.putString(relativePath)
   }
 }
 
 private class LazySourceAndCacheStrategy(override val source: LazySource) : SourceAndCacheStrategy {
-  override val name: String
-    get() = source.name
-
   override fun getHash() = source.hash
 
-  override fun getSize(): Long = 0
+  override fun getSize() = 0L
 
   override fun updateAssetDigest(digest: HashStream64) {
+    digest.putString(source.name)
     digest.putLong(source.hash)
   }
 }
 
 private class NonMavenJarSourceAndCacheStrategy(override val source: ZipSource) : SourceAndCacheStrategy {
-  private var hash: Long = 0
-
-  override val name = source.file.toString()
+  private var hash = 0L
 
   override fun getHash() = hash
 
-  override fun getSize(): Long = Files.size(source.file)
+  override fun getSize() = Files.size(source.file)
 
   override fun updateAssetDigest(digest: HashStream64) {
     val hasher = Hashing.komihash5_0().hashStream()
@@ -135,47 +131,42 @@ private class NonMavenJarSourceAndCacheStrategy(override val source: ZipSource)
   }
 }
 
-private class ModuleOutputSourceAndCacheStrategy(override val source: DirSource, override val name: String) : SourceAndCacheStrategy {
-  private var hash: Long = 0
+private class ModuleOutputSourceAndCacheStrategy(override val source: DirSource, private val name: String) : SourceAndCacheStrategy {
+  private var hash = 0L
 
   override fun getHash() = hash
 
-  override fun getSize(): Long = 0
+  override fun getSize() = 0L
 
   override fun updateAssetDigest(digest: HashStream64) {
+    digest.putString(name)
     hash = computeHashForModuleOutput(source)
     digest.putLong(hash)
   }
 }
 
 private class InMemorySourceAndCacheStrategy(override val source: InMemoryContentSource) : SourceAndCacheStrategy {
-  private var hash: Long = 0
-
-  override val name: String
-    get() = source.relativePath
+  private var hash = 0L
 
   override fun getHash() = hash
 
-  override fun getSize(): Long = 0
+  override fun getSize() = source.data.size.toLong()
 
   override fun updateAssetDigest(digest: HashStream64) {
+    digest.putString(source.relativePath)
     hash = Hashing.komihash5_0().hashBytesToLong(source.data)
     digest.putLong(hash).putInt(source.data.size)
   }
 }
 
 private class FileSourceCacheStrategy(override val source: FileSource) : SourceAndCacheStrategy {
-  private var hash: Long = source.hash
+  override fun getHash() = source.hash
 
-  override val name: String
-    get() = source.relativePath
-
-  override fun getHash() = hash
-
-  override fun getSize(): Long = source.size.toLong()
+  override fun getSize() = source.size.toLong()
 
   override fun updateAssetDigest(digest: HashStream64) {
-    digest.putLong(hash)
+    digest.putString(source.relativePath)
+    digest.putLong(source.hash)
   }
 }
 

platform/build-scripts/src/org/jetbrains/intellij/build/impl/BuildContextImpl.kt

@@ -360,7 +360,7 @@ class BuildContextImpl internal constructor(
           get() = false
 
         override fun updateDigest(digest: HashStream64) {
-          digest.putByte(Byte.MIN_VALUE)
+          digest.putInt(-1)
         }
 
         override suspend fun produce(targetFile: Path) {

platform/build-scripts/src/org/jetbrains/intellij/build/impl/JarPackager.kt

@@ -715,6 +715,7 @@ private data class AssetDescriptor(
   @JvmField val nativeFiles: List<String>?,
   @JvmField val useCacheAsTargetFile: Boolean = true,
 ) {
+  // must be sorted - we use it as is for Jar Cache
   @JvmField
   val sources: MutableList<Source> = mutableListOf()
 
@@ -839,7 +840,7 @@ private suspend fun buildJars(
                     digest.putString(layout.mainModule)
                   }
                   else {
-                    digest.putByte(0)
+                    digest.putInt(0)
                   }
                 }
 
@@ -967,7 +968,6 @@ private fun computeDistributionFileEntries(asset: AssetDescriptor, hasher: HashS
 
     var size = 0
     hasher.reset()
-    hasher.putInt(sources.size)
     for (source in sources) {
       size += source.size
       if (!dryRun) {
@@ -976,6 +976,7 @@ private fun computeDistributionFileEntries(asset: AssetDescriptor, hasher: HashS
         hasher.putInt(source.size)
       }
     }
+    hasher.putInt(sources.size)
 
     val hash = hasher.asLong
     list.add(

platform/build-scripts/src/org/jetbrains/intellij/build/jarCache/LocalDiskJarCacheManager.kt

@@ -3,7 +3,6 @@
 
 package org.jetbrains.intellij.build.jarCache
 
-import com.dynatrace.hash4j.hashing.HashStream64
 import com.dynatrace.hash4j.hashing.Hashing
 import io.opentelemetry.api.common.AttributeKey
 import io.opentelemetry.api.common.Attributes
@@ -29,7 +28,7 @@ import kotlin.time.Duration.Companion.days
 private const val jarSuffix = ".jar"
 private const val metaSuffix = ".bin"
 
-private const val cacheVersion: Byte = 9
+private const val cacheVersion: Byte = 10
 
 internal class LocalDiskJarCacheManager(
   private val cacheDir: Path,
@@ -55,26 +54,29 @@ internal class LocalDiskJarCacheManager(
   ): Path {
     val items = createSourceAndCacheStrategyList(sources = sources, productionClassOutDir = productionClassOutDir)
 
+    val targetFileNamePrefix = targetFile.fileName.toString().removeSuffix(jarSuffix)
+
     val hash = Hashing.komihash5_0().hashStream()
-    hashCommonMeta(hash = hash, items = items, targetFile = targetFile)
+    hash.putByte(cacheVersion)
     for (source in items) {
-      hash.putString(source.name)
       source.updateAssetDigest(hash)
     }
+    hash.putInt(items.size)
 
     val hash1 = java.lang.Long.toUnsignedString(hash.asLong, Character.MAX_RADIX)
 
     // another 64-bit hash without `source.updateAssetDigest` to reduce the chance of collision
     hash.reset()
-    producer.updateDigest(hash)
-    for (source in items.asReversed()) {
-      hash.putString(source.name)
+    hash.putByte(cacheVersion)
+    for (source in items) {
+      hash.putLong(source.getHash())
     }
-    hashCommonMeta(hash = hash, items = items, targetFile = targetFile)
+    hash.putInt(items.size)
+    producer.updateDigest(hash)
 
     val hash2 = java.lang.Long.toUnsignedString(hash.asLong, Character.MAX_RADIX)
 
-    val cacheName = "${targetFile.fileName.toString().removeSuffix(jarSuffix)}-$hash1-$hash2"
+    val cacheName = "$targetFileNamePrefix-$hash1-$hash2"
     val cacheFileName = (cacheName + jarSuffix).takeLast(255)
     val cacheFile = cacheDir.resolve(cacheFileName)
     val cacheMetadataFile = cacheDir.resolve((cacheName + metaSuffix).takeLast(255))
@@ -100,7 +102,7 @@ internal class LocalDiskJarCacheManager(
       }
     }
 
-    val tempFile = cacheDir.resolve("$cacheName.temp-${java.lang.Long.toUnsignedString(Random.nextLong(), Character.MAX_RADIX)}".takeLast(255))
+    val tempFile = cacheDir.resolve("$cacheName.t-${Integer.toUnsignedString(Random.nextInt(), Character.MAX_RADIX)}".takeLast(255))
     var fileMoved = false
     try {
       producer.produce(tempFile)
@@ -118,14 +120,14 @@ internal class LocalDiskJarCacheManager(
       }
     }
 
-    val sourceCacheItems = items.map { source ->
+    val sourceCacheItems = Array(items.size) { index ->
+      val source = items.get(index)
       SourceCacheItem(
-        path = source.name,
         size = source.getSize().toInt(),
         hash = source.getHash(),
         nativeFiles = (source.source as? ZipSource)?.let { nativeFiles?.get(it) } ?: emptyList(),
       )
-    }
+    }.asList()
 
     if (!producer.useCacheAsTargetFile) {
       Files.createDirectories(targetFile.parent)
@@ -158,12 +160,6 @@ internal class LocalDiskJarCacheManager(
   }
 }
 
-private fun hashCommonMeta(hash: HashStream64, items: List<SourceAndCacheStrategy>, targetFile: Path) {
-  hash.putByte(cacheVersion)
-  hash.putInt(items.size)
-  hash.putString(targetFile.fileName.toString())
-}
-
 private fun checkCache(cacheMetadataFile: Path,
                        cacheFile: Path,
                        sources: List<Source>,
@@ -207,7 +203,7 @@ private fun checkSavedAndActualSources(metadata: JarCacheItem, sources: List<Sou
   }
 
   for ((index, metadataItem) in metadata.sources.withIndex()) {
-    if (items.get(index).name != metadataItem.path) {
+    if (items.get(index).getHash() != metadataItem.hash) {
       return false
     }
   }
@@ -236,7 +232,6 @@ private class JarCacheItem(
 
 @Serializable
 private class SourceCacheItem(
-  @JvmField val path: String,
   @JvmField val size: Int,
   @JvmField val hash: Long,
   @JvmField val nativeFiles: List<String> = emptyList(),