jps-to-bazel: get rid of lib-lock.json for jps-to-bazel converter

actually, using only information from lib/MODULE.bazel is enough

GitOrigin-RevId: 8cfae08cd715f360da107c343e27e74c747074b2

platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/BazelBuildFileGenerator.kt

@@ -63,8 +63,7 @@ internal class BazelBuildFileGenerator(
   val ultimateRoot: Path?,
   val communityRoot: Path,
   private val project: JpsProject,
-  @JvmField val ultimateUrlCache: UrlCache?,
-  @JvmField val communityUrlCache: UrlCache,
+  val urlCache: UrlCache,
 ) {
   @JvmField
   val javaExtensionService: JpsJavaExtensionService = JpsJavaExtensionService.getInstance()
@@ -213,8 +212,7 @@ internal class BazelBuildFileGenerator(
         owner = owner,
         jarRepositories = jarRepositories,
         m2Repo = m2Repo,
-        ultimateUrlCache = ultimateUrlCache,
-        communityUrlCache = communityUrlCache,
+        urlCache = urlCache,
         moduleFileToLabelTracker = fileToLabelTracker,
         fileToUpdater = fileToUpdater,
       )

platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/JpsModuleToBazel.kt

@@ -47,15 +47,18 @@ internal class JpsModuleToBazel {
       val project = JpsSerializationManager.getInstance().loadProject(projectDir.toString(), mapOf("MAVEN_REPOSITORY" to m2Repo.toString()), true)
       val jarRepositories = loadJarRepositories(projectDir)
 
-      val ultimateUrlCache = ultimateRoot?.let { UrlCache(cacheFile = it.resolve("build/lib-lock.json")) }
-      val communityUrlCache = UrlCache(cacheFile = communityRoot.resolve("build/lib-lock.json"))
+      val modulesBazel = listOfNotNull(
+        ultimateRoot?.resolve("lib/MODULE.bazel"),
+        communityRoot.resolve("lib/MODULE.bazel"),
+      )
+
+      val urlCache = UrlCache(modulesBazel, jarRepositories)
 
       val generator = BazelBuildFileGenerator(
         ultimateRoot = ultimateRoot,
         communityRoot = communityRoot,
         project = project,
-        ultimateUrlCache = ultimateUrlCache,
-        communityUrlCache = communityUrlCache,
+        urlCache = urlCache,
       )
       val moduleList = generator.computeModuleList()
       // first, generate community to collect libs, that used by community (to separate community and ultimate libs)
@@ -64,6 +67,12 @@ internal class JpsModuleToBazel {
       generator.save(communityResult.moduleBuildFiles)
       generator.save(ultimateResult.moduleBuildFiles)
 
+      generator.generateLibs(jarRepositories = jarRepositories, m2Repo = m2Repo)
+
+      // Check that after all workings of generator, all checksums from urls with checksums
+      // are saved to MODULE.bazel correctly
+      verifyHttpFileTargetsGeneration(urlCache, modulesBazel, jarRepositories)
+
       deleteOldFiles(
         projectDir = communityRoot,
         generatedFiles = communityResult.moduleBuildFiles.keys
@@ -82,16 +91,46 @@ internal class JpsModuleToBazel {
         )
       }
 
-      generator.generateLibs(jarRepositories = jarRepositories, m2Repo = m2Repo)
-
       if (ultimateRoot != null) {
         val targetsFile = ultimateRoot.resolve("build/bazel-targets.json")
         saveTargets(targetsFile, communityResult.moduleTargets + ultimateResult.moduleTargets, moduleList.skippedModules)
       }
+    }
 
-      // save cache only on success. do not surround with try/finally
-      communityUrlCache.save()
-      ultimateUrlCache?.save()
+    private fun verifyHttpFileTargetsGeneration(
+      urlCache: UrlCache,
+      modulesBazel: List<Path>,
+      jarRepositories: List<JarRepository>,
+    ) {
+      val usedEntries = urlCache.getUsedEntries()
+      val mapOnDisk = readModules(modulesBazel, jarRepositories, warningsAsErrors = true)
+
+      if (mapOnDisk != usedEntries) {
+        for (path in usedEntries.keys - mapOnDisk.keys) {
+          error("Cannot find http_file for $path in $modulesBazel, but $path was used in maven libraries")
+        }
+
+        for (path in mapOnDisk.keys - usedEntries.keys) {
+          error("There is an http_file for $path in $modulesBazel, but $path was not used in jps-to-bazel")
+        }
+
+        for (path in mapOnDisk.keys.intersect(usedEntries.keys)) {
+          val onDisk = mapOnDisk[path]
+          val usedEntry = usedEntries[path]
+          if (onDisk != usedEntry) {
+            error(
+              "Different cache entries on disk ($modulesBazel) and what was used in jps-to-bazel." +
+              "on disk $onDisk, used entry $usedEntry"
+            )
+          }
+        }
+
+        // SHOULD NOT BE REACHED
+        error(
+          "http_file entries on disk in $modulesBazel are different from maven libraries used in jps-to-bazel." +
+          "Also, there is a bug in calculating difference between them."
+        )
+      }
     }
 
     @OptIn(ExperimentalSerializationApi::class)

platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/UrlCache.kt

@@ -5,11 +5,6 @@ package org.jetbrains.intellij.build.bazel
 
 import com.intellij.openapi.util.JDOMUtil
 import com.intellij.util.containers.orNull
-import kotlinx.serialization.ExperimentalSerializationApi
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.Transient
-import kotlinx.serialization.encodeToString
-import kotlinx.serialization.json.Json
 import org.jdom.Namespace
 import org.jetbrains.intellij.build.dependencies.BuildDependenciesConstants
 import org.jetbrains.intellij.build.dependencies.TeamCityHelper
@@ -22,19 +17,25 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.security.MessageDigest
-import java.util.*
+import java.util.Base64
 import kotlin.io.path.isRegularFile
-import kotlin.io.path.moveTo
+import kotlin.io.path.readText
 
-@Serializable
 internal data class CacheEntry(
   @JvmField val path: String,
   @JvmField val url: String,
   @JvmField val sha256: String,
-  @Transient @JvmField var used: Boolean = false,
 )
 
-internal data class JarRepository(@JvmField val url: String, @JvmField val isPrivate: Boolean)
+internal data class JarRepository(val url: String, val isPrivate: Boolean) {
+  init {
+    check(!url.endsWith("/")) {
+      "Repository URL must not end with '/': $url"
+    }
+  }
+
+  val urlWithSlash = "$url/"
+}
 
 private fun getAuthFromSystemProperties(): Pair<String, String>? {
   val username = System.getProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME)
@@ -110,39 +111,97 @@ private val authHeaderValue by lazy {
   "Basic " + Base64.getEncoder().encodeToString("${credentials.first}:${credentials.second}".toByteArray())
 }
 
-internal class UrlCache(val cacheFile: Path) {
-  private val httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).build()
+private val httpFileRegex = Regex(
+  "http_file\\(\\s+" +
+  "name\\s+=\\s+\"[^\"]+\",\\s+" +
+  "url\\s+=\\s+\"([^\"]+)\",\\s+" +
+  "sha256 = \"([0-9a-f]{64})\","
+)
 
-  private val cache: MutableMap<String, CacheEntry> by lazy {
-    if (Files.exists(cacheFile)) {
-      Json.decodeFromString<List<CacheEntry>>(Files.readString(cacheFile)).associateByTo(HashMap()) { it.path }
+internal fun readModules(modulesBazel: List<Path>, repositories: List<JarRepository>, warningsAsErrors: Boolean): Map<String, CacheEntry> {
+  fun warn(message: String) {
+    if (warningsAsErrors) {
+      error(message)
     }
     else {
-      HashMap<String, CacheEntry>()
+      println("WARN: $message")
     }
   }
 
-  @OptIn(ExperimentalSerializationApi::class)
-  fun save() {
-    val entries = cache.values.filter { it.used }.toTypedArray()
-    entries.sortBy { it.path }
-    val tempFile = Files.createTempFile(cacheFile.fileName.toString(), ".tmp")
-    @Suppress("JSON_FORMAT_REDUNDANT")
-    Files.writeString(tempFile, Json {
-      prettyPrint = true
-      prettyPrintIndent = "  "
-    }.encodeToString(entries))
-    tempFile.moveTo(target = cacheFile, overwrite = true)
+  val map: MutableMap<String, CacheEntry> = HashMap()
+
+  for (modulesFile in modulesBazel) {
+    if (!modulesFile.isRegularFile()) {
+      warn("File $modulesFile is not found")
+      continue
+    }
+
+    val modulesText = modulesFile.readText()
+    for (match in httpFileRegex.findAll(modulesText)) {
+      val (url, sha256) = match.destructured
+
+      val matchedRepositories = repositories.filter { url.startsWith(it.urlWithSlash) }
+      if (matchedRepositories.isEmpty()) {
+        warn("Cannot find repository for $url across all repositories: ${repositories.map { it.urlWithSlash }}")
+        continue
+      }
+      if (matchedRepositories.size > 1) {
+        warn("Multiple repositories match $url: ${matchedRepositories.map { it.urlWithSlash }}")
+        continue
+      }
+      val repository = matchedRepositories.single()
+
+      val path = url.removePrefix(repository.urlWithSlash)
+      check(path != url) {
+        "Unable to remove prefix '${repository.urlWithSlash}' from '$url'"
+      }
+
+      val existingEntry = map.get(path)
+      if (existingEntry != null) {
+        if (existingEntry.url != url) {
+          warn("Conflicting entries for $path: ${existingEntry.url} and $url")
+          map.remove(path)
+          continue
+        }
+        if (existingEntry.sha256 != sha256) {
+          warn("Conflicting entries for $path: ${existingEntry.sha256} and $sha256")
+          map.remove(path)
+          continue
+        }
+      }
+
+      map[path] = CacheEntry(path = path, url = url, sha256 = sha256)
+    }
   }
 
-  fun getEntry(jarPath: String): CacheEntry? = cache.get(jarPath)?.also { it.used = true }
+  println("DEBUG: read ${map.size} existing entries from $modulesBazel")
+
+  return map
+}
+
+internal class UrlCache(val modulesBazel: List<Path>, val repositories: List<JarRepository>) {
+  private val httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).build()
+  private val usedPaths = mutableSetOf<String>()
+
+  private val cache: MutableMap<String, CacheEntry> by lazy {
+    readModules(modulesBazel, repositories, warningsAsErrors = false).toMutableMap()
+  }
+
+  fun getEntry(jarPath: String): CacheEntry? {
+    usedPaths.add(jarPath)
+    return cache.get(jarPath)
+  }
 
   fun putUrl(jarPath: String, url: String, hash: String): CacheEntry {
-    val entry = CacheEntry(path = jarPath, url = url, sha256 = hash, used = true)
+    val entry = CacheEntry(path = jarPath, url = url, sha256 = hash)
     cache.put(jarPath, entry)
+    usedPaths.add(jarPath)
     return entry
   }
 
+  fun getUsedEntries(): Map<String, CacheEntry> =
+    cache.entries.filter { usedPaths.contains(it.key) }.associate { it.key to it.value }
+
   fun checkUrl(url: String, repo: JarRepository): Boolean {
     val requestBuilder = HttpRequest.newBuilder()
       .uri(URI.create(url))

platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/lib.kt

@@ -65,7 +65,7 @@ private fun getUrlAndSha256(jar: MavenFileDescription, jarRepositories: List<Jar
     error("Cannot find $jar in $jarRepositories (jarPath=$jarPath)")
   }
   check(jar.sha256checksum == null || entry.sha256 == jar.sha256checksum) {
-    "Hash mismatch: got ${jar.sha256checksum} from .idea/libraries, but ${entry.sha256} for ${jar.path} from url cache ${urlCache.cacheFile}"
+    "Hash mismatch: got ${jar.sha256checksum} from .idea/libraries, but ${entry.sha256} for ${jar.path} from lib/MODULE.bazel or community/lib/MODULE.bazel"
   }
   return entry
 }
@@ -167,8 +167,7 @@ internal fun generateBazelModuleSectionsForLibs(
   owner: LibOwnerDescriptor,
   jarRepositories: List<JarRepository>,
   m2Repo: Path,
-  ultimateUrlCache: UrlCache?,
-  communityUrlCache: UrlCache,
+  urlCache: UrlCache,
   moduleFileToLabelTracker: MutableMap<Path, MutableSet<String>>,
   fileToUpdater: MutableMap<Path, BazelFileUpdater>,
 ) {
@@ -182,16 +181,6 @@ internal fun generateBazelModuleSectionsForLibs(
   val labelTracker = moduleFileToLabelTracker.computeIfAbsent(owner.moduleFile) { HashSet() }
   buildFile(bazelFileUpdater, owner.sectionName) {
     for (lib in list) {
-      val urlCache = if (owner.isCommunity) {
-        communityUrlCache
-      } else {
-        check(ultimateUrlCache != null) {
-          error("generating library ${lib.lib.targetName} for ultimate part while ultimate root is missing")
-        }
-
-        ultimateUrlCache
-      }
-
       for (jar in lib.jars) {
         val label = fileToHttpRuleRepoName(jar.path)
         if (!labelTracker.add(label)) {