diff --git a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/BazelBuildFileGenerator.kt b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/BazelBuildFileGenerator.kt index 315d09da40b4..d27f71243519 100644 --- a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/BazelBuildFileGenerator.kt +++ b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/BazelBuildFileGenerator.kt @@ -63,8 +63,7 @@ internal class BazelBuildFileGenerator( val ultimateRoot: Path?, val communityRoot: Path, private val project: JpsProject, - @JvmField val ultimateUrlCache: UrlCache?, - @JvmField val communityUrlCache: UrlCache, + val urlCache: UrlCache, ) { @JvmField val javaExtensionService: JpsJavaExtensionService = JpsJavaExtensionService.getInstance() @@ -213,8 +212,7 @@ internal class BazelBuildFileGenerator( owner = owner, jarRepositories = jarRepositories, m2Repo = m2Repo, - ultimateUrlCache = ultimateUrlCache, - communityUrlCache = communityUrlCache, + urlCache = urlCache, moduleFileToLabelTracker = fileToLabelTracker, fileToUpdater = fileToUpdater, ) diff --git a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/JpsModuleToBazel.kt b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/JpsModuleToBazel.kt index c7ecc1ca7ec8..0d9a68ea95a7 100644 --- a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/JpsModuleToBazel.kt +++ b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/JpsModuleToBazel.kt @@ -47,15 +47,18 @@ internal class JpsModuleToBazel { val project = JpsSerializationManager.getInstance().loadProject(projectDir.toString(), mapOf("MAVEN_REPOSITORY" to m2Repo.toString()), true) val jarRepositories = loadJarRepositories(projectDir) - val ultimateUrlCache = ultimateRoot?.let { UrlCache(cacheFile = it.resolve("build/lib-lock.json")) } - val communityUrlCache = UrlCache(cacheFile = communityRoot.resolve("build/lib-lock.json")) + val modulesBazel = listOfNotNull( + ultimateRoot?.resolve("lib/MODULE.bazel"), + communityRoot.resolve("lib/MODULE.bazel"), + ) + + val urlCache = UrlCache(modulesBazel, jarRepositories) val generator = BazelBuildFileGenerator( ultimateRoot = ultimateRoot, communityRoot = communityRoot, project = project, - ultimateUrlCache = ultimateUrlCache, - communityUrlCache = communityUrlCache, + urlCache = urlCache, ) val moduleList = generator.computeModuleList() // first, generate community to collect libs, that used by community (to separate community and ultimate libs) @@ -64,6 +67,12 @@ internal class JpsModuleToBazel { generator.save(communityResult.moduleBuildFiles) generator.save(ultimateResult.moduleBuildFiles) + generator.generateLibs(jarRepositories = jarRepositories, m2Repo = m2Repo) + + // Check that after all workings of generator, all checksums from urls with checksums + // are saved to MODULE.bazel correctly + verifyHttpFileTargetsGeneration(urlCache, modulesBazel, jarRepositories) + deleteOldFiles( projectDir = communityRoot, generatedFiles = communityResult.moduleBuildFiles.keys @@ -82,16 +91,46 @@ internal class JpsModuleToBazel { ) } - generator.generateLibs(jarRepositories = jarRepositories, m2Repo = m2Repo) - if (ultimateRoot != null) { val targetsFile = ultimateRoot.resolve("build/bazel-targets.json") saveTargets(targetsFile, communityResult.moduleTargets + ultimateResult.moduleTargets, moduleList.skippedModules) } + } - // save cache only on success. do not surround with try/finally - communityUrlCache.save() - ultimateUrlCache?.save() + private fun verifyHttpFileTargetsGeneration( + urlCache: UrlCache, + modulesBazel: List, + jarRepositories: List, + ) { + val usedEntries = urlCache.getUsedEntries() + val mapOnDisk = readModules(modulesBazel, jarRepositories, warningsAsErrors = true) + + if (mapOnDisk != usedEntries) { + for (path in usedEntries.keys - mapOnDisk.keys) { + error("Cannot find http_file for $path in $modulesBazel, but $path was used in maven libraries") + } + + for (path in mapOnDisk.keys - usedEntries.keys) { + error("There is an http_file for $path in $modulesBazel, but $path was not used in jps-to-bazel") + } + + for (path in mapOnDisk.keys.intersect(usedEntries.keys)) { + val onDisk = mapOnDisk[path] + val usedEntry = usedEntries[path] + if (onDisk != usedEntry) { + error( + "Different cache entries on disk ($modulesBazel) and what was used in jps-to-bazel." + + "on disk $onDisk, used entry $usedEntry" + ) + } + } + + // SHOULD NOT BE REACHED + error( + "http_file entries on disk in $modulesBazel are different from maven libraries used in jps-to-bazel." + + "Also, there is a bug in calculating difference between them." + ) + } } @OptIn(ExperimentalSerializationApi::class) diff --git a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/UrlCache.kt b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/UrlCache.kt index a70d3bca3ad5..8e675cd52da4 100644 --- a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/UrlCache.kt +++ b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/UrlCache.kt @@ -5,11 +5,6 @@ package org.jetbrains.intellij.build.bazel import com.intellij.openapi.util.JDOMUtil import com.intellij.util.containers.orNull -import kotlinx.serialization.ExperimentalSerializationApi -import kotlinx.serialization.Serializable -import kotlinx.serialization.Transient -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json import org.jdom.Namespace import org.jetbrains.intellij.build.dependencies.BuildDependenciesConstants import org.jetbrains.intellij.build.dependencies.TeamCityHelper @@ -22,19 +17,25 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.security.MessageDigest -import java.util.* +import java.util.Base64 import kotlin.io.path.isRegularFile -import kotlin.io.path.moveTo +import kotlin.io.path.readText -@Serializable internal data class CacheEntry( @JvmField val path: String, @JvmField val url: String, @JvmField val sha256: String, - @Transient @JvmField var used: Boolean = false, ) -internal data class JarRepository(@JvmField val url: String, @JvmField val isPrivate: Boolean) +internal data class JarRepository(val url: String, val isPrivate: Boolean) { + init { + check(!url.endsWith("/")) { + "Repository URL must not end with '/': $url" + } + } + + val urlWithSlash = "$url/" +} private fun getAuthFromSystemProperties(): Pair? { val username = System.getProperty(BuildDependenciesConstants.JPS_AUTH_SPACE_USERNAME) @@ -110,39 +111,97 @@ private val authHeaderValue by lazy { "Basic " + Base64.getEncoder().encodeToString("${credentials.first}:${credentials.second}".toByteArray()) } -internal class UrlCache(val cacheFile: Path) { - private val httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).build() +private val httpFileRegex = Regex( + "http_file\\(\\s+" + + "name\\s+=\\s+\"[^\"]+\",\\s+" + + "url\\s+=\\s+\"([^\"]+)\",\\s+" + + "sha256 = \"([0-9a-f]{64})\"," +) - private val cache: MutableMap by lazy { - if (Files.exists(cacheFile)) { - Json.decodeFromString>(Files.readString(cacheFile)).associateByTo(HashMap()) { it.path } +internal fun readModules(modulesBazel: List, repositories: List, warningsAsErrors: Boolean): Map { + fun warn(message: String) { + if (warningsAsErrors) { + error(message) } else { - HashMap() + println("WARN: $message") } } - @OptIn(ExperimentalSerializationApi::class) - fun save() { - val entries = cache.values.filter { it.used }.toTypedArray() - entries.sortBy { it.path } - val tempFile = Files.createTempFile(cacheFile.fileName.toString(), ".tmp") - @Suppress("JSON_FORMAT_REDUNDANT") - Files.writeString(tempFile, Json { - prettyPrint = true - prettyPrintIndent = " " - }.encodeToString(entries)) - tempFile.moveTo(target = cacheFile, overwrite = true) + val map: MutableMap = HashMap() + + for (modulesFile in modulesBazel) { + if (!modulesFile.isRegularFile()) { + warn("File $modulesFile is not found") + continue + } + + val modulesText = modulesFile.readText() + for (match in httpFileRegex.findAll(modulesText)) { + val (url, sha256) = match.destructured + + val matchedRepositories = repositories.filter { url.startsWith(it.urlWithSlash) } + if (matchedRepositories.isEmpty()) { + warn("Cannot find repository for $url across all repositories: ${repositories.map { it.urlWithSlash }}") + continue + } + if (matchedRepositories.size > 1) { + warn("Multiple repositories match $url: ${matchedRepositories.map { it.urlWithSlash }}") + continue + } + val repository = matchedRepositories.single() + + val path = url.removePrefix(repository.urlWithSlash) + check(path != url) { + "Unable to remove prefix '${repository.urlWithSlash}' from '$url'" + } + + val existingEntry = map.get(path) + if (existingEntry != null) { + if (existingEntry.url != url) { + warn("Conflicting entries for $path: ${existingEntry.url} and $url") + map.remove(path) + continue + } + if (existingEntry.sha256 != sha256) { + warn("Conflicting entries for $path: ${existingEntry.sha256} and $sha256") + map.remove(path) + continue + } + } + + map[path] = CacheEntry(path = path, url = url, sha256 = sha256) + } } - fun getEntry(jarPath: String): CacheEntry? = cache.get(jarPath)?.also { it.used = true } + println("DEBUG: read ${map.size} existing entries from $modulesBazel") + + return map +} + +internal class UrlCache(val modulesBazel: List, val repositories: List) { + private val httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).build() + private val usedPaths = mutableSetOf() + + private val cache: MutableMap by lazy { + readModules(modulesBazel, repositories, warningsAsErrors = false).toMutableMap() + } + + fun getEntry(jarPath: String): CacheEntry? { + usedPaths.add(jarPath) + return cache.get(jarPath) + } fun putUrl(jarPath: String, url: String, hash: String): CacheEntry { - val entry = CacheEntry(path = jarPath, url = url, sha256 = hash, used = true) + val entry = CacheEntry(path = jarPath, url = url, sha256 = hash) cache.put(jarPath, entry) + usedPaths.add(jarPath) return entry } + fun getUsedEntries(): Map = + cache.entries.filter { usedPaths.contains(it.key) }.associate { it.key to it.value } + fun checkUrl(url: String, repo: JarRepository): Boolean { val requestBuilder = HttpRequest.newBuilder() .uri(URI.create(url)) diff --git a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/lib.kt b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/lib.kt index b68fec7e572b..963f87d5231b 100644 --- a/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/lib.kt +++ b/platform/build-scripts/bazel/src/org/jetbrains/intellij/build/bazel/lib.kt @@ -65,7 +65,7 @@ private fun getUrlAndSha256(jar: MavenFileDescription, jarRepositories: List, m2Repo: Path, - ultimateUrlCache: UrlCache?, - communityUrlCache: UrlCache, + urlCache: UrlCache, moduleFileToLabelTracker: MutableMap>, fileToUpdater: MutableMap, ) { @@ -182,16 +181,6 @@ internal fun generateBazelModuleSectionsForLibs( val labelTracker = moduleFileToLabelTracker.computeIfAbsent(owner.moduleFile) { HashSet() } buildFile(bazelFileUpdater, owner.sectionName) { for (lib in list) { - val urlCache = if (owner.isCommunity) { - communityUrlCache - } else { - check(ultimateUrlCache != null) { - error("generating library ${lib.lib.targetName} for ultimate part while ultimate root is missing") - } - - ultimateUrlCache - } - for (jar in lib.jars) { val label = fileToHttpRuleRepoName(jar.path) if (!labelTracker.add(label)) {