rebuild in case of corrupted storage (test - delete some file on disk, we should rebuild in this case)

GitOrigin-RevId: e34b7183f8c6008e305207f6e9aedf1ab4016180
2026-10-02 00:57:59 +07:00 · 2025-02-27 10:53:27 +00:00
parent 302057c346
commit 81bce2d6bc
12 changed files with 428 additions and 288 deletions
@@ -59,9 +59,9 @@ archive_override(

 http_file(
    name = "jps-standalone_http",
-    url = "https://packages.jetbrains.team/files/p/ij/intellij-build-dependencies/jps/jps-build-standalone-25.02.25.jar",
-    downloaded_file_path = "jps-build-standalone-19.02.25.jar",
-    integrity = "sha256-unxZmzPgZrqUU7rW7HTeR9WGnT6fqi9iqUH3olfCwkg=",
+    url = "https://packages.jetbrains.team/files/p/ij/intellij-build-dependencies/jps/jps-build-standalone-27.02.25.jar",
+    downloaded_file_path = "jps-build-standalone-27.02.25.jar",
+    integrity = "sha256-WIGBhdShgsW93qar4Hj0GcrIbuNfqAkrtjGASpDasfM=",
 )

 http_archive(
@@ -73,7 +73,7 @@ internal fun loadJpsModel(
  val digests = TargetConfigurationDigestContainer()
  digests.set(TargetConfigurationDigestProperty.KOTLIN_VERSION, KOTLINC_VERSION_HASH)
  digests.set(TargetConfigurationDigestProperty.JPS_TRACK_LIB_DEPS, if (isLibTracked) 1 else 0)
-  digests.set(TargetConfigurationDigestProperty.TOOL_VERSION, 6)
+  digests.set(TargetConfigurationDigestProperty.TOOL_VERSION, 7)

  // properties not needed for us (not implemented for java)
  // extension.loadModuleOptions not needed for us (not implemented for java)
@@ -11,9 +11,6 @@ import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.DelicateCoroutinesApi
 import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.GlobalScope
-import kotlinx.coroutines.async
-import kotlinx.coroutines.future.asCompletableFuture
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
 import org.apache.arrow.memory.RootAllocator
@@ -41,13 +38,12 @@ import org.jetbrains.bazel.jvm.jps.state.TargetConfigurationDigestContainer
 import org.jetbrains.bazel.jvm.jps.state.TargetConfigurationDigestProperty
 import org.jetbrains.bazel.jvm.jps.state.createInitialSourceMap
 import org.jetbrains.bazel.jvm.jps.state.saveBuildState
-import org.jetbrains.bazel.jvm.jps.storage.AsyncExecutor
+import org.jetbrains.bazel.jvm.jps.storage.StorageInitializer
 import org.jetbrains.bazel.jvm.kotlin.JvmBuilderFlags
 import org.jetbrains.bazel.jvm.kotlin.parseArgs
 import org.jetbrains.bazel.jvm.span
 import org.jetbrains.bazel.jvm.use
 import org.jetbrains.jps.backwardRefs.JavaBackwardReferenceIndexBuilder
-import org.jetbrains.jps.incremental.RebuildRequestedException
 import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
 import org.jetbrains.jps.incremental.storage.BuildDataManager
 import org.jetbrains.jps.model.JpsModel
@@ -57,7 +53,6 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardCopyOption
 import java.util.concurrent.CancellationException
-import java.util.concurrent.CompletableFuture
 import kotlin.coroutines.coroutineContext

 internal suspend fun incrementalBuild(
@@ -143,11 +138,15 @@ suspend fun buildUsingJps(
  val typeAwareRelativizer = relativizer.typeAwareRelativizer as BazelPathTypeAwareRelativizer
  val log = RequestLog(out = out, parentSpan = parentSpan, tracer = tracer, relativizer = typeAwareRelativizer)

-  val abiJar = args.optionalSingle(JvmBuilderFlags.ABI_OUT)?.let { baseDir.resolve(it).normalize() }
  val outJar = baseDir.resolve(args.mandatorySingle(JvmBuilderFlags.OUT)).normalize()
  val prefix = outJar.fileName.toString().removeSuffix(".jar")
  val bazelOutDir = outJar.parent
  val dataDir = bazelOutDir.resolve("$cachePrefix$prefix-jps-data")
+  val outputs = OutputFiles(
+    outJar = outJar,
+    abiJar = args.optionalSingle(JvmBuilderFlags.ABI_OUT)?.let { baseDir.resolve(it).normalize() },
+    dataDir = dataDir,
+  )

  val (jpsModel, targetDigests) = loadJpsModel(
    sources = sources,
@@ -166,7 +165,7 @@ suspend fun buildUsingJps(
  if (isDebugEnabled) {
    parentSpan.setAttribute("isIncrementalCompilation", isIncrementalCompilation)
    parentSpan.setAttribute("outJar", outJar.toString())
-    parentSpan.setAttribute("abiJar", abiJar?.toString() ?: "")
+    parentSpan.setAttribute("abiJar", outputs.abiJar?.toString() ?: "")
    for (kind in TargetConfigurationDigestProperty.entries) {
      parentSpan.setAttribute(kind.name, targetDigests.get(kind))
    }
@@ -185,7 +184,7 @@ suspend fun buildUsingJps(
    )
  }

-  var rebuildReason = validateFileExistence(outJar = outJar, abiJar = abiJar, cacheDir = dataDir)
+  var rebuildReason = validateFileExistence(outputs = outputs, cacheDir = dataDir)
  val buildStateFile = dataDir.resolve("$prefix-state-v1.arrow")
  val depStateStorageFile = dataDir.resolve("$prefix-lib-roots-v2.arrow")
  val classPath = moduleTarget.module.container.getChild(BazelConfigurationHolder.KIND).classPath
@@ -236,8 +235,7 @@ suspend fun buildUsingJps(
      dataDir = dataDir,
      targetDigests = targetDigests,
      moduleTarget = moduleTarget,
-      outJar = outJar,
-      abiJar = abiJar,
+      outputs = outputs,
      relativizer = relativizer,
      jpsModel = jpsModel,
      dataManager = BazelBuildDataProvider(
@@ -257,12 +255,15 @@ suspend fun buildUsingJps(
      isDebugEnabled = isDebugEnabled,
    )
  }
-  catch (e: RebuildRequestedException) {
+  catch (e: CancellationException) {
+    throw e
+  }
+  catch (e: Throwable) {
    if (isDebugEnabled) {
      log.out.appendLine("rebuild requested: ${e.stackTraceToString()}")
    }
    parentSpan.recordException(e)
-    rebuildReason = e.cause!!.message
+    rebuildReason = e.cause?.message ?: e.message ?: "unknown error"
    -1
  }

@@ -275,8 +276,7 @@ suspend fun buildUsingJps(
      dataDir = dataDir,
      targetDigests = targetDigests,
      moduleTarget = moduleTarget,
-      outJar = outJar,
-      abiJar = abiJar,
+      outputs = outputs,
      relativizer = relativizer,
      jpsModel = jpsModel,
      dataManager = BazelBuildDataProvider(
@@ -316,6 +316,15 @@ internal fun createJavaBuilder(
  }
 }

+internal class OutputFiles(
+  @JvmField val outJar: Path,
+  @JvmField val abiJar: Path?,
+  dataDir: Path,
+) {
+  @JvmField val cachedJar: Path = dataDir.resolve(outJar.fileName)
+  @JvmField val cachedAbiJar: Path? = abiJar?.let { dataDir.resolve(it.fileName) }
+}
+
@OptIn(DelicateCoroutinesApi::class)
 private suspend fun initAndBuild(
  rebuildReason: String?,
@@ -324,8 +333,7 @@ private suspend fun initAndBuild(
  dataDir: Path,
  targetDigests: TargetConfigurationDigestContainer,
  moduleTarget: BazelModuleBuildTarget,
-  outJar: Path,
-  abiJar: Path?,
+  outputs: OutputFiles,
  relativizer: PathRelativizerService,
  jpsModel: JpsModel,
  dataManager: BazelBuildDataProvider,
@@ -340,20 +348,10 @@ private suspend fun initAndBuild(
    .setAttribute("isRebuild", isRebuild)
    .setAttribute("rebuildReason", rebuildReason ?: "")
    .use { span ->
-      if (isRebuild) {
-        storageInitializer.clearAndInit(span)
-      }
-
      storageInitializer.createBuildDataManager(
-        jpsModel = jpsModel,
-        moduleTarget = moduleTarget,
+        isRebuild = isRebuild,
        relativizer = relativizer,
        buildDataProvider = dataManager,
-        executor = object : AsyncExecutor {
-          override fun <T> execute(action: () -> T): CompletableFuture<T> {
-            return GlobalScope.async { action() }.asCompletableFuture()
-          }
-        },
        span = span,
      )
    }
@@ -365,9 +363,6 @@ private suspend fun initAndBuild(
      coroutineContext = coroutineContext,
    )

-    val cachedJar = dataDir.resolve(outJar.fileName)
-    val cachedAbiJar = abiJar?.let { dataDir.resolve(it.fileName) }
-
    val oldJar: Path?
    val oldAbiJar: Path?
    if (isRebuild) {
@@ -375,11 +370,11 @@ private suspend fun initAndBuild(
      oldAbiJar = null
    }
    else {
-      oldJar = cachedJar.takeIf { Files.exists(it) }
-      oldAbiJar = if (cachedJar == null || cachedAbiJar == null) null else cachedAbiJar.takeIf { Files.exists(it) }
+      oldJar = outputs.cachedJar.takeIf { Files.exists(it) }
+      oldAbiJar = if (oldJar == null || outputs.cachedAbiJar == null) null else outputs.cachedAbiJar.takeIf { Files.exists(it) }
    }

-    createOutputSink(oldJar = oldJar, oldAbiJar = oldAbiJar, withAbi = abiJar != null).use { outputSink ->
+    createOutputSink(oldJar = oldJar, oldAbiJar = oldAbiJar, withAbi = outputs.abiJar != null).use { outputSink ->
      val exitCode = tracer.spanBuilder("compile")
        .setAttribute(AttributeKey.booleanKey("isRebuild"), isRebuild)
        .use { span ->
@@ -431,10 +426,7 @@ private suspend fun initAndBuild(
            buildDataManager = buildDataManager,
            success = exitCode == 0,
            moduleTarget = moduleTarget,
-            outJar = outJar,
-            abiJar = abiJar,
-            cachedJar = cachedJar,
-            cachedAbiJar = cachedAbiJar,
+            outputs = outputs,
            context = context,
            targetDigests = targetDigests,
            buildDataProvider = dataManager,
@@ -475,10 +467,7 @@ private val stateFileMetaNames: Array<String> = TargetConfigurationDigestPropert

 private fun CoroutineScope.postBuild(
  moduleTarget: BazelModuleBuildTarget,
-  outJar: Path,
-  abiJar: Path?,
-  cachedJar: Path,
-  cachedAbiJar: Path?,
+  outputs: OutputFiles,
  context: BazelCompileContext,
  targetDigests: TargetConfigurationDigestContainer,
  buildDataProvider: BazelBuildDataProvider,
@@ -521,8 +510,8 @@ private fun CoroutineScope.postBuild(
      writeJarAndAbi(
        tracer = requestLog.tracer,
        outputSink = outputSink,
-        outJar = cachedJar,
-        abiJar = cachedAbiJar,
+        outJar = outputs.cachedJar,
+        abiJar = outputs.cachedAbiJar,
        sourceDescriptors = sourceDescriptors,
      )
    }
@@ -532,9 +521,9 @@ private fun CoroutineScope.postBuild(

    // copy to output
    withContext(Dispatchers.IO) {
-      Files.copy(cachedJar, outJar, StandardCopyOption.REPLACE_EXISTING)
-      if (abiJar != null) {
-        Files.copy(cachedAbiJar, abiJar, StandardCopyOption.REPLACE_EXISTING)
+      Files.copy(outputs.cachedJar, outputs.outJar, StandardCopyOption.REPLACE_EXISTING)
+      if (outputs.abiJar != null) {
+        Files.copy(outputs.cachedAbiJar, outputs.abiJar, StandardCopyOption.REPLACE_EXISTING)
      }
    }
  }
@@ -1,72 +0,0 @@
-// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-@file:Suppress("UnstableApiUsage", "ReplaceGetOrSet")
-
-package org.jetbrains.bazel.jvm.jps
-
-import com.intellij.openapi.util.io.FileUtilRt
-import io.opentelemetry.api.common.AttributeKey
-import io.opentelemetry.api.common.Attributes
-import io.opentelemetry.api.trace.Span
-import org.jetbrains.bazel.jvm.jps.impl.BazelBuildDataProvider
-import org.jetbrains.bazel.jvm.jps.impl.BazelModuleBuildTarget
-import org.jetbrains.bazel.jvm.jps.impl.createDataManager
-import org.jetbrains.bazel.jvm.jps.storage.AsyncExecutor
-import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
-import org.jetbrains.jps.incremental.storage.BuildDataManager
-import org.jetbrains.jps.model.JpsModel
-import java.nio.file.Files
-import java.nio.file.Path
-
-internal class StorageInitializer(private val dataDir: Path) {
-  private var wasCleared = false
-
-  fun clearAndInit(@Suppress("unused") span: Span) {
-    clearStorage()
-    Files.createDirectories(dataDir)
-  }
-
-  fun createBuildDataManager(
-    jpsModel: JpsModel,
-    moduleTarget: BazelModuleBuildTarget,
-    relativizer: PathRelativizerService,
-    buildDataProvider: BazelBuildDataProvider,
-    span: Span,
-    executor: AsyncExecutor,
-  ): BuildDataManager {
-    try {
-      return createDataManager(
-        dataStorageRoot = dataDir,
-        // alwaysScanFS doesn't matter, we use our own version of `BuildOperations.ensureFSStateInitialized`,
-        // see `JpsProjectBuilder.ensureFsStateInitialized`
-        relativizer = relativizer,
-        buildDataProvider = buildDataProvider,
-        executor = executor,
-      )
-    }
-    catch (e: Throwable) {
-      //storageManager!!.forceClose()
-      if (wasCleared) {
-        throw e
-      }
-
-      span.recordException(e, Attributes.of(AttributeKey.stringKey("message"), "cannot open cache storage"))
-    }
-
-    clearStorage()
-
-    return createBuildDataManager(
-      jpsModel = jpsModel,
-      moduleTarget = moduleTarget,
-      relativizer = relativizer,
-      buildDataProvider = buildDataProvider,
-      executor = executor,
-      span = span,
-    )
-  }
-
-  fun clearStorage() {
-    wasCleared = true
-    // todo rename and store
-    FileUtilRt.deleteRecursively(dataDir)
-  }
-}
@@ -22,7 +22,6 @@ import org.jetbrains.jps.dependency.ReferenceID
 import org.jetbrains.jps.dependency.impl.DependencyGraphImpl
 import org.jetbrains.jps.dependency.impl.PathSourceMapper
 import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
-import org.jetbrains.jps.incremental.storage.BuildDataManager.Companion.PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.Future
 import java.util.concurrent.locks.ReentrantReadWriteLock
@@ -31,33 +30,21 @@ import java.util.function.Function
 import kotlin.concurrent.read
 import kotlin.concurrent.write

-private val processConstantsIncrementally = !System.getProperty(PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY, "false").toBoolean()
+private val processConstantsIncrementally = !System.getProperty("compiler.process.constants.non.incremental", "false").toBoolean()

-class BuildDataManager internal constructor(
+internal class BuildDataManager private constructor(
  val dataPaths: BuildDataPaths,
  @get:Deprecated("Use {@link #getTargetStateManager()} or, preferably, avoid using internal APIs.") val targetsState: BuildTargetsState,
  val relativizer: PathRelativizerService,
  private val dataManager: BuildDataProvider,
-  containerFactory: BazelPersistentMapletFactory,
+  depGraph: DependencyGraph,
 ) {
-  private val depGraph: DependencyGraph
+  private val depGraph = SynchronizedDependencyGraph(depGraph)
  private val depGraphPathMapper: NodeSourcePathMapper

  private val targetToStorages = ConcurrentHashMap<Pair<BuildTarget<*>, StorageProvider<StorageOwner>>, StorageOwner>()

  init {
-    try {
-      depGraph = SynchronizedDependencyGraph(DependencyGraphImpl(containerFactory))
-    }
-    catch (e: Throwable) {
-      try {
-        close()
-      }
-      catch (_: Throwable) {
-      }
-      throw e
-    }
-
    val typeAwareRelativizer = relativizer.typeAwareRelativizer!!
    depGraphPathMapper = PathSourceMapper(
      Function { typeAwareRelativizer.toAbsolute(it, RelativePathType.SOURCE) },
@@ -65,6 +52,25 @@ class BuildDataManager internal constructor(
    )
  }

+  companion object {
+    fun open(
+      containerFactory: BazelPersistentMapletFactory,
+      dataPaths: BuildDataPaths,
+      relativizer: PathRelativizerService,
+      targetState: BuildTargetsState,
+      dataManager: BuildDataProvider,
+    ): BuildDataManager {
+      val depGraph = DependencyGraphImpl(containerFactory)
+      return BuildDataManager(
+        dataPaths = dataPaths,
+        targetsState = targetState,
+        dataManager = dataManager,
+        relativizer = relativizer,
+        depGraph = depGraph,
+      )
+    }
+  }
+
  @Suppress("unused")
  fun getMappings(): Mappings? = null

@@ -165,10 +171,6 @@ class BuildDataManager internal constructor(
  fun reportUnhandledRelativizerPaths() {
    relativizer.reportUnhandledPaths()
  }
-
-  companion object {
-    const val PROCESS_CONSTANTS_NON_INCREMENTAL_PROPERTY: String = "compiler.process.constants.non.incremental"
-  }
 }

 private class SynchronizedDependencyGraph(private val delegate: DependencyGraph) : DependencyGraph {
@@ -3,54 +3,20 @@

 package org.jetbrains.bazel.jvm.jps.impl

-import org.jetbrains.bazel.jvm.jps.storage.AsyncExecutor
-import org.jetbrains.bazel.jvm.jps.storage.BazelPersistentMapletFactory
 import org.jetbrains.jps.builders.BuildTarget
 import org.jetbrains.jps.builders.BuildTargetIndex
 import org.jetbrains.jps.builders.BuildTargetRegistry.ModuleTargetSelector
 import org.jetbrains.jps.builders.BuildTargetType
 import org.jetbrains.jps.builders.impl.BuildTargetChunk
 import org.jetbrains.jps.builders.logging.BuildLoggingManager
-import org.jetbrains.jps.builders.storage.BuildDataPaths
 import org.jetbrains.jps.cmdline.ProjectDescriptor
 import org.jetbrains.jps.incremental.CompileContext
 import org.jetbrains.jps.incremental.ModuleBuildTarget
 import org.jetbrains.jps.incremental.fs.BuildFSState
-import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
 import org.jetbrains.jps.incremental.storage.BuildDataManager
-import org.jetbrains.jps.incremental.storage.BuildTargetsState
 import org.jetbrains.jps.indices.IgnoredFileIndex
 import org.jetbrains.jps.model.JpsModel
 import org.jetbrains.jps.model.module.JpsModule
-import java.nio.file.Path
-
-private class BazelBuildDataPaths(private val dir: Path) : BuildDataPaths {
-  override fun getDataStorageDir() = dir
-
-  override fun getTargetsDataRoot(): Path = dir
-
-  override fun getTargetTypeDataRootDir(targetType: BuildTargetType<*>): Path = dir.resolve(targetType.typeId)
-
-  override fun getTargetDataRootDir(target: BuildTarget<*>): Path = dir.resolve(target.targetType.typeId)
-
-  override fun getTargetDataRoot(targetType: BuildTargetType<*>, targetId: String): Path = dir
-}
-
-internal fun createDataManager(
-  dataStorageRoot: Path,
-  relativizer: PathRelativizerService,
-  buildDataProvider: BazelBuildDataProvider,
-  executor: AsyncExecutor,
-): BuildDataManager {
-  val containerFactory = BazelPersistentMapletFactory(dataStorageRoot.resolve("mappings-graph"), executor)
-  return BuildDataManager(
-    dataPaths = BazelBuildDataPaths(dataStorageRoot),
-    targetsState = BuildTargetsState(BazelBuildTargetStateManager),
-    relativizer = relativizer,
-    dataManager = buildDataProvider,
-    containerFactory = containerFactory,
-  )
-}

 internal fun createJpsProjectDescriptor(
  dataManager: BuildDataManager,
@@ -22,11 +22,11 @@ private const val thresholdPercentage = 0.5

 // if output jar doesn't exist, make sure that we do not to use existing cache -
 // set `isRebuild` to true and clear caches in this case
-internal fun validateFileExistence(outJar: Path, abiJar: Path?, cacheDir: Path): String? {
+internal fun validateFileExistence(outputs: OutputFiles, cacheDir: Path): String? {
  return when {
    !Files.isDirectory(cacheDir) -> "cache dir doesn't exist"
-    Files.notExists(cacheDir.resolve(outJar.fileName)) -> "cached output jar doesn't exist"
-    abiJar != null && Files.notExists(cacheDir.resolve(abiJar.fileName)) -> {
+    Files.notExists(outputs.cachedJar) -> "cached output jar doesn't exist"
+    outputs.cachedAbiJar != null && Files.notExists(outputs.cachedAbiJar) -> {
      "cached output JAR exists but not ABI JAR - something wrong, or we enabled ABI JARs"
    }
    else -> null
@@ -0,0 +1,55 @@
+@file:Suppress("UnstableApiUsage")
+
+package org.jetbrains.bazel.jvm.jps.storage
+
+import com.intellij.platform.util.io.storages.DataExternalizerEx
+import com.intellij.platform.util.io.storages.KeyDescriptorEx
+import com.intellij.platform.util.io.storages.durablemap.DurableMap
+import com.intellij.platform.util.io.storages.durablemap.DurableMapFactory
+import com.intellij.util.io.DataExternalizer
+import com.intellij.util.io.KeyDescriptor
+import org.jetbrains.jps.dependency.Maplet
+import java.nio.file.Path
+
+internal class DurableMapMaplet<K : Any, V : Any>(
+  mapFile: Path,
+  keyDescriptor: KeyDescriptor<K>,
+  valueExternalizer: DataExternalizer<V>
+) : Maplet<K, V> {
+  private val map: DurableMap<K, V> = DurableMapFactory
+    .withDefaults<K, V>(KeyDescriptorEx.adapt(keyDescriptor), DataExternalizerEx.adapt(valueExternalizer))
+    .open(mapFile)
+
+  override fun containsKey(key: K): Boolean {
+    return map.containsMapping(key)
+  }
+
+  override fun get(key: K): V? {
+    return map.get(key)
+  }
+
+  override fun put(key: K, value: V?) {
+    map.put(key, value)
+  }
+
+  override fun remove(key: K) {
+    map.remove(key)
+  }
+
+  override fun getKeys(): Iterable<K> {
+    val result = ArrayList<K>(map.size())
+    map.processKeys {
+      result.add(it)
+      true
+    }
+    return result
+  }
+
+  override fun close() {
+    map.close()
+  }
+
+  override fun flush() {
+    map.force()
+  }
+}
@@ -0,0 +1,147 @@
+@file:Suppress("UnstableApiUsage")
+
+package org.jetbrains.bazel.jvm.jps.storage
+
+import com.intellij.util.io.AppendablePersistentMap
+import com.intellij.util.io.DataExternalizer
+import com.intellij.util.io.KeyDescriptor
+import com.intellij.util.io.PersistentMapBase
+import com.intellij.util.io.PersistentMapBuilder
+import org.jetbrains.jps.dependency.MultiMaplet
+import java.io.DataInput
+import java.io.DataInputStream
+import java.io.DataOutput
+import java.nio.file.Path
+import kotlin.math.absoluteValue
+
+internal class DurablePersistentMultiMaplet<K : Any, V : Any, C : MutableCollection<V>>(
+  mapFile: Path,
+  keyDescriptor: KeyDescriptor<K>,
+  valueExternalizer: DataExternalizer<V>,
+  private val collectionFactory: () -> C,
+  private val emptyCollection: Collection<V>,
+) : MultiMaplet<K, V> {
+  private val map: PersistentMapBase<K, C>
+  private val valueExternalizer: DataExternalizer<V>
+
+  init {
+    this.valueExternalizer = valueExternalizer
+
+    val builder = PersistentMapBuilder.newBuilder<K, C>(mapFile, keyDescriptor, object : DataExternalizer<C> {
+      override fun save(out: DataOutput, data: C) {
+        out.writeInt(data.size)
+        for (value in data) {
+          valueExternalizer.save(out, value)
+        }
+      }
+
+      override fun read(`in`: DataInput): C {
+        val result = collectionFactory()
+        val stream = `in` as DataInputStream
+        while (stream.available() > 0) {
+          var size = stream.readInt()
+          val isRemoval = size < 0
+          size = size.absoluteValue
+          while (size-- > 0) {
+            val v = valueExternalizer.read(stream)
+            if (isRemoval) {
+              result.remove(v)
+            }
+            else {
+              result.add(v)
+            }
+          }
+        }
+        return result
+      }
+    })
+    map = builder.buildImplementation()
+  }
+
+  override fun containsKey(key: K?): Boolean {
+    return map.containsKey(key)
+  }
+
+  override fun get(key: K): Collection<V> {
+    return map.get(key) ?: return emptyCollection
+  }
+
+  override fun put(key: K?, values: Iterable<V>) {
+    val data = ensureCollection(values)
+    if (data.isEmpty()) {
+      map.remove(key)
+    }
+    else {
+      map.put(key, data)
+    }
+  }
+
+  @Suppress("UNCHECKED_CAST")
+  private fun ensureCollection(seq: Iterable<V>): C {
+    return when {
+      emptyCollection is MutableSet<*> && seq is MutableSet<*> -> seq as C
+      emptyCollection is MutableList<*> && seq is MutableList<*> -> seq as C
+      else -> seq.toCollection(collectionFactory())
+    }
+  }
+
+  override fun remove(key: K?) {
+    map.remove(key)
+  }
+
+  override fun appendValue(key: K, value: V) {
+    map.appendData(key, AppendablePersistentMap.ValueDataAppender { out ->
+      out.writeInt(1)
+      valueExternalizer.save(out, value)
+    })
+  }
+
+  override fun appendValues(key: K, values: Iterable<V>) {
+    val size = values.count()
+    if (size == 0) {
+      return
+    }
+
+    map.appendData(key, AppendablePersistentMap.ValueDataAppender { out ->
+      out.writeInt(size)
+      for (v in values) {
+        valueExternalizer.save(out, v)
+      }
+    })
+  }
+
+  override fun removeValue(key: K, value: V) {
+    removeValues(key, listOf(value))
+  }
+
+  override fun removeValues(key: K?, values: Iterable<V>) {
+    val size = values.count()
+    if (size == 0) {
+      return
+    }
+
+    map.appendData(key, AppendablePersistentMap.ValueDataAppender { out ->
+      out.writeInt(-size)
+      for (v in values) {
+        valueExternalizer.save(out, v)
+      }
+    })
+  }
+
+  override fun getKeys(): Iterable<K> {
+    val result = ArrayList<K>(map.keysCount())
+    map.processExistingKeys {
+      result.add(it)
+      true
+    }
+    return result
+  }
+
+  override fun close() {
+    map.close()
+  }
+
+  override fun flush() {
+    map.force()
+  }
+}
@@ -2,7 +2,6 @@

 package org.jetbrains.bazel.jvm.jps.storage

-import com.intellij.openapi.diagnostic.logger
 import com.intellij.openapi.util.IntRef
 import com.intellij.platform.util.io.storages.appendonlylog.AppendOnlyLog
 import com.intellij.platform.util.io.storages.appendonlylog.AppendOnlyLogFactory
@@ -14,42 +13,26 @@ import com.intellij.util.io.IOUtil
 import com.intellij.util.io.Unmappable
 import com.intellij.util.io.blobstorage.ByteBufferReader
 import org.jetbrains.annotations.Nullable
-import java.nio.ByteBuffer
 import java.nio.file.Path
-import java.util.concurrent.CancellationException
-import java.util.concurrent.CompletableFuture
 import java.util.function.IntPredicate

-internal class DurableStringEnumerator(
+internal class DurableStringEnumerator private constructor(
  private val valuesLog: AppendOnlyLog,
-  private val valueHashToIdFuture: CompletableFuture<Int2IntMultimap>,
+  private val valueHashToId: Int2IntMultimap,
 ) : DurableDataEnumerator<String>, Unmappable, CleanableStorage {
  private val valueHashLock = Any()

-  /** lazily initialized in [valueHashToId] */
-  private var valueHashToId: Int2IntMultimap? = null
-
  companion object {
-    const val DATA_FORMAT_VERSION: Int = 1
-    const val PAGE_SIZE: Int = 8 shl 20
-
    private val VALUES_LOG_FACTORY = AppendOnlyLogFactory
      .withDefaults()
-      .pageSize(PAGE_SIZE)
-      .failIfDataFormatVersionNotMatch(DATA_FORMAT_VERSION)
+      .failIfDataFormatVersionNotMatch(1)
      .checkIfFileCompatibleEagerly(true)
      .cleanIfFileIncompatible()

-    //fun open(storagePath: Path): DurableStringEnumerator {
-    //  return VALUES_LOG_FACTORY.wrapStorageSafely<DurableStringEnumerator, IOException>(
-    //    storagePath,
-    //    { valuesLog -> DurableStringEnumerator(valuesLog, buildValueToIdIndex(valuesLog)) }
-    //  )
-    //}
-
-    fun openAsync(storagePath: Path, executor: AsyncExecutor): DurableStringEnumerator {
-      return VALUES_LOG_FACTORY.wrapStorageSafely<DurableStringEnumerator, RuntimeException>(storagePath) { valuesLog ->
-        DurableStringEnumerator(valuesLog, executor.execute { buildValueToIdIndex(valuesLog) })
+    fun open(storagePath: Path): DurableStringEnumerator {
+      return executeOrCloseStorage(VALUES_LOG_FACTORY.open(storagePath)) { valuesLog ->
+        val valueHashToId = buildValueToIdIndex(valuesLog)
+        DurableStringEnumerator(valuesLog, valueHashToId)
      }
    }
  }
@@ -67,7 +50,7 @@ internal class DurableStringEnumerator(

    val valueHash = hashOf(value)
    synchronized(valueHashLock) {
-      val valueHashToId = valueHashToId()
+      val valueHashToId = valueHashToId
      val foundId = lookupValue(valueHashToId, value, valueHash)
      if (foundId != DataEnumerator.NULL_ID) {
        return foundId
@@ -86,7 +69,7 @@ internal class DurableStringEnumerator(

    val valueHash = hashOf(value)
    synchronized(valueHashLock) {
-      val valueHashToId = valueHashToId()
+      val valueHashToId = valueHashToId
      return lookupValue(valueHashToId, value, valueHash)
    }
  }
@@ -99,15 +82,6 @@ internal class DurableStringEnumerator(
  }

  override fun close() {
-    try {
-      valueHashToIdFuture.join()
-    }
-    catch (_: CancellationException) {
-    }
-    catch (e: Throwable) {
-      logger<DurableStringEnumerator>().info(".valueHashToId computation failed", e)
-    }
-
    valuesLog.close()
  }

@@ -123,15 +97,6 @@ internal class DurableStringEnumerator(
    valuesLog.closeAndClean()
  }

-  private fun valueHashToId(): Int2IntMultimap {
-    var valueHashToId = valueHashToId
-    if (valueHashToId == null) {
-      valueHashToId = valueHashToIdFuture.get()
-      this.valueHashToId = valueHashToId
-    }
-    return valueHashToId
-  }
-
  private fun lookupValue(valueHashToId: Int2IntMultimap, value: String, hash: Int): Int {
    val foundIdRef = IntRef(DataEnumerator.NULL_ID)
    valueHashToId.lookup(hash, IntPredicate { candidateId ->
@@ -153,18 +118,13 @@ private fun hashOf(value: String): Int {
  val hash = value.hashCode()
  if (hash == Int2IntMultimap.NO_VALUE) {
    //Int2IntMultimap doesn't allow 0 keys/values, hence replace 0 hash with just any value!=0. Hash doesn't
-    // identify name uniquely anyway, hence this replacement just adds another hash collision -- basically,
-    // of collisions
+    // identify name uniquely anyway; hence this replacement just adds another hash collision -- basically, of collisions
    return -1 // any value!=0 will do
  }
  return hash
 }

-private val stringReader = ByteBufferReader { readString(it) }
-
-private fun readString(buffer: ByteBuffer): String {
-  return IOUtil.readString(buffer)
-}
+private val stringReader = ByteBufferReader { IOUtil.readString(it) }

 private fun writeString(value: String, valuesLog: AppendOnlyLog): Int {
  val valueBytes = value.encodeToByteArray()
@@ -175,7 +135,7 @@ private fun writeString(value: String, valuesLog: AppendOnlyLog): Int {
 private fun buildValueToIdIndex(valuesLog: AppendOnlyLog): Int2IntMultimap {
  val valueHashToId = Int2IntMultimap()
  valuesLog.forEachRecord { logId, buffer ->
-    val value = readString(buffer)
+    val value = IOUtil.readString(buffer)
    val id = convertLogIdToValueId(logId)
    val valueHash = hashOf(value)
    valueHashToId.put(valueHash, id)
@@ -184,4 +144,4 @@ private fun buildValueToIdIndex(valuesLog: AppendOnlyLog): Int2IntMultimap {
  return valueHashToId
 }

-private fun convertLogIdToValueId(logId: Long): Int = Math.toIntExact(logId)
+private fun convertLogIdToValueId(logId: Long): Int = Math.toIntExact(logId)
@@ -0,0 +1,85 @@
+// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
+@file:Suppress("UnstableApiUsage", "ReplaceGetOrSet")
+
+package org.jetbrains.bazel.jvm.jps.storage
+
+import com.intellij.openapi.util.io.FileUtilRt
+import io.opentelemetry.api.common.AttributeKey
+import io.opentelemetry.api.common.Attributes
+import io.opentelemetry.api.trace.Span
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import org.jetbrains.bazel.jvm.jps.impl.BazelBuildDataProvider
+import org.jetbrains.bazel.jvm.jps.impl.BazelBuildTargetStateManager
+import org.jetbrains.jps.builders.BuildTarget
+import org.jetbrains.jps.builders.BuildTargetType
+import org.jetbrains.jps.builders.storage.BuildDataPaths
+import org.jetbrains.jps.incremental.relativizer.PathRelativizerService
+import org.jetbrains.jps.incremental.storage.BuildDataManager
+import org.jetbrains.jps.incremental.storage.BuildTargetsState
+import java.nio.file.Files
+import java.nio.file.Path
+
+internal class StorageInitializer(private val dataDir: Path) {
+  private var wasCleared = false
+
+  suspend fun createBuildDataManager(
+    isRebuild: Boolean,
+    relativizer: PathRelativizerService,
+    buildDataProvider: BazelBuildDataProvider,
+    span: Span,
+  ): BuildDataManager {
+    if (isRebuild) {
+      wasCleared = true
+      withContext(Dispatchers.IO) {
+        FileUtilRt.deleteRecursively(dataDir)
+      }
+    }
+
+    while (true) {
+      try {
+        val containerManager = withContext(Dispatchers.IO) {
+          Files.createDirectories(dataDir)
+          BazelPersistentMapletFactory.open(dataDir.resolve("graph"))
+        }
+
+        return executeOrCloseStorage(containerManager) {
+          BuildDataManager.open(
+            dataPaths = BazelBuildDataPaths(dataDir),
+            targetState = BuildTargetsState(BazelBuildTargetStateManager),
+            relativizer = relativizer,
+            dataManager = buildDataProvider,
+            containerFactory = containerManager,
+          )
+        }
+      }
+      catch (e: Throwable) {
+        if (wasCleared) {
+          throw e
+        }
+
+        span.recordException(e, Attributes.of(AttributeKey.stringKey("message"), "cannot open cache storage"))
+      }
+
+      clearStorage()
+    }
+  }
+
+  fun clearStorage() {
+    wasCleared = true
+    // todo rename and store
+    FileUtilRt.deleteRecursively(dataDir)
+  }
+}
+
+private class BazelBuildDataPaths(private val dir: Path) : BuildDataPaths {
+  override fun getDataStorageDir() = dir
+
+  override fun getTargetsDataRoot(): Path = dir
+
+  override fun getTargetTypeDataRootDir(targetType: BuildTargetType<*>): Path = dir.resolve(targetType.typeId)
+
+  override fun getTargetDataRootDir(target: BuildTarget<*>): Path = dir.resolve(target.targetType.typeId)
+
+  override fun getTargetDataRoot(targetType: BuildTargetType<*>, targetId: String): Path = dir
+}
@@ -6,6 +6,7 @@ import com.github.benmanes.caffeine.cache.Caffeine
 import com.github.benmanes.caffeine.cache.LoadingCache
 import com.intellij.util.io.DataExternalizer
 import com.intellij.util.io.KeyDescriptor
+import com.intellij.util.io.Unmappable
 import org.jetbrains.bazel.jvm.hashSet
 import org.jetbrains.jps.dependency.BaseMaplet
 import org.jetbrains.jps.dependency.ExternalizableGraphElement
@@ -18,16 +19,13 @@ import org.jetbrains.jps.dependency.impl.CachingMaplet
 import org.jetbrains.jps.dependency.impl.CachingMultiMaplet
 import org.jetbrains.jps.dependency.impl.GraphDataInputImpl
 import org.jetbrains.jps.dependency.impl.GraphDataOutputImpl
-import org.jetbrains.jps.dependency.impl.PersistentMaplet
-import org.jetbrains.jps.dependency.impl.PersistentMultiMaplet
 import org.jetbrains.jps.incremental.storage.runAllCatching
 import java.io.Closeable
 import java.io.DataInput
 import java.io.DataOutput
-import java.nio.file.Files
+import java.io.IOException
+import java.lang.AutoCloseable
 import java.nio.file.Path
-import java.util.concurrent.CompletableFuture
-import java.util.function.Supplier
 import kotlin.math.max
 import kotlin.math.min

@@ -35,42 +33,33 @@ private const val BASE_CACHE_SIZE = 512

 private val cacheSize = BASE_CACHE_SIZE * min(max(1, (Runtime.getRuntime().maxMemory() / 1073741824L).toInt()), 5)

-internal interface AsyncExecutor {
-  fun <T> execute(action: () -> T): CompletableFuture<T>
-}
-
-internal class BazelPersistentMapletFactory(
+internal class BazelPersistentMapletFactory private constructor(
  private val rootDir: Path,
-  executor: AsyncExecutor,
+  private val stringEnumerator: DurableStringEnumerator,
 ) : MapletFactory, Closeable {
-  private val maps = ArrayList<BaseMaplet<*>>()
-
-  private val stringEnumeratorImpl = DurableStringEnumerator.openAsync(getMapFile("strings"), executor)
-
-  private val stringEnumerator = object : StringEnumerator {
-    override fun enumerate(string: String): Int {
-      return stringEnumeratorImpl.enumerate(string)
-    }
-
-    override fun valueOf(id: Int): String? {
-      return stringEnumeratorImpl.valueOf(id)
+  companion object {
+    internal fun open(rootDir: Path): BazelPersistentMapletFactory {
+      return BazelPersistentMapletFactory(rootDir, DurableStringEnumerator.open(rootDir.resolve("strings")))
    }
  }

+  private val maps = ArrayList<BaseMaplet<*>>()
+
  private val usageInterner: LoadingCache<Usage, Usage> = Caffeine.newBuilder().maximumSize(cacheSize.toLong()).build { it }
  private val usageInternerFunction: (Usage) -> Usage = { usageInterner.get(it) }

-  override fun <K, V> createSetMultiMaplet(
+  override fun <K : Any, V : Any> createSetMultiMaplet(
    storageName: String,
    keyExternalizer: Externalizer<K>,
    valueExternalizer: Externalizer<V>
  ): MultiMaplet<K, V> {
    val container = CachingMultiMaplet(
-      PersistentMultiMaplet(
-        getMapFile(storageName),
-        GraphKeyDescriptor(keyExternalizer, stringEnumerator),
-        GraphDataExternalizer(valueExternalizer, stringEnumerator, usageInternerFunction),
-        Supplier { hashSet() }
+      DurablePersistentMultiMaplet(
+        mapFile = rootDir.resolve(storageName),
+        keyDescriptor = GraphKeyDescriptor(keyExternalizer, stringEnumerator),
+        valueExternalizer = GraphDataExternalizer(valueExternalizer, stringEnumerator, usageInternerFunction),
+        collectionFactory = { hashSet() },
+        emptyCollection = emptySet(),
      ),
      cacheSize,
    )
@@ -78,16 +67,21 @@ internal class BazelPersistentMapletFactory(
    return container
  }

-  override fun <K, V> createMaplet(
+  override fun <K : Any, V : Any> createMaplet(
    storageName: String,
-    keyExternalizer: Externalizer<K?>,
-    valueExternalizer: Externalizer<V?>,
+    keyExternalizer: Externalizer<K>,
+    valueExternalizer: Externalizer<V>,
  ): Maplet<K, V> {
-    val container = CachingMaplet<K, V>(
-      PersistentMaplet(
-        getMapFile(storageName),
+    val container = CachingMaplet(
+      //PersistentMaplet(
+      //  getMapFile(storageName),
+      //  GraphKeyDescriptor(keyExternalizer, stringEnumerator),
+      //  GraphDataExternalizer(valueExternalizer, stringEnumerator, usageInternerFunction)
+      //),
+      DurableMapMaplet(
+        rootDir.resolve(storageName),
        GraphKeyDescriptor(keyExternalizer, stringEnumerator),
-        GraphDataExternalizer(valueExternalizer, stringEnumerator, usageInternerFunction)
+        GraphDataExternalizer(valueExternalizer, stringEnumerator, usageInternerFunction),
      ),
      cacheSize,
    )
@@ -100,26 +94,14 @@ internal class BazelPersistentMapletFactory(
      for (container in maps) {
        yield { container.close() }
      }
-      yield { stringEnumeratorImpl.close() }
+      yield { stringEnumerator.closeAndUnsafelyUnmap() }
    })
  }
-
-  private fun getMapFile(name: String): Path {
-    val file = rootDir.resolve(name)
-    Files.createDirectories(file.parent)
-    return file
-  }
 }

-private interface StringEnumerator {
-  fun enumerate(string: String): Int
-
-  fun valueOf(id: Int): String?
-}
-
-private open class GraphDataExternalizer<T>(
+private open class GraphDataExternalizer<T : Any>(
  private val externalizer: Externalizer<T>,
-  private val stringEnumerator: StringEnumerator,
+  private val stringEnumerator: DurableStringEnumerator,
  private val elementInterner: ((Usage) -> Usage)?,
 ) : DataExternalizer<T> {
  final override fun save(out: DataOutput, value: T?) {
@@ -136,7 +118,7 @@ private open class GraphDataExternalizer<T>(
      object : GraphDataInputImpl(`in`) {
        override fun readUTF(): String {
          val id = readInt()
-          return stringEnumerator.valueOf(id) ?: throw IllegalStateException("$id is not valid")
+          return stringEnumerator.valueOf(id) ?: invalidIdError(id)
        }
      }
    }
@@ -144,7 +126,7 @@ private open class GraphDataExternalizer<T>(
      object : GraphDataInputImpl(`in`) {
        override fun readUTF(): String {
          val id = readInt()
-          return stringEnumerator.valueOf(id) ?: throw IllegalStateException("$id is not valid")
+          return stringEnumerator.valueOf(id) ?: invalidIdError(id)
        }

        override fun <T : ExternalizableGraphElement?> processLoadedGraphElement(element: T?): T? {
@@ -157,9 +139,15 @@ private open class GraphDataExternalizer<T>(
  }
 }

-private class GraphKeyDescriptor<T>(
+private fun invalidIdError(id: Int): Nothing {
+  // throw IOException instead of IllegalStateException because in `PersistentEnumeratorBase.catchCorruption`
+  // we wrap non-IOException into RuntimeException
+  throw IOException("$id is not valid")
+}
+
+private class GraphKeyDescriptor<T : Any>(
  externalizer: Externalizer<T>,
-  stringEnumerator: StringEnumerator,
+  stringEnumerator: DurableStringEnumerator,
 ) : GraphDataExternalizer<T>(externalizer = externalizer, stringEnumerator = stringEnumerator, elementInterner = null), KeyDescriptor<T> {
  override fun isEqual(val1: T?, val2: T?): Boolean {
    return val1 == val2
@@ -168,4 +156,24 @@ private class GraphKeyDescriptor<T>(
  override fun getHashCode(value: T?): Int {
    return value.hashCode()
  }
+}
+
+internal inline fun <Out, In : AutoCloseable> executeOrCloseStorage(storageToClose: In, task: (In) -> Out): Out {
+  try {
+    return task(storageToClose)
+  }
+  catch (mainEx: Throwable) {
+    try {
+      if (storageToClose is Unmappable) {
+        storageToClose.closeAndUnsafelyUnmap()
+      }
+      else {
+        storageToClose.close()
+      }
+    }
+    catch (closeEx: Throwable) {
+      mainEx.addSuppressed(closeEx)
+    }
+    throw mainEx
+  }
 }