[vfs][refactoring] move DurableStringEnumerator to storages module

+ close to other storages

GitOrigin-RevId: c359332a05b4dc9c4196332c1fa35fa09577fa4d
This commit is contained in:
Ruslan Cheremin
2025-06-05 19:13:48 +02:00
committed by intellij-monorepo-bot
parent df322c2719
commit bea8eff0e4
4 changed files with 25 additions and 18 deletions

View File

@@ -9,12 +9,12 @@ import com.intellij.openapi.vfs.newvfs.persistent.dev.content.CompressingAlgo;
import com.intellij.openapi.vfs.newvfs.persistent.dev.content.ContentHashEnumeratorOverDurableEnumerator;
import com.intellij.openapi.vfs.newvfs.persistent.dev.content.ContentStorageAdapter;
import com.intellij.openapi.vfs.newvfs.persistent.dev.content.VFSContentStorageOverMMappedFile;
import com.intellij.openapi.vfs.newvfs.persistent.dev.enumerator.DurableStringEnumerator;
import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSRecoverer;
import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSRecoveryInfo;
import com.intellij.platform.util.io.storages.StorageFactory;
import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageHelper;
import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageOverMMappedFile;
import com.intellij.platform.util.io.storages.enumerator.DurableStringEnumerator;
import com.intellij.platform.util.io.storages.mmapped.MMappedFileStorageFactory;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.concurrency.SequentialTaskExecutor;
@@ -526,7 +526,7 @@ public final class PersistentFSLoader {
//MAYBE RC: remove .mmap suffix, and use namesFile directly? Suffix was needed during transition from regular to mmapped impls,
// and long unused
Path namesPathEx = Path.of(namesFile + ".mmap");
return DurableStringEnumerator.openAsync(namesPathEx, executorService);
return DurableStringEnumerator.openAsync(namesPathEx, executorService::async);
}
public @NotNull VFSContentStorage createContentStorage(@NotNull Path contentsHashesFile,

View File

@@ -1,15 +1,17 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.openapi.vfs.newvfs.persistent.enumerators
import com.intellij.openapi.vfs.newvfs.persistent.App
import com.intellij.openapi.vfs.newvfs.persistent.AppAgent
import com.intellij.openapi.vfs.newvfs.persistent.ExecuteOnThreadPool
import com.intellij.openapi.vfs.newvfs.persistent.dev.enumerator.DurableStringEnumerator
import com.intellij.platform.util.io.storages.CommonKeyDescriptors.stringAsUTF8
import com.intellij.platform.util.io.storages.enumerator.DurableEnumeratorFactory
import com.intellij.platform.util.io.storages.enumerator.DurableStringEnumerator
import com.intellij.util.ConcurrencyUtil
import com.intellij.util.io.DurableDataEnumerator
import java.nio.file.Path
import java.util.concurrent.Executors
import java.util.concurrent.Callable
import java.util.concurrent.CompletableFuture
import kotlin.io.path.absolute
@Suppress("unused")
@@ -26,7 +28,10 @@ class DurableEnumeratorOfStringsApp : App {
DurableStringEnumerator.open(storagePath)
}
"durable-string-enumerator-async" -> { storagePath ->
DurableStringEnumerator.openAsync(storagePath, ExecuteOnThreadPool(Executors.newSingleThreadExecutor()))
DurableStringEnumerator.openAsync(storagePath, object : DurableStringEnumerator.AsyncExecutor {
val wrapped = ExecuteOnThreadPool(ConcurrencyUtil.newSingleThreadExecutor("AsyncExecutor"))
override fun <T : Any?> async(task: Callable<T?>): CompletableFuture<T?> = wrapped.async(task)
})
}
"durable-enumerator-map-in-memory" -> { storagePath ->
DurableEnumeratorFactory

View File

@@ -1,7 +1,6 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.util.io.storages.enumerator;
import com.intellij.openapi.vfs.newvfs.persistent.dev.enumerator.DurableStringEnumerator;
import com.intellij.util.io.StringEnumeratorTestBase;
import org.jetbrains.annotations.NotNull;

View File

@@ -1,15 +1,13 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.openapi.vfs.newvfs.persistent.dev.enumerator;
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.platform.util.io.storages.enumerator;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.IntRef;
import com.intellij.openapi.vfs.newvfs.persistent.VFSAsyncTaskExecutor;
import com.intellij.platform.util.io.storages.appendonlylog.AppendOnlyLogFactory;
import com.intellij.platform.util.io.storages.enumerator.DurableEnumerator;
import com.intellij.util.io.*;
import com.intellij.platform.util.io.storages.StorageFactory;
import com.intellij.platform.util.io.storages.appendonlylog.AppendOnlyLog;
import com.intellij.platform.util.io.storages.appendonlylog.AppendOnlyLogFactory;
import com.intellij.platform.util.io.storages.intmultimaps.Int2IntMultimap;
import com.intellij.util.io.*;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -18,6 +16,7 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
@@ -29,8 +28,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
* Uses append-only log to store strings, and in-memory Map[string.hash->id*].
* Suitable for moderately big enumerators that are used very intensively, so
* increased heap consumption pays off. For general cases use {@link DurableEnumerator}
*
* TODO RC: move it to platform.util.storages as soon, as VFSAsyncTaskExecutor will be moved to some shared util package/module
*/
@ApiStatus.Internal
public final class DurableStringEnumerator implements DurableDataEnumerator<String>,
@@ -96,8 +93,14 @@ public final class DurableStringEnumerator implements DurableDataEnumerator<Stri
);
}
public static @NotNull DurableStringEnumerator openAsync(@NotNull Path storagePath,
@NotNull VFSAsyncTaskExecutor executor) throws IOException {
public interface AsyncExecutor {
<T> @NotNull CompletableFuture<T> async(@NotNull Callable<T> task);
}
public static @NotNull DurableStringEnumerator openAsync(
@NotNull Path storagePath,
@NotNull AsyncExecutor executor
) throws IOException {
return VALUES_LOG_FACTORY.wrapStorageSafely(
storagePath,
valuesLog -> {