MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[vfs][refactoring] make PersistentFSRecordsStorage factory more configurable

Browse Source 
GitOrigin-RevId: f3df6aef41ef4856f002cfe0fc3b63b09c5a6d69
This commit is contained in:
Ruslan Cheremin
2024-03-25 23:31:00 +00:00
committed by intellij-monorepo-bot
parent 311414d181
commit 33f5f89593
5 changed files with 56 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/FSRecordsImpl.java
+1 -1
View File
@@ -244,7 +244,7 @@ public final class FSRecordsImpl implements Closeable {
//bumped main version (60 -> 61) because of PFSRecords header enlargement (HEADER_ERRORS_ACCUMULATED)
final int mainVFSFormatVersion = 61;
//@formatter:off (nextMask better be aligned)
return nextMask(mainVFSFormatVersion + (PersistentFSRecordsStorageKind.storageImplementation().ordinal()), /* acceptable range is [0..255] */ 8,
return nextMask(mainVFSFormatVersion + (PersistentFSRecordsStorageFactory.storageImplementation().getId()), /* acceptable range is [0..255] */ 8,
nextMask(!USE_CONTENT_STORAGE_OVER_MMAPPED_FILE, //former USE_CONTENT_HASHES=true, this is why negation
nextMask(IOUtil.useNativeByteOrderForByteBuffers(),
nextMask(PageCacheUtils.LOCK_FREE_PAGE_CACHE_ENABLED && USE_ATTRIBUTES_OVER_NEW_FILE_PAGE_CACHE,//pageSize was changed on old<->new transition
platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java
+1 -1
View File
@@ -777,7 +777,7 @@ public final class PersistentFSLoader {
}
public @NotNull PersistentFSRecordsStorage createRecordsStorage(@NotNull Path recordsFile) throws IOException {
StorageFactory<PersistentFSRecordsStorage> recordsStorageFactory = PersistentFSRecordsStorageKind.storageImplementation();
StorageFactory<PersistentFSRecordsStorage> recordsStorageFactory = PersistentFSRecordsStorageFactory.storageImplementation();
LOG.trace("VFS uses " + recordsStorageFactory + " storage for main file records table");
return recordsStorageFactory.wrapStorageSafely(recordsFile, records -> {
platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSRecordsStorageKind.kt → platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSRecordsStorageFactory.kt
+37 -26
View File
@@ -8,29 +8,39 @@ import com.intellij.util.io.StorageLockContext
import com.intellij.util.io.dev.StorageFactory
import com.intellij.util.io.dev.mmapped.MMappedFileStorageFactory
import com.intellij.util.io.pagecache.impl.PageContentLockingStrategy
import org.jetbrains.annotations.ApiStatus.Internal
import org.jetbrains.annotations.VisibleForTesting
import java.io.IOException
import java.nio.file.Path
enum class PersistentFSRecordsStorageKind : StorageFactory<PersistentFSRecordsStorage> {
@Internal
abstract class PersistentFSRecordsStorageFactory(val id: Int) : StorageFactory<PersistentFSRecordsStorage> {
/** Currently the default impl */
OVER_MMAPPED_FILE {
data class OverMMappedFile(val pageSize: Int = PersistentFSRecordsLockFreeOverMMappedFile.DEFAULT_MAPPED_CHUNK_SIZE)
: PersistentFSRecordsStorageFactory(id = 0) {
override fun open(storagePath: Path): PersistentFSRecordsLockFreeOverMMappedFile = MMappedFileStorageFactory.withDefaults()
.pageSize(PersistentFSRecordsLockFreeOverMMappedFile.DEFAULT_MAPPED_CHUNK_SIZE)
.wrapStorageSafely<PersistentFSRecordsLockFreeOverMMappedFile, IOException>(storagePath) { storage ->
PersistentFSRecordsLockFreeOverMMappedFile(storage)
.pageSize(pageSize)
.wrapStorageSafely<PersistentFSRecordsLockFreeOverMMappedFile, IOException>(storagePath) { mappedFileStorage ->
PersistentFSRecordsLockFreeOverMMappedFile(mappedFileStorage)
}
},
}
/** Fallback impl for [OVER_MMAPPED_FILE] if something goes terribly wrong, and we can't fix it quickly */
OVER_LOCK_FREE_FILE_CACHE {
/** Fallback impl for [OverMMappedFile] if something goes terribly wrong, and we can't fix it quickly */
data class OverLockFreeFileCache(val pageSize: Int = PageCacheUtils.DEFAULT_PAGE_SIZE) : PersistentFSRecordsStorageFactory(id = 1) {
private val PERSISTENT_FS_STORAGE_CONTEXT_RW = StorageLockContext(true, true, true)
override fun open(storagePath: Path): PersistentFSRecordsOverLockFreePagedStorage {
init {
val recordLength = PersistentFSRecordsOverLockFreePagedStorage.RECORD_SIZE_IN_BYTES
val pageSize = PageCacheUtils.DEFAULT_PAGE_SIZE
val recordsArePageAligned = pageSize % recordLength == 0
if (!recordsArePageAligned) {
throw AssertionError("Bug: record length(=$recordLength) is not aligned with page size(=$pageSize)")
}
}
override fun open(storagePath: Path): PersistentFSRecordsOverLockFreePagedStorage {
if (!PageCacheUtils.LOCK_FREE_PAGE_CACHE_ENABLED) {
throw IOException(
@@ -38,11 +48,6 @@ enum class PersistentFSRecordsStorageKind : StorageFactory<PersistentFSRecordsSt
"=> can't create PersistentFSRecordsOverLockFreePagedStorage if FilePageCacheLockFree is disabled")
}
val recordsArePageAligned = pageSize % recordLength == 0
if (!recordsArePageAligned) {
throw AssertionError("Bug: record length(=$recordLength) is not aligned with page size(=$pageSize)")
}
val pagedStorage = PagedFileStorageWithRWLockedPageContent(
storagePath,
PERSISTENT_FS_STORAGE_CONTEXT_RW,
@@ -55,35 +60,41 @@ enum class PersistentFSRecordsStorageKind : StorageFactory<PersistentFSRecordsSt
PersistentFSRecordsOverLockFreePagedStorage(it)
}
}
},
}
/** For testing/benchmarking: serves as a reference point. Not a prod-level implementation! */
IN_MEMORY {
override fun open(storagePath: Path) =
@Suppress("TestOnlyProblems") (PersistentInMemoryFSRecordsStorage(storagePath, /*max size: */1 shl 24))
};
data class InMemory(val maxRecordsCount: Int = (1 shl 24)) : PersistentFSRecordsStorageFactory(id = 2) {
@Suppress("TestOnlyProblems")
override fun open(storagePath: Path) = PersistentInMemoryFSRecordsStorage(storagePath, maxRecordsCount)
}
companion object {
private var RECORDS_STORAGE_KIND = defaultFromSystemProperties()
private var storageFactory = defaultFromSystemProperties()
@JvmStatic
fun storageImplementation(): PersistentFSRecordsStorageKind = RECORDS_STORAGE_KIND
fun storageImplementation(): PersistentFSRecordsStorageFactory = storageFactory
@VisibleForTesting
@JvmStatic
@JvmName("setStorageImplementation")
fun setStorageImplementation(value: PersistentFSRecordsStorageKind) {
RECORDS_STORAGE_KIND = value
fun setStorageImplementation(value: PersistentFSRecordsStorageFactory) {
storageFactory = value
}
@VisibleForTesting
@JvmStatic
@JvmName("resetStorageImplementation")
fun resetStorageImplementation() {
RECORDS_STORAGE_KIND = defaultFromSystemProperties()
storageFactory = defaultFromSystemProperties()
}
private fun defaultFromSystemProperties() = PersistentFSRecordsStorageKind.valueOf(System.getProperty("vfs.records-storage.impl", OVER_MMAPPED_FILE.name))
private fun defaultFromSystemProperties(): PersistentFSRecordsStorageFactory {
return when (System.getProperty("vfs.records-storage.impl", "OVER_MMAPPED_FILE")) {
"OVER_LOCK_FREE_FILE_CACHE" -> OverLockFreeFileCache()
"IN_MEMORY" -> InMemory()
else -> OverMMappedFile()
}
}
}
}
platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSRecordsStorageOverLockFreePagedStorageTest.java
+1 -1
View File
@@ -48,7 +48,7 @@ public class PersistentFSRecordsStorageOverLockFreePagedStorageTest
@NotNull
@Override
protected PersistentFSRecordsOverLockFreePagedStorage openStorage(final Path storagePath) throws IOException {
return (PersistentFSRecordsOverLockFreePagedStorage)PersistentFSRecordsStorageKind.OVER_LOCK_FREE_FILE_CACHE.open(storagePath);
return new PersistentFSRecordsStorageFactory.OverLockFreeFileCache().open(storagePath);
}
@Test
platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/VFSInitializationTest.java
+16 -15
View File
@@ -3,6 +3,8 @@ package com.intellij.openapi.vfs.newvfs.persistent;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.vfs.newvfs.FileAttribute;
import com.intellij.openapi.vfs.newvfs.persistent.PersistentFSRecordsStorageFactory.OverLockFreeFileCache;
import com.intellij.openapi.vfs.newvfs.persistent.PersistentFSRecordsStorageFactory.OverMMappedFile;
import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSInitializationResult;
import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSRecoverer;
import com.intellij.testFramework.TemporaryDirectory;
@@ -17,7 +19,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static com.intellij.openapi.vfs.newvfs.persistent.PersistentFSRecordsStorageKind.*;
import static com.intellij.openapi.vfs.newvfs.persistent.VFSInitException.ErrorCategory.*;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.*;
@@ -195,17 +196,17 @@ public class VFSInitializationTest {
//skip IN_MEMORY impl, since it is not really persistent
//skip OVER_LOCK_FREE_FILE_CACHE impl if !LOCK_FREE_PAGE_CACHE_ENABLED (fails otherwise)
List<PersistentFSRecordsStorageKind> allStorageKinds = PageCacheUtils.LOCK_FREE_PAGE_CACHE_ENABLED ?
List.of(OVER_LOCK_FREE_FILE_CACHE, OVER_MMAPPED_FILE) :
List.of(OVER_MMAPPED_FILE);
List<PersistentFSRecordsStorageFactory> allStorageKinds = PageCacheUtils.LOCK_FREE_PAGE_CACHE_ENABLED ?
List.of(new OverLockFreeFileCache(), new OverMMappedFile()) :
List.of(new OverMMappedFile());
List<String> filesNotLeadingToVFSRebuild = new ArrayList<>();
for (PersistentFSRecordsStorageKind storageKind : allStorageKinds) {
for (PersistentFSRecordsStorageFactory storageKind : allStorageKinds) {
int vfsFilesCount = 1;
int vfsVersion;
for (int i = 0; i < vfsFilesCount; i++) {
Path cachesDir = temporaryDirectory.createDir();
PersistentFSRecordsStorageKind.setStorageImplementation(storageKind);
PersistentFSRecordsStorageFactory.setStorageImplementation(storageKind);
FSRecordsImpl fsRecords = FSRecordsImpl.connect(cachesDir);
try {
@@ -238,7 +239,7 @@ public class VFSInitializationTest {
FileUtil.delete(fileToDelete);
//reopen:
PersistentFSRecordsStorageKind.setStorageImplementation(storageKind);
PersistentFSRecordsStorageFactory.setStorageImplementation(storageKind);
try {
PersistentFSConnection connection = tryInit(cachesDir, vfsVersion, Collections.emptyList());
try {
@@ -269,15 +270,15 @@ public class VFSInitializationTest {
public void VFS_isRebuilt_OnlyIf_ImplementationVersionChanged() throws Exception {
//skip IN_MEMORY impl, since it is not really persistent
//skip OVER_LOCK_FREE_FILE_CACHE impl if !LOCK_FREE_PAGE_CACHE_ENABLED (will fail)
final List<PersistentFSRecordsStorageKind> allKinds = PageCacheUtils.LOCK_FREE_PAGE_CACHE_ENABLED ?
List.of(OVER_LOCK_FREE_FILE_CACHE, OVER_MMAPPED_FILE) :
List.of(OVER_MMAPPED_FILE);
final List<PersistentFSRecordsStorageFactory> allKinds = PageCacheUtils.LOCK_FREE_PAGE_CACHE_ENABLED ?
List.of(new OverLockFreeFileCache(), new OverMMappedFile()) :
List.of(new OverMMappedFile());
//check all combinations (from->to) of implementations:
for (PersistentFSRecordsStorageKind kindBefore : allKinds) {
for (PersistentFSRecordsStorageKind kindAfter : allKinds) {
for (PersistentFSRecordsStorageFactory kindBefore : allKinds) {
for (PersistentFSRecordsStorageFactory kindAfter : allKinds) {
Path cachesDir = temporaryDirectory.createDir();
PersistentFSRecordsStorageKind.setStorageImplementation(kindBefore);
PersistentFSRecordsStorageFactory.setStorageImplementation(kindBefore);
long firstVfsCreationTimestamp;
FSRecordsImpl vfs = FSRecordsImpl.connect(cachesDir);
try {
@@ -289,7 +290,7 @@ public class VFSInitializationTest {
Thread.sleep(500);//ensure system clock is moving
//reopen:
PersistentFSRecordsStorageKind.setStorageImplementation(kindAfter);
PersistentFSRecordsStorageFactory.setStorageImplementation(kindAfter);
FSRecordsImpl reopenedVfs = FSRecordsImpl.connect(cachesDir);
try {
long reopenedVfsCreationTimestamp = reopenedVfs.getCreationTimestamp();
@@ -413,7 +414,7 @@ public class VFSInitializationTest {
@After
public void tearDown() throws Exception {
PersistentFSRecordsStorageKind.resetStorageImplementation();
PersistentFSRecordsStorageFactory.resetStorageImplementation();
for (PersistentFSConnection connection : connectionsOpened) {
PersistentFSConnector.disconnect(connection);