MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-03-22 15:19:59 +07:00
Code Issues Packages Projects Releases Wiki Activity

[vfs] improve roots persistence

Browse Source 
+ improve VFS.roots storing/restoring: more checks

GitOrigin-RevId: 39e4dce98517e8d6e6fe4a7392e49e81fdadda51
This commit is contained in:
Ruslan Cheremin
2025-06-13 18:47:59 +02:00
committed by intellij-monorepo-bot
parent 582aa6978b
commit 300ab7aa2f
4 changed files with 225 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSTreeAccessor.java
View File

@@ -220,11 +220,19 @@ public class PersistentFSTreeAccessor {
int[] rootsUrlIds = ArrayUtil.newIntArray(rootsCount);
int[] rootsIds = ArrayUtil.newIntArray(rootsCount);
int prevRootId = 0;
int prevUrlId = 0;
int prevRootId = 0;
for (int i = 0; i < rootsCount; i++) {
int urlId = DataInputOutputUtil.readINT(input) + prevUrlId;
int rootId = DataInputOutputUtil.readINT(input) + prevRootId;
int diffUrlId = DataInputOutputUtil.readINT(input);
if (diffUrlId <= 0) {
throw new IOException("SUPER_ROOT.CHILDREN attribute is corrupted: diffUrlId[" + i + "](=" + diffUrlId + ") must be >0");
}
int diffRootId = DataInputOutputUtil.readINT(input);
if (diffRootId <= 0) {
throw new IOException("SUPER_ROOT.CHILDREN attribute is corrupted: diffRootId[" + i + "](=" + diffRootId + ") must be >0");
}
int urlId = diffUrlId + prevUrlId;
int rootId = diffRootId + prevRootId;
checkChildIdValid(SUPER_ROOT_ID, rootId, i, maxAllocatedID);
@@ -263,7 +271,7 @@ public class PersistentFSTreeAccessor {
rootsUrlIds = ArrayUtil.insert(rootsUrlIds, -index - 1, newRootUrlId);
try (DataOutputStream output = attributeAccessor.writeAttribute(SUPER_ROOT_ID, CHILDREN_ATTR)) {
saveNameIdSequenceWithDeltas(rootsUrlIds, rootsIds, output);
saveUrlAndFileIdsAsDiffCompressed(rootsUrlIds, rootsIds, output);
//RC: we should assign connection.records.setNameId(newRootFileId, root_Name_Id), but we don't
// have rootNameId here -- we have only rootUrlId. So, rootNameId is assigned to the root
// in a PersistentFSImpl.findRoot() method, up the stack
@@ -300,7 +308,7 @@ public class PersistentFSTreeAccessor {
rootsIds = ArrayUtil.remove(rootsIds, index);
try (DataOutputStream output = attributeAccessor.writeAttribute(SUPER_ROOT_ID, CHILDREN_ATTR)) {
saveNameIdSequenceWithDeltas(rootsUrlIds, rootsIds, output);
saveUrlAndFileIdsAsDiffCompressed(rootsUrlIds, rootsIds, output);
}
}
@@ -349,15 +357,43 @@ public class PersistentFSTreeAccessor {
connection.enumerateAttributeId(CHILDREN_ATTR.getId()); // trigger writing / loading of vfs attribute ids in top level write action
}
static void saveNameIdSequenceWithDeltas(int[] names, int[] ids, DataOutputStream output) throws IOException {
DataInputOutputUtil.writeINT(output, names.length);
int prevId = 0;
int prevNameId = 0;
for (int i = 0; i < names.length; i++) {
DataInputOutputUtil.writeINT(output, names[i] - prevNameId);
DataInputOutputUtil.writeINT(output, ids[i] - prevId);
prevId = ids[i];
prevNameId = names[i];
/**
* Serializes urlIds and fileIds sorted arrays into output stream, in diff-compressed format:
* <pre>
* {urlIds.length: varint} ({urlId[i]-urlId[i-1]: varint}, {fileId[i]-fileId[i-1]: varint})*
* </pre>
* Both urlIds and fileIds must be sorted, same length, and without duplicates -- otherwise {@link IllegalStateException} is thrown
*/
private static void saveUrlAndFileIdsAsDiffCompressed(int[] urlIds,
int[] fileIds,
@NotNull DataOutputStream output) throws IOException {
if (urlIds.length != fileIds.length) {
throw new IllegalArgumentException("urlIds.length(=" + urlIds.length + ") != fileIds.length(=" + fileIds.length + ")");
}
DataInputOutputUtil.writeINT(output, urlIds.length);
int prevUrlId = 0;
int prevFileId = 0;
for (int i = 0; i < urlIds.length; i++) {
int urlId = urlIds[i];
int fileId = fileIds[i];
int diffUrlId = urlId - prevUrlId;
int diffFileId = fileId - prevFileId;
if (diffUrlId <= 0) {
throw new IllegalStateException(
"urlIds are not sorted: urlIds[" + i + "](=" + urlId + ") <= urlIds[" + (i - 1) + "](=" + prevUrlId + "), " +
"urlIds: " + Arrays.toString(urlIds)
);
}
if (diffFileId <= 0) {
throw new IllegalStateException(
"fileIds are not sorted: fileIds[" + i + "](=" + fileId + ") <= fileIds[" + (i - 1) + "](=" + prevFileId + "), " +
"fileIds: " + Arrays.toString(fileIds)
);
}
DataInputOutputUtil.writeINT(output, diffUrlId);
DataInputOutputUtil.writeINT(output, diffFileId);
prevFileId = fileId;
prevUrlId = urlId;
}
}

14
platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSTreeRawAccessor.java
View File

@@ -151,11 +151,19 @@ public final class PersistentFSTreeRawAccessor extends PersistentFSTreeAccessor
int[] rootsUrlIds = ArrayUtil.newIntArray(rootsCount);
int[] rootsIds = ArrayUtil.newIntArray(rootsCount);
int prevRootId = 0;
int prevUrlId = 0;
int prevRootId = 0;
for (int i = 0; i < rootsCount; i++) {
int urlId = DataInputOutputUtil.readINT(recordBuffer) + prevUrlId;
int rootId = DataInputOutputUtil.readINT(recordBuffer) + prevRootId;
int diffUrlId = DataInputOutputUtil.readINT(recordBuffer);
if (diffUrlId <= 0) {
throw new IOException("SUPER_ROOT.CHILDREN attribute is corrupted: diffUrlId[" + i + "](=" + diffUrlId + ") must be >0");
}
int diffRootId = DataInputOutputUtil.readINT(recordBuffer);
if (diffRootId <= 0) {
throw new IOException("SUPER_ROOT.CHILDREN attribute is corrupted: diffRootId[" + i + "](=" + diffRootId + ") must be >0");
}
int urlId = diffUrlId + prevUrlId;
int rootId = diffRootId + prevRootId;
checkChildIdValid(SUPER_ROOT_ID, rootId, i, maxAllocatedID);

123
platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/FSRecordsImplTest.java
View File

@@ -2,7 +2,6 @@
package com.intellij.openapi.vfs.newvfs.persistent;
import com.intellij.openapi.util.io.ByteArraySequence;
import com.intellij.openapi.util.io.FileAttributes;
import com.intellij.openapi.vfs.newvfs.FileAttribute;
import com.intellij.platform.util.io.storages.StorageTestingUtils;
import com.intellij.util.io.DataOutputStream;
@@ -15,102 +14,53 @@ import org.junit.jupiter.api.io.TempDir;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.*;
import java.util.stream.IntStream;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Stream;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class FSRecordsImplTest {
private FSRecordsImpl vfs;
private Path vfsDir;
@Test
public void allocatedRecordId_CouldBeAlwaysWritten_EvenInMultiThreadedEnv() throws Exception {
//RC: there are EA reports with 'fileId ... outside of allocated range ...' exception
// _just after recordId was allocated_. So the test checks there are no concurrency errors
// that could leads to that:
int CPUs = Runtime.getRuntime().availableProcessors();
int recordsPerThread = 1_000_000;
ExecutorService pool = Executors.newFixedThreadPool(CPUs);
try {
FileAttributes attributes = new FileAttributes(true, false, false, true, 1, 1, true);
Callable<Object> insertingRecordsTask = () -> {
for (int i = 0; i < recordsPerThread; i++) {
int fileId = vfs.createRecord();
vfs.updateRecordFields(fileId, 1, attributes, "file_" + i, true);
}
return null;
};
List<Future<Object>> futures = IntStream.range(0, CPUs)
.mapToObj(i -> insertingRecordsTask)
.map(pool::submit)
.toList();
for (Future<Object> future : futures) {
future.get();//give a chance to deliver exception
}
}
finally {
pool.shutdown();
pool.awaitTermination(15, SECONDS);
public void insertedRoots_CouldBeReadBack() throws Exception {
int totalRoots = 100_000; //too many roots could exceed attribute storage max record size
int[] rootIds = new int[totalRoots];
for (int i = 0; i < totalRoots; i++) {
String rootUrl = "file:///root/" + i;
rootIds[i] = vfs.findOrCreateRootRecord(rootUrl);
}
int[] rootIdsReadBack = vfs.listRoots();
assertArrayEquals(
rootIds,
rootIdsReadBack,
"rootIds stored must be equal to rootIds read back"
);
}
@Test
public void concurrentlyInsertedRoots_AreStillConsistent() throws Exception {
int CPUs = Runtime.getRuntime().availableProcessors();
int totalRoots = 100_000;//too many roots could exceed attribute storage max record size
int rootsPerThread = totalRoots / CPUs;
ExecutorService pool = Executors.newFixedThreadPool(CPUs);
try {
List<Future<List<String>>> futures = IntStream.range(0, CPUs)
.mapToObj(threadNo -> (Callable<List<String>>)() -> {
List<String> rootsUrls = new ArrayList<>(rootsPerThread);
for (int i = 0; i < rootsPerThread; i++) {
String rootUrl = "file:///root/" + threadNo + "/" + i;
rootsUrls.add(rootUrl);
}
for (final String rootUrl : rootsUrls) {
vfs.findOrCreateRootRecord(rootUrl);
}
return rootsUrls;
})
.map(pool::submit)
.toList();
List<String> allRootUrls = new ArrayList<>(totalRoots);
for (Future<List<String>> future : futures) {
allRootUrls.addAll(future.get());//give a chance to deliver exception
}
assertEquals(totalRoots,
vfs.listRoots().length,
"Must be " + totalRoots + " roots");
List<String> allRootUrlsFromVFS = new ArrayList<>(totalRoots);
vfs.forEachRoot((rootUrl, rootId) -> {
allRootUrlsFromVFS.add(rootUrl);
});
Collections.sort(allRootUrls);
Collections.sort(allRootUrlsFromVFS);
assertEquals(
allRootUrlsFromVFS,
allRootUrls,
"Only roots inserted must be returned by .forEachRoot()"
);
}
finally {
pool.shutdown();
pool.awaitTermination(15, SECONDS);
public void insertedRoots_CouldBeReadBack_AfterReinitialization() throws Exception {
int totalRoots = 100_000; //too many roots could exceed attribute storage max record size
int[] rootIds = new int[totalRoots];
for (int i = 0; i < totalRoots; i++) {
String rootUrl = "file:///root/" + i;
rootIds[i] = vfs.findOrCreateRootRecord(rootUrl);
}
vfs = reloadVFS();
int[] rootIdsReadBack = vfs.listRoots();
assertArrayEquals(
rootIds,
rootIdsReadBack,
"rootIds stored must be equal to rootIds read back even after VFS was re-initialized"
);
}
@@ -203,7 +153,6 @@ public class FSRecordsImplTest {
}
}
@Test
public void fileRecordModCountChanges_ifFileAttributeWritten_regardlessOfActualValueChange() throws IOException {
FileAttribute fileAttribute = new FileAttribute("X");
@@ -240,8 +189,13 @@ public class FSRecordsImplTest {
}
}
/* ========================= infrastructure =========================================================================== */
@BeforeEach
void setUp(@TempDir Path vfsDir) {
this.vfsDir = vfsDir;
vfs = FSRecordsImpl.connect(vfsDir, FSRecordsImpl.ON_ERROR_RETHROW);
}
@@ -252,6 +206,11 @@ public class FSRecordsImplTest {
}
}
private FSRecordsImpl reloadVFS() throws Exception {
StorageTestingUtils.bestEffortToCloseAndUnmap(vfs);
return FSRecordsImpl.connect(vfsDir, FSRecordsImpl.ON_ERROR_RETHROW);
}
private void writeContent(int fileId,
@NotNull String content) {
vfs.writeContent(fileId, new ByteArraySequence(content.getBytes(UTF_8)), true);

123
platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/FSRecordsImpl_MultiThreaded_Test.java Normal file
View File

@@ -0,0 +1,123 @@
// Copyright 2000-2025 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.openapi.vfs.newvfs.persistent;
import com.intellij.openapi.util.io.FileAttributes;
import com.intellij.platform.util.io.storages.StorageTestingUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.IntStream;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class FSRecordsImpl_MultiThreaded_Test {
private FSRecordsImpl vfs;
@Test
public void allocatedRecordId_CouldBeAlwaysWritten_EvenInMultiThreadedEnv() throws Exception {
//RC: there are EA reports with 'fileId ... outside of allocated range ...' exception
// _just after recordId was allocated_. So the test checks there are no concurrency errors
// that could leads to that:
int CPUs = Runtime.getRuntime().availableProcessors();
int recordsPerThread = 1_000_000;
ExecutorService pool = Executors.newFixedThreadPool(CPUs);
try {
FileAttributes attributes = new FileAttributes(true, false, false, true, 1, 1, true);
Callable<Object> insertingRecordsTask = () -> {
for (int i = 0; i < recordsPerThread; i++) {
int fileId = vfs.createRecord();
vfs.updateRecordFields(fileId, 1, attributes, "file_" + i, true);
}
return null;
};
List<Future<Object>> futures = IntStream.range(0, CPUs)
.mapToObj(i -> insertingRecordsTask)
.map(pool::submit)
.toList();
for (Future<Object> future : futures) {
future.get();//give a chance to deliver exception
}
}
finally {
pool.shutdown();
pool.awaitTermination(15, SECONDS);
}
}
@Test
public void concurrentlyInsertedRoots_AreStillConsistent() throws Exception {
int CPUs = Runtime.getRuntime().availableProcessors();
int totalRoots = 100_000;//too many roots could exceed attribute storage max record size
int rootsPerThread = totalRoots / CPUs;
ExecutorService pool = Executors.newFixedThreadPool(CPUs);
try {
List<Future<List<String>>> futures = IntStream.range(0, CPUs)
.mapToObj(threadNo -> (Callable<List<String>>)() -> {
List<String> rootsUrls = new ArrayList<>(rootsPerThread);
for (int i = 0; i < rootsPerThread; i++) {
String rootUrl = "file:///root/" + threadNo + "/" + i;
rootsUrls.add(rootUrl);
}
for (String rootUrl : rootsUrls) {
vfs.findOrCreateRootRecord(rootUrl);
}
return rootsUrls;
})
.map(pool::submit)
.toList();
List<String> allRootUrls = new ArrayList<>(totalRoots);
for (Future<List<String>> future : futures) {
allRootUrls.addAll(future.get());//give a chance to deliver exception
}
assertEquals(totalRoots,
vfs.listRoots().length,
"Must be " + totalRoots + " roots");
List<String> allRootUrlsFromVFS = new ArrayList<>(totalRoots);
vfs.forEachRoot((rootUrl, rootId) -> {
allRootUrlsFromVFS.add(rootUrl);
});
Collections.sort(allRootUrls);
Collections.sort(allRootUrlsFromVFS);
assertEquals(
allRootUrlsFromVFS,
allRootUrls,
"Only roots inserted must be returned by .forEachRoot()"
);
}
finally {
pool.shutdown();
pool.awaitTermination(15, SECONDS);
}
}
@BeforeEach
void setUp(@TempDir Path vfsDir) {
vfs = FSRecordsImpl.connect(vfsDir, FSRecordsImpl.ON_ERROR_RETHROW);
}
@AfterEach
void tearDown() throws Exception {
if (vfs != null) {
StorageTestingUtils.bestEffortToCloseAndClean(vfs);
}
}
}