diff --git a/platform/core-api/api-dump-unreviewed.txt b/platform/core-api/api-dump-unreviewed.txt index 6a57074f89aa..774b658dd00f 100644 --- a/platform/core-api/api-dump-unreviewed.txt +++ b/platform/core-api/api-dump-unreviewed.txt @@ -2662,7 +2662,6 @@ c:com.intellij.openapi.vfs.InvalidVirtualFileAccessException com.intellij.openapi.vfs.LocalFileProvider com.intellij.openapi.vfs.NonPhysicalFileSystem f:com.intellij.openapi.vfs.PersistentFSConstants -- sf:FILE_LENGTH_TO_CACHE_THRESHOLD:J - s:getMaxIntellisenseFileSize():I - s:setMaxIntellisenseFileSize(I):V a:com.intellij.openapi.vfs.ReadonlyStatusHandler diff --git a/platform/core-api/src/com/intellij/openapi/vfs/PersistentFSConstants.java b/platform/core-api/src/com/intellij/openapi/vfs/PersistentFSConstants.java index 6b668b18cb16..41db2a3627ca 100644 --- a/platform/core-api/src/com/intellij/openapi/vfs/PersistentFSConstants.java +++ b/platform/core-api/src/com/intellij/openapi/vfs/PersistentFSConstants.java @@ -3,24 +3,35 @@ package com.intellij.openapi.vfs; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.util.io.FileUtilRt; -import com.intellij.openapi.vfs.limits.FileSizeLimit; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.TestOnly; +import static com.intellij.util.SystemProperties.getIntProperty; + public final class PersistentFSConstants { - /** Max file size to cache inside VFS */ - public static final long FILE_LENGTH_TO_CACHE_THRESHOLD = FileSizeLimit.getFileLengthToCacheThreshold(); + /** + * Max file size to cache inside VFS. + * Cache huge files is rarely useful, but time-consuming (freeze-prone), and could quickly overflow VFS content storage capacity + */ + @ApiStatus.Internal + public static final int MAX_FILE_LENGTH_TO_CACHE = getIntProperty( + "idea.vfs.max-file-length-to-cache", + FileUtilRt.MEGABYTE + ); /** - * Must always be in range [0, {@link #FILE_LENGTH_TO_CACHE_THRESHOLD}] + * Must always be in range [0, {@link FileUtilRt#LARGE_FOR_CONTENT_LOADING}] *

* Currently, this is always true, because - * 

FILE_LENGTH_TO_CACHE_THRESHOLD = ... = max(20Mb, userFileSizeLimit, userContentLoadLimit)
+ * 
LARGE_FOR_CONTENT_LOADING = ... = max(20Mb, userFileSizeLimit, userContentLoadLimit)
* but could be changed in the future, hence .min(...) here is to ensure that. + * TODO: move into FileSizeLimit */ - private static int ourMaxIntellisenseFileSize = Math.min(FileUtilRt.getUserFileSizeLimit(), (int)FILE_LENGTH_TO_CACHE_THRESHOLD); + private static int ourMaxIntellisenseFileSize = Math.min(FileUtilRt.getUserFileSizeLimit(), FileUtilRt.LARGE_FOR_CONTENT_LOADING); /** @deprecated Prefer using {@link com.intellij.openapi.vfs.limits.FileSizeLimit#getIntellisenseLimit(String)} */ + //TODO: move into FileSizeLimit @SuppressWarnings("DeprecatedIsStillUsed") @Deprecated() public static int getMaxIntellisenseFileSize() { diff --git a/platform/core-api/src/com/intellij/openapi/vfs/limits/FileSizeLimit.kt b/platform/core-api/src/com/intellij/openapi/vfs/limits/FileSizeLimit.kt index 981c9cec76c1..e5685fb58a68 100644 --- a/platform/core-api/src/com/intellij/openapi/vfs/limits/FileSizeLimit.kt +++ b/platform/core-api/src/com/intellij/openapi/vfs/limits/FileSizeLimit.kt @@ -21,11 +21,6 @@ interface FileSizeLimit { private val limitsByExtension: AtomicReference?> = AtomicReference(null) - /** Max file size to cache inside VFS */ - @JvmStatic - @ApiStatus.Internal - fun getFileLengthToCacheThreshold(): Int = FileUtilRt.LARGE_FOR_CONTENT_LOADING - private fun getLimitsByExtension(): Map { return limitsByExtension.get() ?: limitsByExtension.updateAndGet { getLimits() }!! } @@ -56,12 +51,12 @@ interface FileSizeLimit { @JvmStatic fun getContentLoadLimit(extension: String?): Int { @Suppress("DEPRECATION") - val limit = findApplicable(extension ?: "")?.content ?: FileUtilRt.LARGE_FOR_CONTENT_LOADING + val limit = findApplicable(extension ?: "")?.content ?: getDefaultContentLoadLimit() return limit } @JvmStatic - fun getDefaultContentLoadLimit(): Int = getContentLoadLimit(null) + fun getDefaultContentLoadLimit(): Int = FileUtilRt.LARGE_FOR_CONTENT_LOADING @JvmStatic fun getIntellisenseLimit(): Int = getIntellisenseLimit(null) diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java index 06af21288aa0..b0a902df24c3 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java @@ -155,7 +155,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable { setupOTelMonitoring(TelemetryManager.getInstance().getMeter(PlatformScopesKt.VFS)); - LOG.info("VFS.maxFileLengthToCache: " + PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD); + LOG.info("VFS.MAX_FILE_LENGTH_TO_CACHE: " + PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE); } @ApiStatus.Internal @@ -909,7 +909,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable { } private static boolean shouldCacheFileContentInVFS(long fileLength) { - return fileLength <= PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD; + return fileLength <= PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE; } private @NotNull InputStream createReplicatorAndStoreContent(@NotNull VirtualFile file, @@ -1541,8 +1541,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable { if (!(vf instanceof VirtualDirectoryImpl)) { return; } - parent = - (VirtualDirectoryImpl)vf; // retain in `myIdToDirCache` at least for the duration of this block, so that subsequent `findFileById` won't crash + parent = (VirtualDirectoryImpl)vf; // retain in `myIdToDirCache` at least for the duration of this block, so that subsequent `findFileById` won't crash NewVirtualFileSystem fs = getFileSystem(parent); List childrenAdded = new ArrayList<>(createEvents.size()); @@ -1559,8 +1558,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable { childrenAdded.sort(ChildInfo.BY_ID); boolean caseSensitive = parent.isCaseSensitive(); vfsPeer.update(parent, parentId, oldChildren -> oldChildren.merge(vfsPeer, childrenAdded, caseSensitive)); - parent.createAndAddChildren(childrenAdded, false, (__, ___) -> { - }); + parent.createAndAddChildren(childrenAdded, false, (__, ___) -> {}); saveScannedChildrenRecursively(createEvents, fs, parent.isCaseSensitive()); } diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java index 4dfba5a4550f..67c471400ba8 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java @@ -4,9 +4,7 @@ package com.intellij.openapi.vfs.newvfs.persistent; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.NotNullLazyValue; import com.intellij.openapi.util.io.FileUtil; -import com.intellij.openapi.util.io.FileUtilRt; -import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageHelper; -import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageOverMMappedFile; +import com.intellij.openapi.vfs.PersistentFSConstants; import com.intellij.openapi.vfs.newvfs.persistent.dev.content.CompressingAlgo; import com.intellij.openapi.vfs.newvfs.persistent.dev.content.ContentHashEnumeratorOverDurableEnumerator; import com.intellij.openapi.vfs.newvfs.persistent.dev.content.ContentStorageAdapter; @@ -15,6 +13,8 @@ import com.intellij.openapi.vfs.newvfs.persistent.dev.enumerator.DurableStringEn import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSRecoverer; import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSRecoveryInfo; import com.intellij.platform.util.io.storages.StorageFactory; +import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageHelper; +import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageOverMMappedFile; import com.intellij.platform.util.io.storages.mmapped.MMappedFileStorageFactory; import com.intellij.util.ExceptionUtil; import com.intellij.util.concurrency.SequentialTaskExecutor; @@ -23,7 +23,9 @@ import com.intellij.util.io.*; import com.intellij.util.io.blobstorage.SpaceAllocationStrategy; import com.intellij.util.io.blobstorage.SpaceAllocationStrategy.DataLengthPlusFixedPercentStrategy; import com.intellij.util.io.blobstorage.StreamlinedBlobStorage; -import com.intellij.util.io.storage.*; +import com.intellij.util.io.storage.RefCountingContentStorage; +import com.intellij.util.io.storage.RefCountingContentStorageImpl; +import com.intellij.util.io.storage.VFSContentStorage; import com.intellij.util.io.storage.lf.RefCountingContentStorageImplLF; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntList; @@ -529,10 +531,10 @@ public final class PersistentFSLoader { //Use larger pages: content storage is usually quite big. int pageSize = 64 * IOUtil.MiB; - if (pageSize <= FileUtilRt.LARGE_FOR_CONTENT_LOADING) { + if (pageSize <= PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE) { //pageSize is an upper limit on record size for AppendOnlyLogOverMMappedFile: LOG.warn("ContentStorage.pageSize(=" + pageSize + ") " + - "must be > FileUtilRt.LARGE_FOR_CONTENT_LOADING(=" + FileUtilRt.LARGE_FOR_CONTENT_LOADING + "b), " + + "must be > PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE(=" + PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE + "b), " + "otherwise large content can't fit"); } CompressingAlgo compressionAlgo = switch (FSRecordsImpl.COMPRESSION_ALGO) { diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/impl/FileTypesTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/impl/FileTypesTest.java index 4c4cf2c355af..328a225d6f23 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/impl/FileTypesTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/impl/FileTypesTest.java @@ -899,7 +899,7 @@ public class FileTypesTest extends HeavyPlatformTestCase { if (random.nextInt(3) == 0) { WriteCommandAction.writeCommandAction(getProject()).run(() -> { - byte[] bytes = new byte[(int)PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD + (isText ? 1 : 0)]; + byte[] bytes = new byte[PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE + (isText ? 1 : 0)]; Arrays.fill(bytes, (byte)' '); virtualFile.setBinaryContent(bytes); }); @@ -928,7 +928,7 @@ public class FileTypesTest extends HeavyPlatformTestCase { FrequentEventDetector.disableUntil(getTestRootDisposable()); File f = createTempFile("xx.asd_kjf_hlk_asj_dhf", - StringUtil.repeatSymbol(' ', (int)PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD - 100)); + StringUtil.repeatSymbol(' ', PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE - 100)); VirtualFile virtualFile = getVirtualFile(f); assertEquals(PlainTextFileType.INSTANCE, getFileType(virtualFile)); PsiFile psiFile = getPsiManager().findFile(virtualFile); diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/PersistentFsTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/PersistentFsTest.java index 586d06ccab52..2fc57216682a 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/PersistentFsTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/PersistentFsTest.java @@ -18,6 +18,7 @@ import com.intellij.openapi.roots.ModuleRootModificationUtil; import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.SystemInfo; +import com.intellij.openapi.util.ThrowableComputable; import com.intellij.openapi.util.io.ByteArraySequence; import com.intellij.openapi.util.io.FileAttributes; import com.intellij.openapi.util.io.FileUtil; @@ -66,6 +67,7 @@ import static com.intellij.testFramework.EdtTestUtil.runInEdtAndWait; import static com.intellij.testFramework.UsefulTestCase.*; import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -73,7 +75,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.junit.Assert.*; import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; @@ -1021,14 +1022,17 @@ public class PersistentFsTest extends BareTestFixtureTestCase { @Test public void testContentReadingWhileModification() throws IOException { - byte[] initialContent = StringUtil.repeat("one_two", 500_000).getBytes(UTF_8); + byte[] initialContent = StringUtil.repeat( + "one_two", + PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE / "one_two".length() - 1 + ).getBytes(UTF_8); File file = tempDirectory.newFile("test.txt", initialContent); VirtualFile vFile = refreshAndFind(file); int id = ((VirtualFileWithId)vFile).getId(); vFile.contentsToByteArray(); InputStream stream = FSRecords.getInstance().readContent(id); - assertNotNull(stream); + assertNotNull("Content must be cached", stream); byte[] bytes = stream.readNBytes(initialContent.length); assertArrayEquals(initialContent, bytes); InputStream stream2 = FSRecords.getInstance().readContent(id); @@ -1040,6 +1044,38 @@ public class PersistentFsTest extends BareTestFixtureTestCase { assertArrayEquals(initialContent, ArrayUtil.mergeArrays(portion1, portion2)); } + @Test + public void testHugeFileContentIsNotCachedInVFS() throws IOException { + byte[] hugeContent = StringUtil.repeat( + "anything", + PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE / "anything".length() + 1 + ).getBytes(UTF_8); + assertTrue("Content must be larger than limit", + hugeContent.length > PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE); + + File file = tempDirectory.newFile("test.txt", hugeContent); + VirtualFile vFile = refreshAndFind(file); + int id = ((VirtualFileWithId)vFile).getId(); + + //load content: for smaller files this should trigger file content caching, but not for huge files + vFile.contentsToByteArray(); + + assertNull("Content must NOT be cached in VFS during reading", + FSRecords.getInstance().readContent(id)); + + //save content: for smaller files this should trigger file content caching, but not for huge files + hugeContent[10] = 'A';//introduce a change + ApplicationManager.getApplication().runWriteAction((ThrowableComputable)() -> { + try (var out = vFile.getOutputStream(this)) { + out.write(hugeContent); + } + return null; + }); + + assertNull("Content must NOT be cached in VFS during saving", + FSRecords.getInstance().readContent(id)); + } + @Test public void testSearchingForJarRootWhenItsNotCached() throws IOException { // IDEA-341011 PersistentFSImpl.cacheMissedRootFromPersistence fails to find jars because it uses name instead of path diff --git a/platform/util-rt/src/com/intellij/openapi/util/io/FileUtilRt.java b/platform/util-rt/src/com/intellij/openapi/util/io/FileUtilRt.java index b59ce81f4d23..a4b41d063a8d 100644 --- a/platform/util-rt/src/com/intellij/openapi/util/io/FileUtilRt.java +++ b/platform/util-rt/src/com/intellij/openapi/util/io/FileUtilRt.java @@ -30,11 +30,9 @@ public final class FileUtilRt { public static final int MEGABYTE = KILOBYTE * KILOBYTE; /** - * This threshold has a mixed semantics, it is used in at least 2 roles: - * 1. Max file size to store ('cache') in VFS (i.e. PersistentFSImpl) - * 2. Just 'big enough' file size, to switch from simple one-chunk loading to incremental loading, or not load it at all - * (e.g. {@linkplain #isTooLarge(long)}, JupyterFileUtils, JBZipEntry) - * TODO RC: those roles are different enough to split them into independent thresholds + * File size that is 'big enough' to load. + * Used to either skip the file content loading completely, if larger -- or at least to switch from simple + * one-chunk loading to more memory-efficient incremental loading * @deprecated Prefer using @link {@link com.intellij.openapi.vfs.limits.FileSizeLimit#getContentLoadLimit} */ @SuppressWarnings("DeprecatedIsStillUsed") diff --git a/platform/vcs-api/src/com/intellij/vcsUtil/VcsUtil.java b/platform/vcs-api/src/com/intellij/vcsUtil/VcsUtil.java index fa8023fea2a8..905713d70e94 100644 --- a/platform/vcs-api/src/com/intellij/vcsUtil/VcsUtil.java +++ b/platform/vcs-api/src/com/intellij/vcsUtil/VcsUtil.java @@ -64,7 +64,7 @@ public final class VcsUtil { } private static int computeLoadedFileSize() { - long result = PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD; + long result = PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE; try { String userLimitKb = System.getProperty(MAX_VCS_LOADED_SIZE_KB); if (userLimitKb != null) {