[vfs][refactoring] PY-77389: extract VFS content caching limit into dedicated constant

`FileUtilRt.LARGE_FOR_CONTENT_LOADING` was used in 2 roles: as just 'big file, load with caution', and also 'do not cache file content in VFS' -- but those roles are quite different really, and should not be mixed =>
- `PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE` is now the limit for VFS caching: default value 1Mb, `-Didea.vfs.max-file-length-to-cache=...` to override
- `FileUtilRt.LARGE_FOR_CONTENT_LOADING` is still used for everything else

(cherry picked from commit f7642bf36cba9984f5a6438c88fcecbe769335a8)


(cherry picked from commit 77ef2bc348054154fba9b612f75bcc41ac880f64)

IJ-CR-150186

GitOrigin-RevId: e8a95f377142a793f171d5ba055ab54dc9bc3d6c

platform/core-api/api-dump-unreviewed.txt

@@ -2662,7 +2662,6 @@ c:com.intellij.openapi.vfs.InvalidVirtualFileAccessException
 com.intellij.openapi.vfs.LocalFileProvider
 com.intellij.openapi.vfs.NonPhysicalFileSystem
 f:com.intellij.openapi.vfs.PersistentFSConstants
-- sf:FILE_LENGTH_TO_CACHE_THRESHOLD:J
 - s:getMaxIntellisenseFileSize():I
 - s:setMaxIntellisenseFileSize(I):V
 a:com.intellij.openapi.vfs.ReadonlyStatusHandler

platform/core-api/src/com/intellij/openapi/vfs/PersistentFSConstants.java

@@ -3,24 +3,35 @@ package com.intellij.openapi.vfs;
 
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.util.io.FileUtilRt;
-import com.intellij.openapi.vfs.limits.FileSizeLimit;
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.TestOnly;
 
+import static com.intellij.util.SystemProperties.getIntProperty;
+
 public final class PersistentFSConstants {
 
-  /** Max file size to cache inside VFS */
-  public static final long FILE_LENGTH_TO_CACHE_THRESHOLD = FileSizeLimit.getFileLengthToCacheThreshold();
+  /**
+   * Max file size to cache inside VFS.
+   * Cache huge files is rarely useful, but time-consuming (freeze-prone), and could quickly overflow VFS content storage capacity
+   */
+  @ApiStatus.Internal
+  public static final int MAX_FILE_LENGTH_TO_CACHE = getIntProperty(
+    "idea.vfs.max-file-length-to-cache",
+    FileUtilRt.MEGABYTE
+  );
 
   /**
-   * Must always be in range [0, {@link #FILE_LENGTH_TO_CACHE_THRESHOLD}]
+   * Must always be in range [0, {@link FileUtilRt#LARGE_FOR_CONTENT_LOADING}]
    * <p>
    * Currently, this is always true, because
-   * <pre>FILE_LENGTH_TO_CACHE_THRESHOLD = ... = max(20Mb, userFileSizeLimit, userContentLoadLimit)</pre>
+   * <pre>LARGE_FOR_CONTENT_LOADING = ... = max(20Mb, userFileSizeLimit, userContentLoadLimit)</pre>
    * but could be changed in the future, hence .min(...) here is to ensure that.
+   * TODO: move into FileSizeLimit
    */
-  private static int ourMaxIntellisenseFileSize = Math.min(FileUtilRt.getUserFileSizeLimit(), (int)FILE_LENGTH_TO_CACHE_THRESHOLD);
+  private static int ourMaxIntellisenseFileSize = Math.min(FileUtilRt.getUserFileSizeLimit(), FileUtilRt.LARGE_FOR_CONTENT_LOADING);
 
   /** @deprecated Prefer using {@link com.intellij.openapi.vfs.limits.FileSizeLimit#getIntellisenseLimit(String)} */
+  //TODO: move into FileSizeLimit
   @SuppressWarnings("DeprecatedIsStillUsed")
   @Deprecated()
   public static int getMaxIntellisenseFileSize() {

platform/core-api/src/com/intellij/openapi/vfs/limits/FileSizeLimit.kt

@@ -21,11 +21,6 @@ interface FileSizeLimit {
 
     private val limitsByExtension: AtomicReference<Map<String, ExtensionSizeLimitInfo>?> = AtomicReference(null)
 
-    /** Max file size to cache inside VFS */
-    @JvmStatic
-    @ApiStatus.Internal
-    fun getFileLengthToCacheThreshold(): Int = FileUtilRt.LARGE_FOR_CONTENT_LOADING
-
     private fun getLimitsByExtension(): Map<String, ExtensionSizeLimitInfo> {
       return limitsByExtension.get() ?: limitsByExtension.updateAndGet { getLimits() }!!
     }
@@ -56,12 +51,12 @@ interface FileSizeLimit {
     @JvmStatic
     fun getContentLoadLimit(extension: String?): Int {
       @Suppress("DEPRECATION")
-      val limit = findApplicable(extension ?: "")?.content ?: FileUtilRt.LARGE_FOR_CONTENT_LOADING
+      val limit = findApplicable(extension ?: "")?.content ?: getDefaultContentLoadLimit()
       return limit
     }
 
     @JvmStatic
-    fun getDefaultContentLoadLimit(): Int = getContentLoadLimit(null)
+    fun getDefaultContentLoadLimit(): Int = FileUtilRt.LARGE_FOR_CONTENT_LOADING
 
     @JvmStatic
     fun getIntellisenseLimit(): Int = getIntellisenseLimit(null)

platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java

@@ -155,7 +155,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable {
 
     setupOTelMonitoring(TelemetryManager.getInstance().getMeter(PlatformScopesKt.VFS));
 
-    LOG.info("VFS.maxFileLengthToCache: " + PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD);
+    LOG.info("VFS.MAX_FILE_LENGTH_TO_CACHE: " + PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE);
   }
 
   @ApiStatus.Internal
@@ -909,7 +909,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable {
   }
 
   private static boolean shouldCacheFileContentInVFS(long fileLength) {
-    return fileLength <= PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD;
+    return fileLength <= PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE;
   }
 
   private @NotNull InputStream createReplicatorAndStoreContent(@NotNull VirtualFile file,
@@ -1541,8 +1541,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable {
     if (!(vf instanceof VirtualDirectoryImpl)) {
       return;
     }
-    parent =
-      (VirtualDirectoryImpl)vf;  // retain in `myIdToDirCache` at least for the duration of this block, so that subsequent `findFileById` won't crash
+    parent = (VirtualDirectoryImpl)vf;  // retain in `myIdToDirCache` at least for the duration of this block, so that subsequent `findFileById` won't crash
     NewVirtualFileSystem fs = getFileSystem(parent);
 
     List<ChildInfo> childrenAdded = new ArrayList<>(createEvents.size());
@@ -1559,8 +1558,7 @@ public final class PersistentFSImpl extends PersistentFS implements Disposable {
     childrenAdded.sort(ChildInfo.BY_ID);
     boolean caseSensitive = parent.isCaseSensitive();
     vfsPeer.update(parent, parentId, oldChildren -> oldChildren.merge(vfsPeer, childrenAdded, caseSensitive));
-    parent.createAndAddChildren(childrenAdded, false, (__, ___) -> {
-    });
+    parent.createAndAddChildren(childrenAdded, false, (__, ___) -> {});
 
     saveScannedChildrenRecursively(createEvents, fs, parent.isCaseSensitive());
   }

platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java

@@ -4,9 +4,7 @@ package com.intellij.openapi.vfs.newvfs.persistent;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.NotNullLazyValue;
 import com.intellij.openapi.util.io.FileUtil;
-import com.intellij.openapi.util.io.FileUtilRt;
-import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageHelper;
-import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageOverMMappedFile;
+import com.intellij.openapi.vfs.PersistentFSConstants;
 import com.intellij.openapi.vfs.newvfs.persistent.dev.content.CompressingAlgo;
 import com.intellij.openapi.vfs.newvfs.persistent.dev.content.ContentHashEnumeratorOverDurableEnumerator;
 import com.intellij.openapi.vfs.newvfs.persistent.dev.content.ContentStorageAdapter;
@@ -15,6 +13,8 @@ import com.intellij.openapi.vfs.newvfs.persistent.dev.enumerator.DurableStringEn
 import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSRecoverer;
 import com.intellij.openapi.vfs.newvfs.persistent.recovery.VFSRecoveryInfo;
 import com.intellij.platform.util.io.storages.StorageFactory;
+import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageHelper;
+import com.intellij.platform.util.io.storages.blobstorage.StreamlinedBlobStorageOverMMappedFile;
 import com.intellij.platform.util.io.storages.mmapped.MMappedFileStorageFactory;
 import com.intellij.util.ExceptionUtil;
 import com.intellij.util.concurrency.SequentialTaskExecutor;
@@ -23,7 +23,9 @@ import com.intellij.util.io.*;
 import com.intellij.util.io.blobstorage.SpaceAllocationStrategy;
 import com.intellij.util.io.blobstorage.SpaceAllocationStrategy.DataLengthPlusFixedPercentStrategy;
 import com.intellij.util.io.blobstorage.StreamlinedBlobStorage;
-import com.intellij.util.io.storage.*;
+import com.intellij.util.io.storage.RefCountingContentStorage;
+import com.intellij.util.io.storage.RefCountingContentStorageImpl;
+import com.intellij.util.io.storage.VFSContentStorage;
 import com.intellij.util.io.storage.lf.RefCountingContentStorageImplLF;
 import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.ints.IntList;
@@ -529,10 +531,10 @@ public final class PersistentFSLoader {
       //Use larger pages: content storage is usually quite big.
       int pageSize = 64 * IOUtil.MiB;
 
-      if (pageSize <= FileUtilRt.LARGE_FOR_CONTENT_LOADING) {
+      if (pageSize <= PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE) {
         //pageSize is an upper limit on record size for AppendOnlyLogOverMMappedFile:
         LOG.warn("ContentStorage.pageSize(=" + pageSize + ") " +
-                 "must be > FileUtilRt.LARGE_FOR_CONTENT_LOADING(=" + FileUtilRt.LARGE_FOR_CONTENT_LOADING + "b), " +
+                 "must be > PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE(=" + PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE + "b), " +
                  "otherwise large content can't fit");
       }
       CompressingAlgo compressionAlgo = switch (FSRecordsImpl.COMPRESSION_ALGO) {

platform/platform-tests/testSrc/com/intellij/openapi/fileTypes/impl/FileTypesTest.java

@@ -899,7 +899,7 @@ public class FileTypesTest extends HeavyPlatformTestCase {
 
           if (random.nextInt(3) == 0) {
             WriteCommandAction.writeCommandAction(getProject()).run(() -> {
-              byte[] bytes = new byte[(int)PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD + (isText ? 1 : 0)];
+              byte[] bytes = new byte[PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE + (isText ? 1 : 0)];
               Arrays.fill(bytes, (byte)' ');
               virtualFile.setBinaryContent(bytes);
             });
@@ -928,7 +928,7 @@ public class FileTypesTest extends HeavyPlatformTestCase {
     FrequentEventDetector.disableUntil(getTestRootDisposable());
 
     File f = createTempFile("xx.asd_kjf_hlk_asj_dhf",
-                            StringUtil.repeatSymbol(' ', (int)PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD - 100));
+                            StringUtil.repeatSymbol(' ', PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE - 100));
     VirtualFile virtualFile = getVirtualFile(f);
     assertEquals(PlainTextFileType.INSTANCE, getFileType(virtualFile));
     PsiFile psiFile = getPsiManager().findFile(virtualFile);

platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/PersistentFsTest.java

@@ -18,6 +18,7 @@ import com.intellij.openapi.roots.ModuleRootModificationUtil;
 import com.intellij.openapi.util.Disposer;
 import com.intellij.openapi.util.Ref;
 import com.intellij.openapi.util.SystemInfo;
+import com.intellij.openapi.util.ThrowableComputable;
 import com.intellij.openapi.util.io.ByteArraySequence;
 import com.intellij.openapi.util.io.FileAttributes;
 import com.intellij.openapi.util.io.FileUtil;
@@ -66,6 +67,7 @@ import static com.intellij.testFramework.EdtTestUtil.runInEdtAndWait;
 import static com.intellij.testFramework.UsefulTestCase.*;
 import static java.nio.charset.StandardCharsets.ISO_8859_1;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -73,7 +75,6 @@ import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.junit.Assert.*;
 import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 
@@ -1021,14 +1022,17 @@ public class PersistentFsTest extends BareTestFixtureTestCase {
 
   @Test
   public void testContentReadingWhileModification() throws IOException {
-    byte[] initialContent = StringUtil.repeat("one_two", 500_000).getBytes(UTF_8);
+    byte[] initialContent = StringUtil.repeat(
+      "one_two",
+      PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE / "one_two".length() - 1
+    ).getBytes(UTF_8);
     File file = tempDirectory.newFile("test.txt", initialContent);
     VirtualFile vFile = refreshAndFind(file);
     int id = ((VirtualFileWithId)vFile).getId();
     vFile.contentsToByteArray();
 
     InputStream stream = FSRecords.getInstance().readContent(id);
-    assertNotNull(stream);
+    assertNotNull("Content must be cached", stream);
     byte[] bytes = stream.readNBytes(initialContent.length);
     assertArrayEquals(initialContent, bytes);
     InputStream stream2 = FSRecords.getInstance().readContent(id);
@@ -1040,6 +1044,38 @@ public class PersistentFsTest extends BareTestFixtureTestCase {
     assertArrayEquals(initialContent, ArrayUtil.mergeArrays(portion1, portion2));
   }
 
+  @Test
+  public void testHugeFileContentIsNotCachedInVFS() throws IOException {
+    byte[] hugeContent = StringUtil.repeat(
+      "anything",
+      PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE / "anything".length() + 1
+    ).getBytes(UTF_8);
+    assertTrue("Content must be larger than limit",
+               hugeContent.length > PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE);
+
+    File file = tempDirectory.newFile("test.txt", hugeContent);
+    VirtualFile vFile = refreshAndFind(file);
+    int id = ((VirtualFileWithId)vFile).getId();
+
+    //load content: for smaller files this should trigger file content caching, but not for huge files
+    vFile.contentsToByteArray();
+
+    assertNull("Content must NOT be cached in VFS during reading",
+               FSRecords.getInstance().readContent(id));
+
+    //save content: for smaller files this should trigger file content caching, but not for huge files
+    hugeContent[10] = 'A';//introduce a change
+    ApplicationManager.getApplication().runWriteAction((ThrowableComputable<Object, IOException>)() -> {
+      try (var out = vFile.getOutputStream(this)) {
+        out.write(hugeContent);
+      }
+      return null;
+    });
+
+    assertNull("Content must NOT be cached in VFS during saving",
+               FSRecords.getInstance().readContent(id));
+  }
+
   @Test
   public void testSearchingForJarRootWhenItsNotCached() throws IOException {
     // IDEA-341011 PersistentFSImpl.cacheMissedRootFromPersistence fails to find jars because it uses name instead of path

platform/util-rt/src/com/intellij/openapi/util/io/FileUtilRt.java

@@ -30,11 +30,9 @@ public final class FileUtilRt {
   public static final int MEGABYTE = KILOBYTE * KILOBYTE;
 
   /**
-   * This threshold has a mixed semantics, it is used in at least 2 roles:
-   * 1. Max file size to store ('cache') in VFS (i.e. PersistentFSImpl)
-   * 2. Just 'big enough' file size, to switch from simple one-chunk loading to incremental loading, or not load it at all
-   *    (e.g. {@linkplain #isTooLarge(long)}, JupyterFileUtils, JBZipEntry)
-   * TODO RC: those roles are different enough to split them into independent thresholds
+   * File size that is 'big enough' to load.
+   * Used to either skip the file content loading completely, if larger -- or at least to switch from simple
+   * one-chunk loading to more memory-efficient incremental loading
    * @deprecated Prefer using @link {@link com.intellij.openapi.vfs.limits.FileSizeLimit#getContentLoadLimit}
    */
   @SuppressWarnings("DeprecatedIsStillUsed")

platform/vcs-api/src/com/intellij/vcsUtil/VcsUtil.java

@@ -64,7 +64,7 @@ public final class VcsUtil {
   }
 
   private static int computeLoadedFileSize() {
-    long result = PersistentFSConstants.FILE_LENGTH_TO_CACHE_THRESHOLD;
+    long result = PersistentFSConstants.MAX_FILE_LENGTH_TO_CACHE;
     try {
       String userLimitKb = System.getProperty(MAX_VCS_LOADED_SIZE_KB);
       if (userLimitKb != null) {