From 49eaf96f59c48a0b5b4aa0b5e4e7f50271a6ba8c Mon Sep 17 00:00:00 2001
From: Ruslan Cheremin <ruslan.cheremin@jetbrains.com>
Date: Tue, 14 May 2024 19:41:50 +0200
Subject: [PATCH] [vfs] option for MMappedFileStorage to clean file if
 !page-aligned

+ In addition to expand/throw exception options existent before, option 'clean and re-create'. Useful to self-heal in case of different external causes of file truncation.

GitOrigin-RevId: 547ccabbbfe175904721045be0b62ca104201afc
---
 .../newvfs/persistent/PersistentFSLoader.java |  3 +-
 .../mapped/MappedFileStorageHelper.java       |  2 +
 .../mapped/MMappedFileStorageTest.java        | 27 +++++++--
 .../mmapped/MMappedFileStorageFactory.java    | 59 +++++++++++++------
 4 files changed, 67 insertions(+), 24 deletions(-)

diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java
index f2a100d1d23c..2eaec78a8daf 100644
--- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java
+++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSLoader.java
@@ -51,6 +51,7 @@ import java.util.function.Function;
 
 import static com.intellij.openapi.vfs.newvfs.persistent.PersistentFSRecordAccessor.hasDeletedFlag;
 import static com.intellij.openapi.vfs.newvfs.persistent.VFSInitException.ErrorCategory.*;
+import static com.intellij.platform.util.io.storages.mmapped.MMappedFileStorageFactory.IfNotPageAligned.EXPAND_FILE;
 import static com.intellij.util.io.storage.CapacityAllocationPolicy.FIVE_PERCENT_FOR_GROWTH;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.stream.Collectors.joining;
@@ -638,7 +639,7 @@ public final class PersistentFSLoader {
           .pageSize(pageSize)
           //mmapped and !mmapped storages have the same binary layout, so mmapped storage could inherit all the
           // data from non-mmapped -- the only 'migration' needed is to page-align the file:
-          .expandFileIfNotPageAligned(true)
+          .ifFileIsNotPageAligned(EXPAND_FILE)
           .wrapStorageSafely(
             attributesFile,
             storage -> new StreamlinedBlobStorageOverMMappedFile(storage, allocationStrategy)
diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/mapped/MappedFileStorageHelper.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/mapped/MappedFileStorageHelper.java
index 8a714835a0bc..ef633571f52e 100644
--- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/mapped/MappedFileStorageHelper.java
+++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/mapped/MappedFileStorageHelper.java
@@ -27,6 +27,7 @@ import java.util.function.IntSupplier;
 import java.util.function.IntUnaryOperator;
 import java.util.function.LongUnaryOperator;
 
+import static com.intellij.platform.util.io.storages.mmapped.MMappedFileStorageFactory.IfNotPageAligned.CLEAN;
 import static com.intellij.util.io.IOUtil.MiB;
 import static java.lang.invoke.MethodHandles.byteBufferViewVarHandle;
 import static java.nio.ByteOrder.nativeOrder;
@@ -88,6 +89,7 @@ public final class MappedFileStorageHelper implements Closeable, CleanableStorag
 
       return MMappedFileStorageFactory.withDefaults()
         .pageSize(DEFAULT_PAGE_SIZE)
+        .ifFileIsNotPageAligned(CLEAN)
         .wrapStorageSafely(
           absoluteStoragePath,
           mappedFileStorage -> {
diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/mapped/MMappedFileStorageTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/mapped/MMappedFileStorageTest.java
index 0313a2877fb0..271fc930b19c 100644
--- a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/mapped/MMappedFileStorageTest.java
+++ b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/newvfs/persistent/mapped/MMappedFileStorageTest.java
@@ -18,6 +18,7 @@ import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.concurrent.*;
 
+import static com.intellij.platform.util.io.storages.mmapped.MMappedFileStorageFactory.IfNotPageAligned.*;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -260,7 +261,7 @@ public class MMappedFileStorageTest {
     try {
       var storage = MMappedFileStorageFactory.withDefaults()
         .pageSize(PAGE_SIZE)
-        .expandFileIfNotPageAligned(false)
+        .ifFileIsNotPageAligned(THROW_EXCEPTION)
         .open(storagePath);
       storage.closeAndClean();
       fail("Storage must fail to open file with size != N*pageSize");
@@ -271,13 +272,13 @@ public class MMappedFileStorageTest {
   }
 
   @Test
-  public void mappedStorage_Factory_CanExpandStorageFileIfAskedTo_IfFileSize_IsNotPageAligned(@TempDir Path tempDir) throws IOException {
+  public void mappedStorage_Factory_CanExpandStorageFile_IfAskedTo_IfFileSize_IsNotPageAligned(@TempDir Path tempDir) throws IOException {
     Path storagePath = tempDir.resolve("storage.file").toAbsolutePath();
     //page un-aligned size:
     Files.write(storagePath, new byte[3 * PAGE_SIZE + 1]);
     try (var storage = MMappedFileStorageFactory.withDefaults()
       .pageSize(PAGE_SIZE)
-      .expandFileIfNotPageAligned(true)
+      .ifFileIsNotPageAligned(EXPAND_FILE)
       .open(storagePath)) {
       assertEquals(Files.size(storagePath),
                    4 * PAGE_SIZE,
@@ -288,6 +289,24 @@ public class MMappedFileStorageTest {
     }
   }
 
+  @Test
+  public void mappedStorage_Factory_CanCleanStorageFile_IfAskedTo_IfFileSize_IsNotPageAligned(@TempDir Path tempDir) throws IOException {
+    Path storagePath = tempDir.resolve("storage.file").toAbsolutePath();
+    //page un-aligned size:
+    Files.write(storagePath, new byte[3 * PAGE_SIZE + 1]);
+    try (var storage = MMappedFileStorageFactory.withDefaults()
+      .pageSize(PAGE_SIZE)
+      .ifFileIsNotPageAligned(CLEAN)
+      .open(storagePath)) {
+      assertEquals(Files.size(storagePath),
+                   0,
+                   "Storage file should be truncated");
+    }
+    finally {
+      storage.closeAndClean();
+    }
+  }
+
   @Test
   public void mappedStorage_Factory_OpensStorageSuccessfully_IfFileSize_IsNotPageAligned_ButThereIsUnfinishedMappingSign(@TempDir Path tempDir)
     throws IOException {
@@ -302,7 +321,7 @@ public class MMappedFileStorageTest {
 
     var storage = MMappedFileStorageFactory.withDefaults()
       .pageSize(PAGE_SIZE)
-      .expandFileIfNotPageAligned(false)
+      .ifFileIsNotPageAligned(THROW_EXCEPTION)
       .open(storagePath);
     storage.closeAndClean();
   }
diff --git a/platform/util/storages/src/com/intellij/platform/util/io/storages/mmapped/MMappedFileStorageFactory.java b/platform/util/storages/src/com/intellij/platform/util/io/storages/mmapped/MMappedFileStorageFactory.java
index f51f54942ac2..d031db0fba28 100644
--- a/platform/util/storages/src/com/intellij/platform/util/io/storages/mmapped/MMappedFileStorageFactory.java
+++ b/platform/util/storages/src/com/intellij/platform/util/io/storages/mmapped/MMappedFileStorageFactory.java
@@ -1,6 +1,8 @@
 // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package com.intellij.platform.util.io.storages.mmapped;
 
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.platform.util.io.storages.StorageFactory;
 import com.intellij.util.io.IOUtil;
 import org.jetbrains.annotations.ApiStatus;
@@ -12,16 +14,18 @@ import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 
+import static com.intellij.platform.util.io.storages.mmapped.MMappedFileStorageFactory.IfNotPageAligned.THROW_EXCEPTION;
 import static java.nio.file.StandardOpenOption.*;
 
 
 @ApiStatus.Internal
 public class MMappedFileStorageFactory implements StorageFactory<MMappedFileStorage> {
+  private static final Logger LOG = Logger.getInstance(MMappedFileStorageFactory.class);
 
   public static final int DEFAULT_PAGE_SIZE = IOUtil.MiB;
 
   public static MMappedFileStorageFactory withDefaults() {
-    return new MMappedFileStorageFactory(DEFAULT_PAGE_SIZE, false, true);
+    return new MMappedFileStorageFactory(DEFAULT_PAGE_SIZE, THROW_EXCEPTION, true);
   }
 
 
@@ -35,13 +39,13 @@ public class MMappedFileStorageFactory implements StorageFactory<MMappedFileStor
    * Useful mostly useful for something like backward compatibility: e.g. increase pageSize is a safe change for many
    * storages -- there is no migration needed, except to align the file size to the new pageSize.
    */
-  private final boolean expandFileIfNotPageAligned;
+  private final IfNotPageAligned ifFileNotPageAligned;
 
   /** If directories along the path to the file do not exist yet -- create them */
   private final boolean createParentDirectoriesIfNotExist;
 
   private MMappedFileStorageFactory(int pageSize,
-                                    boolean expandFileIfNotPageAligned,
+                                    @NotNull IfNotPageAligned ifFileNotPageAligned,
                                     boolean createParentDirectoriesIfNotExist) {
     if (pageSize <= 0) {
       throw new IllegalArgumentException("pageSize(=" + pageSize + ") must be >0");
@@ -50,22 +54,23 @@ public class MMappedFileStorageFactory implements StorageFactory<MMappedFileStor
       throw new IllegalArgumentException("pageSize(=" + pageSize + ") must be a power of 2");
     }
     this.pageSize = pageSize;
-    this.expandFileIfNotPageAligned = expandFileIfNotPageAligned;
+    this.ifFileNotPageAligned = ifFileNotPageAligned;
     this.createParentDirectoriesIfNotExist = createParentDirectoriesIfNotExist;
   }
 
   public MMappedFileStorageFactory pageSize(int pageSize) {
-    return new MMappedFileStorageFactory(pageSize, expandFileIfNotPageAligned, createParentDirectoriesIfNotExist);
+    return new MMappedFileStorageFactory(pageSize, ifFileNotPageAligned, createParentDirectoriesIfNotExist);
   }
 
   /**
    * What to do if fileSize is not page-aligned (i.e. fileSize != N*pageSize), and there is no marker of unfinished
    * file expansion?
-   * true: expand (and fill with zeroes) the file, so it is page-aligned (fileSize=N * pageSize)
-   * false: throw IOException
+   * {@link IfNotPageAligned#EXPAND_FILE} (and fill with zeroes) the file, so it is page-aligned (fileSize=N * pageSize)
+   * {@link IfNotPageAligned#THROW_EXCEPTION}: throw IOException, {@link IfNotPageAligned#CLEAN} drop the current file
+   * content, and open the file as-new
    */
-  public MMappedFileStorageFactory expandFileIfNotPageAligned(boolean expand) {
-    return new MMappedFileStorageFactory(pageSize, expand, createParentDirectoriesIfNotExist);
+  public MMappedFileStorageFactory ifFileIsNotPageAligned(@NotNull IfNotPageAligned ifFileNotPageAligned) {
+    return new MMappedFileStorageFactory(pageSize, ifFileNotPageAligned, createParentDirectoriesIfNotExist);
   }
 
   /**
@@ -73,7 +78,7 @@ public class MMappedFileStorageFactory implements StorageFactory<MMappedFileStor
    * false: throw {@link NoSuchFileException} if parent directory doesn't exist
    */
   public MMappedFileStorageFactory createParentDirectories(boolean createParentDirectories) {
-    return new MMappedFileStorageFactory(pageSize, expandFileIfNotPageAligned, createParentDirectories);
+    return new MMappedFileStorageFactory(pageSize, ifFileNotPageAligned, createParentDirectories);
   }
 
   @Override
@@ -115,16 +120,24 @@ public class MMappedFileStorageFactory implements StorageFactory<MMappedFileStor
       return;
     }
 
-    if (expandFileIfNotPageAligned) {
-      //expand (zeroes) file up to the next page:
-      long fileSizeRoundedUpToPageSize = ((fileSize / pageSize) + 1) * pageSize;
-      try (FileChannel channel = FileChannel.open(storagePath, WRITE)) {
-        IOUtil.allocateFileRegion(channel, fileSizeRoundedUpToPageSize);
+    switch (ifFileNotPageAligned) {
+      case THROW_EXCEPTION -> {
+        throw new IOException("[" + storagePath + "]: fileSize(=" + fileSize + " b) is not page(=" + pageSize + " b)-aligned");
       }
-      return;
-    }
 
-    throw new IOException("[" + storagePath + "]: fileSize(=" + fileSize + " b) is not page(=" + pageSize + " b)-aligned");
+      case EXPAND_FILE -> {
+        LOG.warn("[" + storagePath + "]: fileSize(=" + fileSize + " b) is not page(=" + pageSize + " b)-aligned -> expand until aligned");
+        //expand (zeroes) file up to the next page:
+        long fileSizeRoundedUpToPageSize = ((fileSize / pageSize) + 1) * pageSize;
+        try (FileChannel channel = FileChannel.open(storagePath, WRITE)) {
+          IOUtil.allocateFileRegion(channel, fileSizeRoundedUpToPageSize);
+        }
+      }
+      case CLEAN -> {
+        LOG.warn("[" + storagePath + "]: fileSize(=" + fileSize + " b) is not page(=" + pageSize + " b)-aligned -> delete & re-create");
+        FileUtil.delete(storagePath);
+      }
+    }
   }
 
   private void checkParentDirectories(@NotNull Path storagePath) throws IOException {
@@ -145,8 +158,16 @@ public class MMappedFileStorageFactory implements StorageFactory<MMappedFileStor
   public String toString() {
     return "MMappedFileStorageFactory{" +
            "pageSize: " + pageSize +
-           ", expandFileIfNotPageAligned: " + expandFileIfNotPageAligned +
+           ", ifNotPageAligned: " + ifFileNotPageAligned +
            ", createParentDirectoriesIfNotExist: " + createParentDirectoriesIfNotExist +
            '}';
   }
+
+  public enum IfNotPageAligned {
+    /** Expand the file (and fill with zeros) until it is N*pageSize */
+    EXPAND_FILE,
+    THROW_EXCEPTION,
+    /** Clear the file content, and open as-if-fresh-new */
+    CLEAN
+  }
 }