[git-index] introduce vfs for index files

GitIndexFileSystem is introduced to allow viewing and editing index content directly. As doing so requires running git commands, some of the operations could not be performed synchronously since it requires running git in edt. To deal with this problem, read and write on disk are performed in background and are protected by a lock (separate for each file). Each file has cached information kept in an AtomicReference, containing hash, length and executable flag.

Refresh is done in two stages: CachedData for refreshed files is retrieved in background and then assigned in write action. Write is done in three stages: "before" events are sent in write action, actual write is performed in background outside of write action, "after" events are sent in write action.

GitOrigin-RevId: 7fe99dd3e52b3208a1d38707a877b5d84831f599

plugins/git4idea/resources/META-INF/stage-plugin.xml

@@ -5,6 +5,10 @@
                         displayNameSupplierClassName="git4idea.index.GitStageDisplayNameSupplier"/>
 
     <projectService serviceImplementation="git4idea.index.GitStageTracker"/>
+    <projectService serviceImplementation="git4idea.index.vfs.GitIndexVirtualFileCache"/>
+    <projectService serviceImplementation="git4idea.index.vfs.GitIndexFileSystemRefresher"/>
+
+    <virtualFileSystem implementationClass="git4idea.index.vfs.GitIndexFileSystem" key="gitIndexFs" physical="true"/>
   </extensions>
 
   <actions>

plugins/git4idea/src/git4idea/index/GitIndexUtil.java

@@ -59,11 +59,20 @@ public class GitIndexUtil {
   }
 
   @NotNull
-  public static List<StagedFile> listStaged(@NotNull GitRepository repository, @NotNull Collection<? extends FilePath> filePaths) throws VcsException {
-    List<StagedFile> result = new ArrayList<>();
+  public static List<StagedFile> listStaged(@NotNull GitRepository repository, @NotNull Collection<? extends FilePath> filePaths)
+    throws VcsException {
+    Project project = repository.getProject();
     VirtualFile root = repository.getRoot();
 
-    GitLineHandler h = new GitLineHandler(repository.getProject(), root, GitCommand.LS_FILES);
+    return listStaged(project, root, filePaths);
+  }
+
+  @NotNull
+  public static List<StagedFile> listStaged(@NotNull Project project,
+                                            @NotNull VirtualFile root,
+                                            @NotNull Collection<? extends FilePath> filePaths) throws VcsException {
+    List<StagedFile> result = new ArrayList<>();
+    GitLineHandler h = new GitLineHandler(project, root, GitCommand.LS_FILES);
     h.addParameters("-s");
     h.endOptions();
     h.addRelativePaths(filePaths);
@@ -176,20 +185,24 @@ public class GitIndexUtil {
   }
 
   @NotNull
-  public static Hash write(@NotNull GitRepository repository,
-                           @NotNull FilePath filePath,
-                           @NotNull InputStream content,
+  public static Hash write(@NotNull GitRepository repository, @NotNull FilePath filePath, @NotNull InputStream content,
                            boolean executable) throws VcsException {
-    Hash hash = hashObject(repository, filePath, content);
-    updateIndex(repository, filePath, hash, executable);
+    return write(repository.getProject(), repository.getRoot(), filePath, content, executable);
+  }
+
+  @NotNull
+  public static Hash write(@NotNull Project project, @NotNull VirtualFile root,
+                           @NotNull FilePath filePath, @NotNull InputStream content,
+                           boolean executable) throws VcsException {
+    Hash hash = hashObject(project, root, filePath, content);
+    updateIndex(project, root, filePath, hash, executable);
     return hash;
   }
 
   @NotNull
-  private static Hash hashObject(@NotNull GitRepository repository,
-                                 @NotNull FilePath filePath,
+  private static Hash hashObject(@NotNull Project project, @NotNull VirtualFile root, @NotNull FilePath filePath,
                                  @NotNull InputStream content) throws VcsException {
-    GitLineHandler h = new GitLineHandler(repository.getProject(), repository.getRoot(), GitCommand.HASH_OBJECT);
+    GitLineHandler h = new GitLineHandler(project, root, GitCommand.HASH_OBJECT);
     h.setSilent(true);
     h.addParameters("-w", "--stdin");
     h.addParameters("--path");
@@ -204,11 +217,17 @@ public class GitIndexUtil {
                                  @NotNull FilePath filePath,
                                  @NotNull Hash blobHash,
                                  boolean isExecutable) throws VcsException {
-    String mode = isExecutable ? EXECUTABLE_MODE : DEFAULT_MODE;
-    String path = VcsFileUtil.relativePath(repository.getRoot(), filePath);
+    updateIndex(repository.getProject(), repository.getRoot(), filePath, blobHash, isExecutable);
+  }
 
-    GitLineHandler h = new GitLineHandler(repository.getProject(), repository.getRoot(), GitCommand.UPDATE_INDEX);
-    if (GitVersionSpecialty.CACHEINFO_SUPPORTS_SINGLE_PARAMETER_FORM.existsIn(repository)) {
+  public static void updateIndex(@NotNull Project project, @NotNull VirtualFile root, @NotNull FilePath filePath,
+                                 @NotNull Hash blobHash,
+                                 boolean isExecutable) throws VcsException {
+    String mode = isExecutable ? EXECUTABLE_MODE : DEFAULT_MODE;
+    String path = VcsFileUtil.relativePath(root, filePath);
+
+    GitLineHandler h = new GitLineHandler(project, root, GitCommand.UPDATE_INDEX);
+    if (GitVersionSpecialty.CACHEINFO_SUPPORTS_SINGLE_PARAMETER_FORM.existsIn(project)) {
       h.addParameters("--cacheinfo", mode + "," + blobHash.asString() + "," + path);
     }
     else {

plugins/git4idea/src/git4idea/index/vfs/GitIndexFileSystem.kt

@@ -0,0 +1,56 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package git4idea.index.vfs
+
+import com.intellij.openapi.components.service
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileListener
+import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.openapi.vfs.VirtualFileSystem
+import com.intellij.openapi.vfs.newvfs.VirtualFileFilteringListener
+import com.intellij.util.containers.ContainerUtil
+
+private const val PROTOCOL = "gitIndexFs"
+
+class GitIndexFileSystem : VirtualFileSystem() {
+  private val listenerWrappers: MutableMap<VirtualFileListener, VirtualFileListener> = ContainerUtil.newConcurrentMap()
+
+  override fun getProtocol(): String = PROTOCOL
+  override fun isReadOnly(): Boolean = false
+
+  override fun findFileByPath(path: String): VirtualFile? {
+    val (project, virtualFile, filePath) = GitIndexVirtualFile.decode(path) ?: return null
+
+    project.service<GitIndexFileSystemRefresher>() // init service
+    return project.service<GitIndexVirtualFileCache>().get(virtualFile, filePath)
+  }
+
+  override fun refreshAndFindFileByPath(path: String): VirtualFile? = findFileByPath(path)
+  override fun refresh(asynchronous: Boolean) {}
+
+  override fun extractPresentableUrl(path: String): String = "Staged: " + super.extractPresentableUrl(path)
+
+  override fun addVirtualFileListener(listener: VirtualFileListener) {
+    val wrapper: VirtualFileListener = VirtualFileFilteringListener(listener, this)
+    VirtualFileManager.getInstance().addVirtualFileListener(wrapper)
+    listenerWrappers[listener] = wrapper
+  }
+
+  override fun removeVirtualFileListener(listener: VirtualFileListener) {
+    listenerWrappers.remove(listener)?.let {
+      VirtualFileManager.getInstance().removeVirtualFileListener(it)
+    }
+  }
+
+  override fun deleteFile(requestor: Any?, vFile: VirtualFile) = throw UnsupportedOperationException()
+  override fun moveFile(requestor: Any?, vFile: VirtualFile, newParent: VirtualFile) = throw UnsupportedOperationException()
+  override fun renameFile(requestor: Any?, vFile: VirtualFile, newName: String) = throw UnsupportedOperationException()
+  override fun createChildFile(requestor: Any?, vDir: VirtualFile, fileName: String): VirtualFile = throw UnsupportedOperationException()
+  override fun createChildDirectory(requestor: Any?, vDir: VirtualFile, dirName: String): VirtualFile = throw UnsupportedOperationException()
+  override fun copyFile(requestor: Any?, virtualFile: VirtualFile, newParent: VirtualFile, copyName: String): VirtualFile  = throw UnsupportedOperationException()
+
+  companion object {
+    @JvmStatic
+    val instance: GitIndexFileSystem
+      get() = VirtualFileManager.getInstance().getFileSystem(PROTOCOL) as GitIndexFileSystem
+  }
+}

plugins/git4idea/src/git4idea/index/vfs/GitIndexFileSystemRefresher.kt

@@ -0,0 +1,101 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package git4idea.index.vfs
+
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.components.serviceIfCreated
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.openapi.vfs.newvfs.BulkFileListener
+import com.intellij.openapi.vfs.newvfs.events.VFileContentChangeEvent
+import com.intellij.openapi.vfs.newvfs.events.VFileEvent
+import com.intellij.util.concurrency.AppExecutorUtil
+import com.intellij.util.messages.MessageBusConnection
+import com.intellij.vcs.log.submitSafe
+import git4idea.repo.GitRepositoryManager
+import git4idea.repo.GitUntrackedFilesHolder
+import org.jetbrains.annotations.CalledWithWriteLock
+
+class GitIndexFileSystemRefresher(private val project: Project) : Disposable {
+  private val executor = AppExecutorUtil.createBoundedApplicationPoolExecutor("Git Index Read/Write Thread for " + project.name, 1)
+
+  init {
+    val connection: MessageBusConnection = project.messageBus.connect(this)
+    connection.subscribe(VirtualFileManager.VFS_CHANGES, object : BulkFileListener {
+      override fun after(events: List<VFileEvent>) {
+        val roots = GitRepositoryManager.getInstance(project).repositories.filter { repo ->
+          events.any { e -> GitUntrackedFilesHolder.indexChanged(repo, e.path) }
+        }.map { it.root }
+        if (roots.isNotEmpty()) {
+          LOG.debug("Scheduling refresh for ${roots.joinToString { it.name }}")
+          refresh(roots)
+        }
+      }
+    })
+  }
+
+  private fun refresh(roots: List<VirtualFile>) {
+    executor.submitSafe(LOG) {
+      val refreshRunnables = mutableListOf<GitIndexVirtualFile.Refresh>()
+      project.serviceIfCreated<GitIndexVirtualFileCache>()?.forEachFile { file ->
+        if (roots.contains(file.root)) {
+          file.getRefresh()?.let { refreshRunnables.add(it) }
+          LOG.debug("Preparing refresh for $file")
+        }
+      }
+      if (refreshRunnables.isNotEmpty()) execute(refreshRunnables)
+    }
+  }
+
+  internal fun execute(refreshList: List<GitIndexVirtualFile.Refresh>, postRunnable: Runnable? = null) {
+    val events = refreshList.map { it.event }
+    writeInEdt {
+      ApplicationManager.getApplication().messageBus.syncPublisher(VirtualFileManager.VFS_CHANGES).before(events)
+      refreshList.forEach { it.run() }
+      ApplicationManager.getApplication().messageBus.syncPublisher(VirtualFileManager.VFS_CHANGES).after(events)
+      postRunnable?.run()
+    }
+  }
+
+  internal fun runTask(asynchronous: Boolean, task: () -> Unit) {
+    if (ApplicationManager.getApplication().isDispatchThread || asynchronous) {
+      executor.submitSafe(LOG, task)
+    }
+    else {
+      task()
+    }
+  }
+
+  @CalledWithWriteLock
+  internal fun changeContent(file: GitIndexVirtualFile, requestor: Any?, modificationStamp: Long, writeCommand: () -> Unit) {
+    ApplicationManager.getApplication().assertWriteAccessAllowed()
+
+    val event = VFileContentChangeEvent(requestor, file, file.modificationStamp, modificationStamp, false)
+    ApplicationManager.getApplication().messageBus.syncPublisher(VirtualFileManager.VFS_CHANGES).before(listOf(event))
+    executor.submitSafe(LOG) {
+      try {
+        writeCommand()
+      }
+      finally {
+        writeInEdt { ApplicationManager.getApplication().messageBus.syncPublisher(VirtualFileManager.VFS_CHANGES).after(listOf(event)) }
+      }
+    }
+  }
+
+  override fun dispose() {
+  }
+
+  companion object {
+    private val LOG = Logger.getInstance(GitIndexFileSystemRefresher::class.java)
+  }
+}
+
+fun writeInEdt(action: () -> Unit) {
+  ApplicationManager.getApplication().invokeLater {
+    ApplicationManager.getApplication().runWriteAction {
+      action()
+    }
+  }
+}

plugins/git4idea/src/git4idea/index/vfs/GitIndexVirtualFile.kt

@@ -0,0 +1,206 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package git4idea.index.vfs
+
+import com.intellij.openapi.components.service
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ProjectManager
+import com.intellij.openapi.util.text.StringUtil
+import com.intellij.openapi.vcs.FilePath
+import com.intellij.openapi.vcs.VcsException
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.openapi.vfs.VfsUtilCore
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.newvfs.events.VFileContentChangeEvent
+import com.intellij.util.LocalTimeCounter
+import com.intellij.vcs.log.Hash
+import com.intellij.vcs.log.impl.HashImpl
+import com.intellij.vcsUtil.VcsFileUtil
+import com.intellij.vcsUtil.VcsUtil
+import git4idea.commands.Git
+import git4idea.commands.GitCommand
+import git4idea.commands.GitLineHandler
+import git4idea.index.GitIndexUtil
+import git4idea.util.GitFileUtils
+import java.io.*
+import java.util.concurrent.atomic.AtomicReference
+import java.util.concurrent.locks.ReentrantReadWriteLock
+import kotlin.concurrent.read
+import kotlin.concurrent.write
+
+class GitIndexVirtualFile(private val project: Project,
+                          val root: VirtualFile,
+                          val filePath: FilePath) : VirtualFile() {
+  private val lock = ReentrantReadWriteLock()
+  private val refresher: GitIndexFileSystemRefresher get() = project.service()
+
+  private val cachedData: AtomicReference<CachedData?> = AtomicReference()
+
+  @Volatile
+  private var modificationStamp = LocalTimeCounter.currentTime()
+
+  init {
+    refresher.runTask(false) {
+      cachedData.compareAndSet(null, readCachedData())
+    }
+  }
+
+  override fun getFileSystem(): GitIndexFileSystem = GitIndexFileSystem.instance
+  override fun getParent(): VirtualFile? = null
+  override fun getChildren(): Array<VirtualFile> = EMPTY_ARRAY
+  override fun isWritable(): Boolean = true
+  override fun isDirectory(): Boolean = false
+  override fun isValid(): Boolean = cachedData.get() != null
+  override fun getName(): String = filePath.name
+  override fun getPath(): String = encode(project, root, filePath)
+  override fun getLength(): Long = cachedData.get()?.length ?: 0
+  override fun getTimeStamp(): Long = 0
+  override fun getModificationStamp(): Long = modificationStamp
+  override fun refresh(asynchronous: Boolean, recursive: Boolean, postRunnable: Runnable?) {
+    refresher.runTask(asynchronous) {
+      getRefresh()?.also { refresher.execute(listOf(it), postRunnable) } ?: writeInEdt { postRunnable?.run() }
+    }
+  }
+
+  internal fun getRefresh(): Refresh? {
+    val newCachedData = readCachedData()
+    val oldCachedData = cachedData.get()
+    if (oldCachedData != newCachedData) {
+      LOG.debug("Preparing refresh for $this")
+      return Refresh(oldCachedData, newCachedData, modificationStamp)
+    }
+    return null
+  }
+
+  @Throws(IOException::class)
+  override fun getOutputStream(requestor: Any?,
+                               newModificationStamp: Long,
+                               newTimeStamp: Long): OutputStream {
+    val outputStream: ByteArrayOutputStream = object : ByteArrayOutputStream() {
+      override fun close() = write(requestor, toByteArray(), newModificationStamp)
+    }
+    return VfsUtilCore.outputStreamAddingBOM(outputStream, this)
+  }
+
+  private fun write(requestor: Any?, newContent: ByteArray, newModificationStamp: Long) {
+    val newModStamp = if (newModificationStamp > 0) newModificationStamp else LocalTimeCounter.currentTime()
+    refresher.changeContent(this, requestor, modificationStamp) {
+      lock.write {
+        val oldCachedData = cachedData.get()
+        if (oldCachedData != readCachedData()) {
+          // TODO
+          LOG.warn("Skipping write for $this as it is not up to date")
+          return@write
+        }
+
+        val isExecutable = oldCachedData?.isExecutable ?: false
+        val newHash = GitIndexUtil.write(project, root, filePath, ByteArrayInputStream(newContent), isExecutable)
+        LOG.debug("Written $this. newHash=$newHash")
+
+        modificationStamp = newModStamp
+        if (oldCachedData?.hash != newHash) {
+          cachedData.compareAndSet(oldCachedData, CachedData(newHash, calculateLength(newHash.asString()), isExecutable))
+        }
+      }
+    }
+  }
+
+  @Throws(IOException::class)
+  override fun getInputStream(): InputStream {
+    return VfsUtilCore.byteStreamSkippingBOM(contentsToByteArray(), this)
+  }
+
+  @Throws(IOException::class)
+  override fun contentsToByteArray(): ByteArray {
+    return try {
+      lock.read {
+        GitFileUtils.getFileContent(project, root, "", VcsFileUtil.relativePath(root, filePath))
+      }
+    }
+    catch (e: VcsException) {
+      throw IOException(e)
+    }
+  }
+
+  private fun readCachedData(): CachedData? {
+    lock.read {
+      val stagedFile = GitIndexUtil.listStaged(project, root, listOf(filePath)).singleOrNull() ?: return null
+      return CachedData(HashImpl.build(stagedFile.blobHash), calculateLength(stagedFile.blobHash), stagedFile.isExecutable)
+    }
+  }
+
+  @Throws(VcsException::class)
+  private fun calculateLength(hash: String): Long {
+    val h = GitLineHandler(project, root, GitCommand.CAT_FILE)
+    h.setSilent(true)
+    h.addParameters("-s")
+    h.addParameters(hash)
+    h.endOptions()
+    val output = Git.getInstance().runCommand(h).getOutputOrThrow()
+    return java.lang.Long.valueOf(output.trim())
+  }
+
+  override fun equals(other: Any?): Boolean {
+    if (this === other) return true
+    if (javaClass != other?.javaClass) return false
+
+    other as GitIndexVirtualFile
+
+    if (root != other.root) return false
+    if (filePath != other.filePath) return false
+
+    return true
+  }
+
+  override fun hashCode(): Int {
+    var result = root.hashCode()
+    result = 31 * result + filePath.hashCode()
+    return result
+  }
+
+  override fun toString(): String {
+    return "GitIndexVirtualFile: [${root.name}]/${VcsFileUtil.relativePath(root, filePath)}"
+  }
+
+  internal inner class Refresh internal constructor(private val oldCachedData: CachedData?,
+                                                    private val newCachedData: CachedData?,
+                                                    oldModificationStamp: Long) {
+    val event: VFileContentChangeEvent = VFileContentChangeEvent(null, this@GitIndexVirtualFile, oldModificationStamp, -1,
+                                                                 0, 0,
+                                                                 oldCachedData?.length ?: 0, newCachedData?.length ?: 0, true)
+
+    fun run(): Boolean {
+      LOG.debug("Refreshing ${this@GitIndexVirtualFile}")
+      return cachedData.compareAndSet(oldCachedData, newCachedData)
+    }
+
+    override fun toString(): String {
+      return "GitIndexVirtualFile.Refresh: ${this@GitIndexVirtualFile}"
+    }
+  }
+
+  companion object {
+    private val LOG = Logger.getInstance(GitIndexVirtualFile::class.java)
+    const val SEPARATOR = ':'
+
+    private fun encode(project: Project, root: VirtualFile, filePath: FilePath): String {
+      return StringUtil.escapeChar(project.locationHash, SEPARATOR) + SEPARATOR +
+             StringUtil.escapeChar(root.path, SEPARATOR) + SEPARATOR +
+             StringUtil.escapeChar(filePath.path, SEPARATOR)
+    }
+
+    fun decode(path: String): Triple<Project, VirtualFile, FilePath>? {
+      val components = path.split(SEPARATOR)
+      if (components.size != 3) return null
+
+      val locationHash = StringUtil.unescapeChar(components[0], SEPARATOR)
+      val project = ProjectManager.getInstance().openProjects.firstOrNull { it.locationHash == locationHash } ?: return null
+      val root = LocalFileSystem.getInstance().findFileByPath(StringUtil.unescapeChar(components[1], SEPARATOR)) ?: return null
+      val filePath = VcsUtil.getFilePath(StringUtil.unescapeChar(components[2], SEPARATOR))
+
+      return Triple(project, root, filePath)
+    }
+  }
+
+  internal data class CachedData(val hash: Hash, val length: Long, val isExecutable: Boolean = false)
+}

plugins/git4idea/src/git4idea/index/vfs/GitIndexVirtualFileCache.kt

@@ -0,0 +1,29 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package git4idea.index.vfs
+
+import com.google.common.cache.CacheBuilder
+import com.google.common.cache.CacheLoader
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vcs.FilePath
+import com.intellij.openapi.vfs.VirtualFile
+
+class GitIndexVirtualFileCache(private val project: Project) : Disposable {
+  private val cache = CacheBuilder.newBuilder().weakValues().build<Key, GitIndexVirtualFile>(CacheLoader.from { key ->
+    GitIndexVirtualFile(project, key!!.root, key.filePath)
+  })
+
+  fun get(root: VirtualFile, filePath: FilePath): GitIndexVirtualFile {
+    return cache.get(Key(root, filePath))
+  }
+
+  fun forEachFile(function: (GitIndexVirtualFile) -> Unit) {
+    cache.asMap().forEach { (_, file) -> function(file) }
+  }
+
+  override fun dispose() {
+    cache.invalidateAll()
+  }
+
+  private data class Key(val root: VirtualFile, val filePath: FilePath)
+}

plugins/git4idea/src/git4idea/repo/GitRepositoryFiles.java

@@ -361,6 +361,14 @@ public class GitRepositoryFiles {
     VfsUtil.markDirtyAndRefresh(true, true, false, myMainDir, myWorktreeDir);
   }
 
+  /**
+   * Refresh .git/index asynchronously.
+   */
+  public void refreshIndexFile() {
+    VirtualFile indexFilePath = LocalFileSystem.getInstance().refreshAndFindFileByPath(myIndexFilePath);
+    VfsUtil.markDirtyAndRefresh(true, false, false, indexFilePath);
+  }
+
   /**
    * Refresh that part of .git repository files, which is not covered by {@link GitRepository#update()}, e.g. the {@code refs/tags/} dir.
    *

plugins/git4idea/src/git4idea/repo/GitUntrackedFilesHolder.java

@@ -279,7 +279,7 @@ public class GitUntrackedFilesHolder implements Disposable, AsyncVfsEventsListen
     return repository.getRepositoryFiles().isOrigHeadFile(path);
   }
 
-  private static boolean indexChanged(@NotNull GitRepository repository, @NotNull String path) {
+  public static boolean indexChanged(@NotNull GitRepository repository, @NotNull String path) {
     return repository.getRepositoryFiles().isIndexFile(path);
   }