From 4d05f19631cfb8d7a106371ce0e8d22370c5bb30 Mon Sep 17 00:00:00 2001 From: Dmitry Avdeev Date: Thu, 7 Oct 2010 15:35:40 +0400 Subject: [PATCH 1/4] avoid deserializing on update --- .../src/com/intellij/psi/stubs/SerializedStubTree.java | 9 ++++++--- .../src/com/intellij/psi/stubs/StubUpdatingIndex.java | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java b/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java index 89d999989cf3..ff98fd7edd2b 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/SerializedStubTree.java @@ -19,19 +19,22 @@ */ package com.intellij.psi.stubs; +import org.jetbrains.annotations.Nullable; + import java.io.ByteArrayInputStream; -import java.io.DataInputStream; import java.util.Arrays; public class SerializedStubTree { private final byte[] myBytes; + private final StubElement myStub; - public SerializedStubTree(final byte[] bytes) { + public SerializedStubTree(final byte[] bytes, @Nullable StubElement stub) { myBytes = bytes; + myStub = stub; } public StubElement getStub() { - return SerializationManager.getInstance().deserialize(new ByteArrayInputStream(myBytes)); + return myStub == null ? SerializationManager.getInstance().deserialize(new ByteArrayInputStream(myBytes)) : myStub; } public boolean equals(final Object that) { diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index e403b2cb302d..83c475c3a805 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -63,7 +63,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi int len = in.readInt(); byte[] result = new byte[len]; in.readFully(result); - return new SerializedStubTree(result); + return new SerializedStubTree(result, null); } }; @@ -118,7 +118,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi SerializationManager.getInstance().serialize(rootStub, bytes); final int key = Math.abs(FileBasedIndex.getFileId(inputData.getFile())); - result.put(key, new SerializedStubTree(bytes.toByteArray())); + result.put(key, new SerializedStubTree(bytes.toByteArray(), rootStub)); } }); From 466391f12088ace3de43969e555587c128555888 Mon Sep 17 00:00:00 2001 From: Dmitry Avdeev Date: Fri, 8 Oct 2010 12:39:37 +0400 Subject: [PATCH 2/4] do not scan too deep --- plugins/git4idea/src/git4idea/vfs/GitRootTracker.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/git4idea/src/git4idea/vfs/GitRootTracker.java b/plugins/git4idea/src/git4idea/vfs/GitRootTracker.java index 39ac3fb76830..004b369f5caa 100644 --- a/plugins/git4idea/src/git4idea/vfs/GitRootTracker.java +++ b/plugins/git4idea/src/git4idea/vfs/GitRootTracker.java @@ -265,7 +265,7 @@ public class GitRootTracker implements VcsListener { if (!hasInvalidRoots) { // check if roots have a problem for (final VirtualFile root : rootSet) { - hasInvalidRoots = hasUnmappedSubroots(root, rootSet); + hasInvalidRoots = hasUnmappedSubroots(root, rootSet, 0); if (hasInvalidRoots) { break; } @@ -318,8 +318,12 @@ public class GitRootTracker implements VcsListener { * * @param directory the content root to check * @param rootSet the mapped root set + * @param depth */ - private static boolean hasUnmappedSubroots(final VirtualFile directory, final @NotNull HashSet rootSet) { + private static boolean hasUnmappedSubroots(final VirtualFile directory, final @NotNull HashSet rootSet, int depth) { + if (depth > 3) { // three is quite enough + return false; + } VirtualFile[] children = ApplicationManager.getApplication().runReadAction(new Computable() { public VirtualFile[] compute() { return directory.isValid() ? directory.getChildren() : VirtualFile.EMPTY_ARRAY; @@ -333,7 +337,7 @@ public class GitRootTracker implements VcsListener { if (child.getName().equals(".git")) { return !rootSet.contains(child.getParent()); } - if (hasUnmappedSubroots(child, rootSet)) { + if (hasUnmappedSubroots(child, rootSet, depth + 1)) { return true; } } From 278d0173cd10a474e48b626bc684bd0514258419 Mon Sep 17 00:00:00 2001 From: Dmitry Avdeev Date: Fri, 8 Oct 2010 12:42:34 +0400 Subject: [PATCH 3/4] native file system (temporarily disabled) --- .../vfs/impl/local/LocalFileSystemImpl.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java b/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java index 10c098936fe0..6d05b65c28b1 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/impl/local/LocalFileSystemImpl.java @@ -21,8 +21,10 @@ import com.intellij.openapi.components.ApplicationComponent; import com.intellij.openapi.fileEditor.FileDocumentManager; import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.vfs.JarFileSystem; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.impl.win32.Win32LocalFileSystem; import com.intellij.openapi.vfs.newvfs.ManagingFS; import com.intellij.openapi.vfs.newvfs.NewVirtualFile; import com.intellij.openapi.vfs.newvfs.RefreshQueue; @@ -51,6 +53,8 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap private final FileWatcher myWatcher; + private final LocalFileSystemBase myNativeFileSystem; + private static class WatchRequestImpl implements WatchRequest { public final String myRootPath; @@ -117,6 +121,14 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap if (myWatcher.isOperational()) { new StoreRefreshStatusThread().start(); } + if (SystemInfo.isWindows && + Win32LocalFileSystem.isAvailable() && + Registry.is("filesystem.useNative")) { + myNativeFileSystem = null; //Win32LocalFileSystem.getWin32Instance(); + } + else { + myNativeFileSystem = null; + } } public void initComponent() { @@ -439,4 +451,28 @@ public final class LocalFileSystemImpl extends LocalFileSystemBase implements Ap } } + public boolean exists(final VirtualFile fileOrDirectory) { + if (myNativeFileSystem == null) return super.exists(fileOrDirectory); + else return myNativeFileSystem.exists(fileOrDirectory); + } + + public long getTimeStamp(final VirtualFile file) { + if (myNativeFileSystem == null) return super.getTimeStamp(file); + else return myNativeFileSystem.getTimeStamp(file); + } + + public boolean isDirectory(final VirtualFile file) { + if (myNativeFileSystem == null) return super.isDirectory(file); + else return myNativeFileSystem.isDirectory(file); + } + + public boolean isWritable(final VirtualFile file) { + if (myNativeFileSystem == null) return super.isWritable(file); + else return myNativeFileSystem.isWritable(file); + } + + public String[] list(final VirtualFile file) { + if (myNativeFileSystem == null) return super.list(file); + else return myNativeFileSystem.list(file); + } } From 10d2859f6884cac75545dd4f9db23ed48ead6ee3 Mon Sep 17 00:00:00 2001 From: Dmitry Avdeev Date: Fri, 8 Oct 2010 13:14:08 +0400 Subject: [PATCH 4/4] vfs: creating new records optimized --- .../vfs/newvfs/persistent/FSRecords.java | 54 ++++++++++++++----- .../vfs/newvfs/persistent/PersistentFS.java | 26 +++++---- 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/FSRecords.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/FSRecords.java index edd4a4dd1382..67ef64cd6a65 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/FSRecords.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/FSRecords.java @@ -28,9 +28,7 @@ import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.ArrayUtil; import com.intellij.util.containers.IntArrayList; -import com.intellij.util.io.PagedFileStorage; -import com.intellij.util.io.PersistentStringEnumerator; -import com.intellij.util.io.ResizeableMappedFile; +import com.intellij.util.io.*; import com.intellij.util.io.storage.AbstractStorage; import com.intellij.util.io.storage.HeavyProcessLatch; import com.intellij.util.io.storage.RefCountingStorage; @@ -71,6 +69,8 @@ public class FSRecords implements Disposable, Forceable { private static final int RECORD_SIZE = LENGTH_OFFSET + LENGTH_SIZE; + private final byte[] myRecordBuffer = new byte[RECORD_SIZE]; + private static final byte[] ZEROES = new byte[RECORD_SIZE]; private static final int HEADER_VERSION_OFFSET = 0; @@ -98,20 +98,20 @@ public class FSRecords implements Disposable, Forceable { } private static class DbConnection { + private static int refCount = 0; private static final Object LOCK = new Object(); private static final TObjectIntHashMap myAttributeIds = new TObjectIntHashMap(); - private static PersistentStringEnumerator myNames; + private static Storage myAttributes; private static RefCountingStorage myContents; private static ResizeableMappedFile myRecords; private static final TIntArrayList myFreeRecords = new TIntArrayList(); - private static boolean myDirty = false; + private static ScheduledFuture myFlushingFuture; private static boolean myCorrupted = false; - public static DbConnection connect() { synchronized (LOCK) { if (refCount == 0) { @@ -334,7 +334,6 @@ public class FSRecords implements Disposable, Forceable { return myDirty || myNames.isDirty() || myAttributes.isDirty() || myContents.isDirty() || myRecords.isDirty(); } - private static int getVersion() throws IOException { final int recordsVersion = myRecords.getInt(HEADER_VERSION_OFFSET); if (myAttributes.getVersion() != recordsVersion || myContents.getVersion() != recordsVersion) return -1; @@ -342,6 +341,7 @@ public class FSRecords implements Disposable, Forceable { return recordsVersion; } + public static long getTimestamp() { return myRecords.getLong(HEADER_TIMESTAMP_OFFSET); } @@ -434,8 +434,8 @@ public class FSRecords implements Disposable, Forceable { public static void addFreeRecord(final int id) { myFreeRecords.add(id); } - } + } public FSRecords() { } @@ -465,7 +465,7 @@ public class FSRecords implements Disposable, Forceable { return DbConnection.getNames(); } - public static int createRecord() { + public static int createRecord(boolean clean) { synchronized (lock) { try { DbConnection.markDirty(); @@ -475,12 +475,16 @@ public class FSRecords implements Disposable, Forceable { final int filelength = (int)getRecords().length(); LOG.assertTrue(filelength % RECORD_SIZE == 0); int newrecord = filelength / RECORD_SIZE; - DbConnection.cleanRecord(newrecord); - assert filelength + RECORD_SIZE == getRecords().length(); + if (clean) { + DbConnection.cleanRecord(newrecord); + assert filelength + RECORD_SIZE == getRecords().length(); + } return newrecord; } else { - DbConnection.cleanRecord(free); + if (clean) { + DbConnection.cleanRecord(free); + } return free; } } @@ -614,7 +618,7 @@ public class FSRecords implements Disposable, Forceable { final DataOutputStream output = writeAttribute(1, CHILDREN_ATT); int id; try { - id = createRecord(); + id = createRecord(true); output.writeInt(names.length + 1); for (int i = 0; i < names.length; i++) { output.writeInt(names[i]); @@ -783,6 +787,30 @@ public class FSRecords implements Disposable, Forceable { } } + public void saveRecord(int id, int parent, String name, long timeStamp, int flags, long length) { + if (id == parent) { + LOG.error("Cyclic parent/child relations"); + return; + } + synchronized (lock) { + try { + DbConnection.markDirty(); + incModCount(id); + Bits.putInt(myRecordBuffer, PARENT_OFFSET, parent); + Bits.putInt(myRecordBuffer, NAME_OFFSET, getNames().enumerate(name)); + Bits.putInt(myRecordBuffer, FLAGS_OFFSET, flags); + Bits.putLong(myRecordBuffer, TIMESTAMP_OFFSET, timeStamp); + Bits.putLong(myRecordBuffer, LENGTH_OFFSET, length); + + getRecords().put(getOffset(id, PARENT_OFFSET), myRecordBuffer, 0, RECORD_SIZE); + + } + catch (Throwable e) { + throw DbConnection.handleError(e); + } + } + } + public static String getName(int id) { synchronized (lock) { try { diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFS.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFS.java index bf689f3aa6da..120bdc479409 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFS.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFS.java @@ -170,7 +170,7 @@ public class PersistentFS extends ManagingFS implements ApplicationComponent { childrenIds[i] = currentIds[idx]; } else { - int childId = FSRecords.createRecord(); + int childId = FSRecords.createRecord(false); copyRecordFromDelegateFS(childId, id, new FakeVirtualFile(file, name), delegate); childrenIds[i] = childId; } @@ -247,21 +247,15 @@ public class PersistentFS extends ManagingFS implements ApplicationComponent { } String name = file.getName(); - - if (name.length() > 0 && namesEqual(delegate, name, FSRecords.getName(id))) return; // TODO: Handle root attributes change. - if (name.length() == 0) { // TODO: hack if (areChildrenLoaded(id)) return; } - myRecords.setParent(id, parentId); - myRecords.setName(id, name); - - myRecords.setTimestamp(id, delegate.getTimeStamp(file)); - myRecords.setFlags(id, (delegate.isDirectory(file) ? IS_DIRECTORY_FLAG : 0) | (delegate.isWritable(file) ? 0 : IS_READ_ONLY), true); - - myRecords.setLength(id, -1L); + boolean isDirectory = delegate.isDirectory(file); + int flags = (isDirectory ? IS_DIRECTORY_FLAG : 0) | (delegate.isWritable(file) ? 0 : IS_READ_ONLY); + myRecords.saveRecord(id, parentId, name, delegate.getTimeStamp(file), flags, isDirectory ? -1L : delegate.getLength(file)); + // // TODO!!!: More attributes? } @@ -330,7 +324,7 @@ public class PersistentFS extends ManagingFS implements ApplicationComponent { VirtualFile fake = new FakeVirtualFile(parent, childName); if (delegate.exists(fake)) { - int child = FSRecords.createRecord(); + int child = FSRecords.createRecord(false); copyRecordFromDelegateFS(child, parentId, fake, delegate); myRecords.updateList(parentId, ArrayUtil.append(children, child)); return child; @@ -645,7 +639,11 @@ public class PersistentFS extends ManagingFS implements ApplicationComponent { } if (!fs.exists(root)) return null; - copyRecordFromDelegateFS(rootId, 0, root, fs); + String name = root.getName(); + + if (name.length() == 0 || !namesEqual(fs, name, FSRecords.getName(rootId))) { + copyRecordFromDelegateFS(rootId, 0, root, fs); + } } catch (IOException e) { throw new RuntimeException(e); @@ -810,7 +808,7 @@ public class PersistentFS extends ManagingFS implements ApplicationComponent { VirtualFile fakeFile = new FakeVirtualFile(parent, name); if (delegate.exists(fakeFile)) { final int parentId = getFileId(parent); - int childId = FSRecords.createRecord(); + int childId = FSRecords.createRecord(false); copyRecordFromDelegateFS(childId, parentId, fakeFile, delegate); appendIdToParentList(parentId, childId);