diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/ex/temp/TempFileSystem.java b/platform/platform-impl/src/com/intellij/openapi/vfs/ex/temp/TempFileSystem.java index 83b1112a2c0c..38dcbaa0402e 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/ex/temp/TempFileSystem.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/ex/temp/TempFileSystem.java @@ -162,6 +162,12 @@ public class TempFileSystem extends LocalFileSystemBase { return fsItem.list(); } + @NotNull + @Override + public String getCanonicallyCasedName(@NotNull VirtualFile file) { + return file.getName(); + } + @Override public boolean isDirectory(@NotNull final VirtualFile file) { return convert(file) instanceof FSDir; diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VfsData.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VfsData.java index 4f13fcfa6997..0429052eea32 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VfsData.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VfsData.java @@ -26,17 +26,16 @@ import com.intellij.util.containers.ConcurrentBitSet; import com.intellij.util.containers.ConcurrentIntObjectMap; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.keyFMap.KeyFMap; -import com.intellij.util.text.CaseInsensitiveStringHashingStrategy; +import com.intellij.util.text.CaseInsensitiveCharSequenceHashingStrategy; import gnu.trove.THashSet; import gnu.trove.TIntHashSet; -import gnu.trove.TObjectHashingStrategy; import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -270,8 +269,8 @@ public class VfsData { public static class DirectoryData { private static final AtomicFieldUpdater updater = AtomicFieldUpdater.forFieldOfType(DirectoryData.class, KeyFMap.class); @NotNull volatile KeyFMap myUserMap = KeyFMap.EMPTY_MAP; - @NotNull int[] myChildrenIds = ArrayUtil.EMPTY_INT_ARRAY; - private Set myAdoptedNames; + @NotNull int[] myChildrenIds = ArrayUtil.EMPTY_INT_ARRAY; // guarded by this + private Set myAdoptedNames; // guarded by this @NotNull VirtualFileSystemEntry[] getFileChildren(int fileId, VirtualDirectoryImpl parent) { @@ -287,11 +286,11 @@ public class VfsData { return updater.compareAndSet(this, oldMap, newMap); } - boolean isAdoptedName(String name) { + boolean isAdoptedName(CharSequence name) { return myAdoptedNames != null && myAdoptedNames.contains(name); } - void removeAdoptedName(String name) { + void removeAdoptedName(CharSequence name) { if (myAdoptedNames != null) { myAdoptedNames.remove(name); if (myAdoptedNames.isEmpty()) { @@ -299,16 +298,16 @@ public class VfsData { } } } - void addAdoptedName(String name, boolean caseSensitive) { + void addAdoptedName(CharSequence name, boolean caseSensitive) { if (myAdoptedNames == null) { - //noinspection unchecked - myAdoptedNames = new THashSet<>(0, caseSensitive ? TObjectHashingStrategy.CANONICAL : CaseInsensitiveStringHashingStrategy.INSTANCE); + myAdoptedNames = new THashSet<>(0, caseSensitive ? ContainerUtil.canonicalStrategy() : CaseInsensitiveCharSequenceHashingStrategy.INSTANCE); } myAdoptedNames.add(name); } - List getAdoptedNames() { - return myAdoptedNames == null ? Collections.emptyList() : ContainerUtil.newArrayList(myAdoptedNames); + @NotNull + Collection getAdoptedNames() { + return myAdoptedNames == null ? Collections.emptyList() : myAdoptedNames; } void clearAdoptedNames() { diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualDirectoryImpl.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualDirectoryImpl.java index 199a7c525223..7eef82888ee3 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualDirectoryImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualDirectoryImpl.java @@ -19,6 +19,7 @@ import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.impl.ApplicationInfoImpl; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.util.io.FileAttributes; import com.intellij.openapi.util.io.FileUtilRt; @@ -26,6 +27,7 @@ import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.InvalidVirtualFileAccessException; import com.intellij.openapi.vfs.LocalFileSystem; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileWithId; import com.intellij.openapi.vfs.impl.win32.Win32LocalFileSystem; import com.intellij.openapi.vfs.newvfs.NewVirtualFile; import com.intellij.openapi.vfs.newvfs.NewVirtualFileSystem; @@ -36,9 +38,11 @@ import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; import com.intellij.openapi.vfs.newvfs.persistent.PersistentFSImpl; import com.intellij.psi.impl.PsiCachedValue; import com.intellij.util.ArrayUtil; +import com.intellij.util.Function; import com.intellij.util.UriUtil; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.keyFMap.KeyFMap; +import gnu.trove.TIntArrayList; import gnu.trove.TIntHashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -46,15 +50,13 @@ import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; /** * @author max */ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { - private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl"); + private static final Logger LOG = Logger.getInstance(VirtualDirectoryImpl.class); private static final boolean CHECK = ApplicationManager.getApplication().isUnitTestMode(); @@ -90,8 +92,8 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { boolean doRefresh, boolean ensureCanonicalName, @NotNull NewVirtualFileSystem delegate) { - boolean ignoreCase = !delegate.isCaseSensitive(); - VirtualFileSystemEntry result = doFindChild(name, ensureCanonicalName, delegate, ignoreCase); + boolean caseSensitive = delegate.isCaseSensitive(); + VirtualFileSystemEntry result = doFindChild(name, ensureCanonicalName, delegate, caseSensitive); //noinspection UseVirtualFileEquals if (result == NULL_VIRTUAL_FILE) { @@ -105,28 +107,28 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { return result; } - private void addToAdoptedChildren(final boolean ignoreCase, @NotNull final String name) { + private void removeFromRealAndAddToAdopted(final boolean caseSensitive, @NotNull final String name) { if (myData.isAdoptedName(name)) return; //already added if (!allChildrenLoaded()) { - myData.addAdoptedName(name, getFileSystem().isCaseSensitive()); + myData.addAdoptedName(name, caseSensitive); } - int indexInReal = findIndex(myData.myChildrenIds, name, ignoreCase); + int indexInReal = findIndex(myData.myChildrenIds, name, caseSensitive); if (indexInReal >= 0) { - // there suddenly can be that we ask to add name to adopted whereas it already contains in the real part + // there suddenly can be that we ask to add name to adopted whereas it already contained in the real part // in this case we should remove it from there removeFromArray(indexInReal); } - assertConsistency(ignoreCase, name); + assertConsistency(caseSensitive, name); } @Nullable // null if there can't be a child with this name, NULL_VIRTUAL_FILE - private VirtualFileSystemEntry doFindChildInArray(@NotNull String name, boolean ignoreCase) { + private VirtualFileSystemEntry doFindChildInArray(@NotNull String name, boolean caseSensitive) { synchronized (myData) { if (myData.isAdoptedName(name)) return NULL_VIRTUAL_FILE; int[] array = myData.myChildrenIds; - int indexInReal = findIndex(array, name, ignoreCase); + int indexInReal = findIndex(array, name, caseSensitive); if (indexInReal >= 0) { return VfsData.getFileById(array[indexInReal], this); } @@ -138,7 +140,7 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { private VirtualFileSystemEntry doFindChild(@NotNull String name, boolean ensureCanonicalName, @NotNull NewVirtualFileSystem delegate, - boolean ignoreCase) { + boolean caseSensitive) { if (name.isEmpty()) { return null; } @@ -146,7 +148,7 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { throw new InvalidVirtualFileAccessException(this); } - VirtualFileSystemEntry found = doFindChildInArray(name, ignoreCase); + VirtualFileSystemEntry found = doFindChildInArray(name, caseSensitive); if (found != null) return found; if (ensureCanonicalName) { @@ -154,7 +156,7 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { if (trimmedName.indexOf('/') != -1) return null; // name must not contain slashes in the middle if (trimmedName.isEmpty()) return null; if (!trimmedName.equals(name)) { - found = doFindChildInArray(trimmedName, ignoreCase); + found = doFindChildInArray(trimmedName, caseSensitive); if (found != null) return found; name = trimmedName; } @@ -176,7 +178,7 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { if (myData.isAdoptedName(name)) return NULL_VIRTUAL_FILE; int[] array = myData.myChildrenIds; - int indexInReal = findIndex(array, name, ignoreCase); + int indexInReal = findIndex(array, name, caseSensitive); // double check if (indexInReal >= 0) { return VfsData.getFileById(array[indexInReal], this); @@ -188,13 +190,13 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { // do not extract getId outside the synchronized block since it will cause a concurrency problem. int id = ourPersistence.getId(this, name, delegate); if (id <= 0) { - myData.addAdoptedName(name, !ignoreCase); + myData.addAdoptedName(name, caseSensitive); return null; } child = createChild(FileNameCache.storeName(name), id, delegate); insertChildAt(child, indexInReal); - assertConsistency(!delegate.isCaseSensitive(), name); + assertConsistency(caseSensitive, name); } if (!child.isDirectory()) { @@ -271,8 +273,8 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { @Override @Nullable public NewVirtualFile findChildIfCached(@NotNull String name) { - final boolean ignoreCase = !getFileSystem().isCaseSensitive(); - VirtualFileSystemEntry found = doFindChildInArray(name, ignoreCase); + final boolean caseSensitive = getFileSystem().isCaseSensitive(); + VirtualFileSystemEntry found = doFindChildInArray(name, caseSensitive); //noinspection UseVirtualFileEquals return found == NULL_VIRTUAL_FILE ? null : found; } @@ -301,10 +303,10 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { throw new InvalidVirtualFileAccessException(this); } NewVirtualFileSystem delegate = getFileSystem(); - final boolean ignoreCase = !delegate.isCaseSensitive(); + final boolean caseSensitive = delegate.isCaseSensitive(); synchronized (myData) { if (allChildrenLoaded()) { - assertConsistency(ignoreCase, ""); + assertConsistency(caseSensitive, ""); return getArraySafely(); } @@ -318,10 +320,10 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { Arrays.sort(childrenIds, (o1, o2) -> { CharSequence name1 = o1.name; CharSequence name2 = o2.name; - int cmp = compareNames(name1, name2, ignoreCase); + int cmp = compareNames(name1, name2, caseSensitive); if (cmp == 0 && name1 != name2) { LOG.error(ourPersistence + " returned duplicate file names(" + name1 + "," + name2 + ")" + - " ignoreCase: " + ignoreCase + + " caseSensitive: " + caseSensitive + " SystemInfo.isFileSystemCaseSensitive: " + SystemInfo.isFileSystemCaseSensitive + " SystemInfo.OS: " + SystemInfo.OS_NAME + " " + SystemInfo.OS_VERSION + " wasChildrenLoaded: " + wasChildrenLoaded + @@ -351,7 +353,7 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { if (getId() > 0) { myData.myChildrenIds = result; if (CHECK) { - assertConsistency(ignoreCase, Arrays.asList(childrenIds)); + assertConsistency(caseSensitive, Arrays.asList(childrenIds)); } setChildrenLoaded(); } @@ -360,7 +362,7 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { } } - private void assertConsistency(boolean ignoreCase, @NotNull Object details) { + private void assertConsistency(boolean caseSensitive, @NotNull Object details) { if (!CHECK || ApplicationInfoImpl.isInStressTest()) return; int[] childrenIds = myData.myChildrenIds; for (int i = 1; i < childrenIds.length; i++) { @@ -368,7 +370,7 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { int prev = childrenIds[i - 1]; CharSequence name = VfsData.getNameByFileId(id); CharSequence prevName = VfsData.getNameByFileId(prev); - int cmp = compareNames(name, prevName, ignoreCase); + int cmp = compareNames(name, prevName, caseSensitive); if (cmp <= 0) { error(verboseToString(VfsData.getFileById(prev, this)) + " is wrongly placed before " + @@ -420,18 +422,56 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { throw new IOException("Cannot get content of directory: " + this); } + // optimisation: works faster than added.forEach(this::addChild) + public void addChildren(@NotNull List> added) { + if (added.size()<=3) { + //noinspection ForLoopReplaceableByForEach + for (int i = 0; i < added.size(); i++) { + VirtualFile file = added.get(i).getFirst(); + addChild((VirtualFileSystemEntry)file); + } + return; + } + + // merge sorted added and existing lists just like in merge sort + final boolean caseSensitive = getFileSystem().isCaseSensitive(); + Comparator> pairComparator = (p1, p2) -> compareNames(p1.getSecond(), p2.getSecond(), caseSensitive); + added.sort(pairComparator); + TIntArrayList mergedIds = new TIntArrayList(myData.myChildrenIds.length + added.size()); + synchronized (myData) { + ContainerUtil.processSortedListsInOrder(added, new AbstractList>() { + @Override + public Pair get(int index) { + VirtualFileSystemEntry file = VfsData.getFileById(myData.myChildrenIds[index], VirtualDirectoryImpl.this); + return Pair.create(file, file.getNameSequence()); + } + + @Override + public int size() { + return myData.myChildrenIds.length; + } + }, pairComparator, true, pair -> { + mergedIds.add(((VirtualFileWithId)pair.getFirst()).getId()); + return true; + }); + myData.myChildrenIds = mergedIds.toNativeArray(); + ((PersistentFSImpl)PersistentFS.getInstance()).incStructuralModificationCount(); + assertConsistency(caseSensitive, added); + } + } + public void addChild(@NotNull VirtualFileSystemEntry child) { final String childName = child.getName(); - final boolean ignoreCase = !getFileSystem().isCaseSensitive(); + final boolean caseSensitive = getFileSystem().isCaseSensitive(); synchronized (myData) { - int indexInReal = findIndex(myData.myChildrenIds, childName, ignoreCase); + int indexInReal = findIndex(myData.myChildrenIds, childName, caseSensitive); myData.removeAdoptedName(childName); if (indexInReal < 0) { insertChildAt(child, indexInReal); } // else already stored - assertConsistency(ignoreCase, child); + assertConsistency(caseSensitive, child); } } @@ -448,14 +488,79 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { } public void removeChild(@NotNull VirtualFile file) { - boolean ignoreCase = !getFileSystem().isCaseSensitive(); + boolean caseSensitive = getFileSystem().isCaseSensitive(); String name = file.getName(); synchronized (myData) { - addToAdoptedChildren(ignoreCase, name); - assertConsistency(ignoreCase, file); + removeFromRealAndAddToAdopted(caseSensitive, name); + assertConsistency(caseSensitive, file); } } + // faster than names.forEach(this::removeChild) + public void removeChildren(@NotNull List names) { + boolean caseSensitive = getFileSystem().isCaseSensitive(); + synchronized (myData) { + Comparator comparator = (n1, n2) -> compareNames(n1, n2, caseSensitive); + names.sort(comparator); + + boolean allChildrenLoaded = allChildrenLoaded(); + if (!allChildrenLoaded) { + for (CharSequence name : names) { + myData.addAdoptedName(name, caseSensitive); + } + } + + // remove from array by merging two sorted lists + int[] newIds = new int[myData.myChildrenIds.length]; + int length = processSortedListsAndRemoveFirstFromTheSecond(names, myData.myChildrenIds, comparator, newIds); + if (length != newIds.length) { + newIds = length == 0 ? ArrayUtil.EMPTY_INT_ARRAY : Arrays.copyOf(newIds, length); + } + myData.myChildrenIds = newIds; + ((PersistentFSImpl)PersistentFS.getInstance()).incStructuralModificationCount(); + + assertConsistency(caseSensitive, names); + } + } + + // returns output length + private static int processSortedListsAndRemoveFirstFromTheSecond(@NotNull List namesToRemove, + @NotNull int[] ids, + @NotNull Comparator comparator, + @NotNull int[] output) { + int index1 = 0; + int index2 = 0; + int o = 0; + while (index2 < ids.length) { + int e; + if (index1 >= namesToRemove.size()) { + e = ids[index2++]; + } + else { + CharSequence nameToRemove = namesToRemove.get(index1); + int id = ids[index2]; + CharSequence element2 = VfsData.getNameByFileId(id); + int c = comparator.compare(nameToRemove, element2); + if (c == 0) { + // remove + index2++; + continue; + } + if (c < 0) { + // next name to remove + index1++; + continue; + } + e = id; + index2++; + } + output[o++] = e; + } + + return o; + } + + private void removeFromArray(int index) { myData.myChildrenIds = ArrayUtil.remove(myData.myChildrenIds, index); ((PersistentFSImpl)PersistentFS.getInstance()).incStructuralModificationCount(); @@ -472,18 +577,20 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { @NotNull public List getSuspiciousNames() { synchronized (myData) { - return myData.getAdoptedNames(); + Collection names = myData.getAdoptedNames(); + //noinspection unchecked + return ContainerUtil.map(names, Function.TO_STRING); } } @SuppressWarnings("Duplicates") - private static int findIndex(final int[] array, @NotNull CharSequence name, boolean ignoreCase) { + private static int findIndex(final int[] array, @NotNull CharSequence name, boolean caseSensitive) { int low = 0; int high = array.length - 1; while (low <= high) { int mid = low + high >>> 1; - int cmp = -compareNames(VfsData.getNameByFileId(array[mid]), name, ignoreCase); + int cmp = compareNames(name, VfsData.getNameByFileId(array[mid]), caseSensitive); if (cmp > 0) low = mid + 1; else if (cmp < 0) high = mid - 1; else return mid; @@ -492,12 +599,12 @@ public class VirtualDirectoryImpl extends VirtualFileSystemEntry { return -(low + 1); } - private static int compareNames(@NotNull CharSequence name1, @NotNull CharSequence name2, boolean ignoreCase) { + private static int compareNames(@NotNull CharSequence name1, @NotNull CharSequence name2, boolean caseSensitive) { int d = name1.length() - name2.length(); if (d != 0) return d; for (int i = 0; i < name1.length(); i++) { // com.intellij.openapi.util.text.StringUtil.compare(String,String,boolean) inconsistent - d = StringUtil.compare(name1.charAt(i), name2.charAt(i), ignoreCase); + d = StringUtil.compare(name1.charAt(i), name2.charAt(i), !caseSensitive); if (d != 0) return d; } return 0; diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualFileImpl.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualFileImpl.java index f99494d6c2f7..def677039346 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualFileImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/impl/VirtualFileImpl.java @@ -53,7 +53,7 @@ import java.util.Collections; public class VirtualFileImpl extends VirtualFileSystemEntry { - VirtualFileImpl(int id, VfsData.Segment segment, VirtualDirectoryImpl parent) { + public VirtualFileImpl(int id, VfsData.Segment segment, VirtualDirectoryImpl parent) { super(id, segment, parent); } diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java index 478a6e658aec..72eddda997c1 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/newvfs/persistent/PersistentFSImpl.java @@ -23,6 +23,7 @@ import com.intellij.openapi.components.ApplicationComponent; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Comparing; import com.intellij.openapi.util.LowMemoryWatcher; +import com.intellij.openapi.util.Pair; import com.intellij.openapi.util.ShutDownTracker; import com.intellij.openapi.util.io.*; import com.intellij.openapi.util.text.StringUtil; @@ -391,15 +392,9 @@ public class PersistentFSImpl extends PersistentFS implements ApplicationCompone @Override public long getLength(@NotNull VirtualFile file) { - long len; - if (mustReloadContent(file)) { - len = reloadLengthFromDelegate(file, getDelegate(file)); - } - else { - len = getLastRecordedLength(file); - } - - return len; + return mustReloadContent(file) ? + reloadLengthFromDelegate(file, getDelegate(file)) : + getLastRecordedLength(file); } @Override @@ -688,7 +683,7 @@ public class PersistentFSImpl extends PersistentFS implements ApplicationCompone @NotNull private static List validateEvents(@NotNull List events) { - final List deletionEvents = ContainerUtil.newArrayList(); + final List deletionEvents = new ArrayList<>(); for (int i = 0, size = events.size(); i < size; i++) { final VFileEvent event = events.get(i); if (event instanceof VFileDeleteEvent && event.isValid()) { @@ -776,14 +771,17 @@ public class PersistentFSImpl extends PersistentFS implements ApplicationCompone private void applyChildrenChangeEvents(@NotNull VirtualFile parent, @NotNull List events) { final NewVirtualFileSystem delegate = getDelegate(parent); - TIntArrayList childrenIdsUpdated = new TIntArrayList(); final int parentId = getFileId(parent); assert parentId != 0; - TIntHashSet parentChildrenIds = new TIntHashSet(FSRecords.list(parentId)); + int[] oldIds = FSRecords.list(parentId); + TIntHashSet parentChildrenIds = new TIntHashSet(Math.max(events.size(), oldIds.length)); + parentChildrenIds.addAll(oldIds); boolean hasRemovedChildren = false; - List childrenToBeUpdated = new SmartList<>(); + List> childrenAdded = new SmartList<>(); + List childrenDeleted = new SmartList<>(); + List childrenNamesDeleted = new SmartList<>(); for (VFileEvent event : events) { if (event instanceof VFileCreateEvent) { String name = ((VFileCreateEvent)event).getChildName(); @@ -795,8 +793,7 @@ public class PersistentFSImpl extends PersistentFS implements ApplicationCompone assert parent instanceof VirtualDirectoryImpl : parent; final VirtualDirectoryImpl dir = (VirtualDirectoryImpl)parent; VirtualFileSystemEntry child = dir.createChild(name, childId, dir.getFileSystem()); - childrenToBeUpdated.add(child); - childrenIdsUpdated.add(childId); + childrenAdded.add(Pair.create(child, name)); parentChildrenIds.add(childId); } } @@ -810,8 +807,8 @@ public class PersistentFSImpl extends PersistentFS implements ApplicationCompone hasRemovedChildren = true; int id = getFileId(file); - childrenToBeUpdated.add(file); - childrenIdsUpdated.add(-id); + childrenDeleted.add(file); + childrenNamesDeleted.add(file.getNameSequence()); parentChildrenIds.remove(id); } } @@ -821,18 +818,15 @@ public class PersistentFSImpl extends PersistentFS implements ApplicationCompone if (hasRemovedChildren) clearIdCache(); VirtualDirectoryImpl parentImpl = (VirtualDirectoryImpl)parent; - for (int i = 0, len = childrenIdsUpdated.size(); i < len; ++i) { - final int childId = childrenIdsUpdated.get(i); - final VirtualFile childFile = childrenToBeUpdated.get(i); - - if (childId > 0) { - parentImpl.addChild((VirtualFileSystemEntry)childFile); - } - else { - FSRecords.deleteRecordRecursively(-childId); - parentImpl.removeChild(childFile); + if (!childrenDeleted.isEmpty()) { + for (final VirtualFile childFile : childrenDeleted) { + FSRecords.deleteRecordRecursively(getFileId(childFile)); invalidateSubtree(childFile); } + parentImpl.removeChildren(childrenNamesDeleted); + } + if (!childrenAdded.isEmpty()) { + parentImpl.addChildren(childrenAdded); } } diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/VfsUtilPerformanceTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/VfsUtilPerformanceTest.java index 71a140a6a28f..7ff17dc68b0d 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/VfsUtilPerformanceTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/VfsUtilPerformanceTest.java @@ -17,35 +17,52 @@ package com.intellij.openapi.vfs; import com.intellij.concurrency.JobLauncher; import com.intellij.concurrency.JobSchedulerImpl; +import com.intellij.openapi.application.impl.ApplicationInfoImpl; import com.intellij.openapi.command.WriteCommandAction; import com.intellij.openapi.progress.ProgressManager; -import com.intellij.openapi.util.Disposer; -import com.intellij.openapi.util.Ref; +import com.intellij.openapi.util.*; +import com.intellij.openapi.util.io.FileAttributes; import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.io.IoTestUtil; import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.ex.temp.TempFileSystem; import com.intellij.openapi.vfs.newvfs.ManagingFS; import com.intellij.openapi.vfs.newvfs.NewVirtualFile; -import com.intellij.testFramework.EdtTestUtil; -import com.intellij.testFramework.JITSensitive; -import com.intellij.testFramework.PlatformTestUtil; -import com.intellij.testFramework.SkipSlowTestLocally; +import com.intellij.openapi.vfs.newvfs.events.VFileCreateEvent; +import com.intellij.openapi.vfs.newvfs.events.VFileDeleteEvent; +import com.intellij.openapi.vfs.newvfs.events.VFileEvent; +import com.intellij.openapi.vfs.newvfs.impl.VfsData; +import com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl; +import com.intellij.openapi.vfs.newvfs.impl.VirtualFileImpl; +import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry; +import com.intellij.openapi.vfs.newvfs.persistent.FSRecords; +import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; +import com.intellij.testFramework.*; import com.intellij.testFramework.fixtures.BareTestFixtureTestCase; import com.intellij.testFramework.fixtures.impl.LightTempDirTestFixtureImpl; import com.intellij.testFramework.rules.TempDirectory; import com.intellij.util.ExceptionUtil; import com.intellij.util.ThrowableRunnable; import com.intellij.util.TimeoutUtil; +import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.FileBasedIndexImpl; +import com.intellij.util.ref.GCUtil; import com.intellij.util.ui.UIUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.Rule; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.*; @@ -76,7 +93,7 @@ public class VfsUtilPerformanceTest extends BareTestFixtureTestCase { UIUtil.pump(); // wait for all event handlers to calm down System.out.println("Start searching..."); - PlatformTestUtil.startPerformanceTest("find child is slow", 1000, () -> { + PlatformTestUtil.startPerformanceTest("finding child", 1000, () -> { for (int i = 0; i < 1000000; i++) { VirtualFile child = vDir.findChild("5111.txt"); assertEquals(theChild, child); @@ -103,7 +120,7 @@ public class VfsUtilPerformanceTest extends BareTestFixtureTestCase { String path = jar.getPath() + "!/"; NewVirtualFile root = ManagingFS.getInstance().findRoot(path, fs); PlatformTestUtil.startPerformanceTest( - "find root is slow", 5000, + "finding root", 5000, () -> JobLauncher.getInstance().invokeConcurrentlyUnderProgress( Collections.nCopies(500, null), null, false, false, o -> { @@ -154,13 +171,13 @@ public class VfsUtilPerformanceTest extends BareTestFixtureTestCase { } }; int time = 1200; - PlatformTestUtil.startPerformanceTest("getParent is slow before movement", time, checkPerformance).useLegacyScaling().assertTiming(); + PlatformTestUtil.startPerformanceTest("getParent before movement", time, checkPerformance).useLegacyScaling().assertTiming(); VirtualFile dir1 = vDir.createChildDirectory(this, "dir1"); VirtualFile dir2 = vDir.createChildDirectory(this, "dir2"); for (int i = 0; i < 13; i++) { /*13 is max length with THashMap capacity of 17, we get plenty collisions then*/ dir1.createChildData(this, "a" + i + ".txt").move(this, dir2); } - PlatformTestUtil.startPerformanceTest("getParent is slow after movement", time, checkPerformance).useLegacyScaling().assertTiming(); + PlatformTestUtil.startPerformanceTest("getParent after movement", time, checkPerformance).useLegacyScaling().assertTiming(); } }.execute(); } @@ -185,7 +202,7 @@ public class VfsUtilPerformanceTest extends BareTestFixtureTestCase { "fff.txt"; VirtualFile file = fixture.findOrCreateDir(path); - PlatformTestUtil.startPerformanceTest("VF.getPath() performance failed", 4000, () -> { + PlatformTestUtil.startPerformanceTest("VF.getPath()", 4000, () -> { for (int i = 0; i < 1000000; ++i) { file.getPath(); } @@ -268,4 +285,170 @@ public class VfsUtilPerformanceTest extends BareTestFixtureTestCase { IoTestUtil.assertTimestampsEqual(timestamp[i], child.getTimeStamp()); } } -} \ No newline at end of file + + @Test + public void addingManyChildrenToTheSameDirectoryMustNotBeQuadratic() throws IOException { + int N = 1_000_000; + // measures create N children inside one directory. + // to avoid slow local file system try to use MyFakeVirtualFile which doesn't actually query disk + // then call PersistentFS.getInstance().processEvents(events); directly instead of agonizingly slow refresh. + ApplicationInfoImpl.setInStressTest(true); //wtf wrong with you fixtures? + Disposer.register(getTestRootDisposable(), ()-> ApplicationInfoImpl.setInStressTest(false)); + List toDelete = new ArrayList<>(); + try { + UIUtil.invokeLaterIfNeeded(()->{ + PlatformTestUtil.startPerformanceTest("adding many children", 15000, () -> { + VirtualFile validVTemp = new MyFakeDirectory("vtemp"); + toDelete.add(validVTemp); + List events = IntStream.range(0, N) + .mapToObj(i -> new VFileCreateEvent(this, validVTemp, i + ".txt", false, false)) + .collect(Collectors.toList()); + + WriteCommandAction.runWriteCommandAction(null, () -> { + PersistentFS.getInstance().processEvents(events); + }); + + assertEquals(N, validVTemp.getChildren().length); + }).assertTiming(); + }); + } + finally { + toDelete.forEach(VfsTestUtil::deleteFile); + } + } + + @Test + public void deleteManyChildrenFromTheSameDirectoryMustNotBeQuadratic() throws IOException { + int N = 1_000_000; + // measures delete N children inside one directory. + // to avoid slow local file system try to use MyFakeVirtualFile which doesn't actually query disk + // then call PersistentFS.getInstance().processEvents(events); directly instead of agonizingly slow refresh. + ApplicationInfoImpl.setInStressTest(true); //wtf wrong with you fixtures? + //TranslatingCompilerFilesMonitor.getInstance().disable(getTestRootDisposable()); + Disposer.register(getTestRootDisposable(), ()-> ApplicationInfoImpl.setInStressTest(false)); + List toDelete = new ArrayList<>(); + try { + UIUtil.invokeLaterIfNeeded(()->{ + final VirtualDirectoryImpl[] validVTemp = new VirtualDirectoryImpl[1]; + List deleteEvents = new ArrayList<>(); + PlatformTestUtil.startPerformanceTest("deleting many children", 30000, () -> { + WriteCommandAction.runWriteCommandAction(null, () -> { + PersistentFS.getInstance().processEvents(deleteEvents); + }); + + assertEquals(0, validVTemp[0].getChildren().length); + }).setup(()-> { + // prepare fake dir with N fake children + ((FileBasedIndexImpl) FileBasedIndex.getInstance()).cleanupForNextTest(); + GCUtil.tryForceGC(); + + validVTemp[0] = new MyFakeDirectory("vtemp"); + validVTemp[0].getChildren(); + toDelete.add(validVTemp[0]); + + List createEvents = IntStream.range(0, N) + .mapToObj(i -> new VFileCreateEvent(this, validVTemp[0], i + ".txt", false, false)) + .collect(Collectors.toList()); + + WriteCommandAction.runWriteCommandAction(null, () -> { + PersistentFS.getInstance().processEvents(createEvents); + }); + assertEquals(N, validVTemp[0].getChildren().length); + + deleteEvents.clear(); + deleteEvents.addAll(Arrays.stream(validVTemp[0].getChildren()) + .map(v -> new VFileDeleteEvent(this, new MyFakeFile(v.getName(), validVTemp[0]), false)) + .collect(Collectors.toList())); + + }).assertTiming(); + }); + } + finally { + toDelete.forEach(VfsTestUtil::deleteFile); + } + } + + private static class MyFakeFile extends VirtualFileImpl { + private static final VfsData.Segment SEGMENT = new VfsData.Segment(); + private final String name; + + MyFakeFile(String name, VirtualDirectoryImpl parent) { + super(FSRecords.createRecord(), SEGMENT, parent); + this.name = name; + } + + @NotNull + @Override + public CharSequence getNameSequence() { + return name; + } + + @NotNull + @Override + public String getUrl() { + return getName(); + } + + @NotNull + @Override + public String getPath() { + return getName(); + } + + UserDataHolder data; + @Override + public void putUserData(@NotNull Key key, @Nullable T value) { + if (data == null) data = new UserDataHolderBase(); + data.putUserData(key, value); + } + + @Override + public T getUserData(@NotNull Key key) { + try { + return data.getUserData(key); + } + finally { + data.putUserData(key, null); + if (((UserDataHolderBase)data).isUserDataEmpty()) data = null; + } + } + } + + private static class MyFakeDirectory extends VirtualDirectoryImpl { + private final String name; + + MyFakeDirectory(String name) { + super(FSRecords.createRecord(), new VfsData.Segment(), new VfsData.DirectoryData(), null, new TempFileSystem(){ + @Override + public FileAttributes getAttributes(@NotNull VirtualFile file) { + return new FileAttributes(false, false, false, false, 0, 1, true); + } + }); + this.name = name; + } + + @Nullable + @Override + public VirtualFileSystemEntry findChild(@NotNull String name) { + return null; // hack for VCreateEvent.isValid() + } + + @NotNull + @Override + public String getPath() { + return getName(); + } + + @NotNull + @Override + public CharSequence getNameSequence() { + return name; + } + + @NotNull + @Override + public String getUrl() { + return getPath(); + } + } +} diff --git a/platform/util/src/com/intellij/util/text/CaseInsensitiveCharSequenceHashingStrategy.java b/platform/util/src/com/intellij/util/text/CaseInsensitiveCharSequenceHashingStrategy.java new file mode 100644 index 000000000000..14ac99510ac0 --- /dev/null +++ b/platform/util/src/com/intellij/util/text/CaseInsensitiveCharSequenceHashingStrategy.java @@ -0,0 +1,37 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intellij.util.text; + +import com.intellij.openapi.util.text.StringUtil; +import gnu.trove.TObjectHashingStrategy; + +/** + * @author max + */ +public class CaseInsensitiveCharSequenceHashingStrategy implements TObjectHashingStrategy { + public static final CaseInsensitiveCharSequenceHashingStrategy INSTANCE = new CaseInsensitiveCharSequenceHashingStrategy(); + + @Override + public int computeHashCode(final CharSequence s) { + return StringUtil.stringHashCodeInsensitive(s); + } + + @Override + public boolean equals(final CharSequence s1, final CharSequence s2) { + return StringUtil.equalsIgnoreCase(s1, s2); + } +}