[performance] avoid O(NumberOfDirectories^2) for init vfs / refresh for jars by storing compact parent -> children mapping

platform/core-api/src/com/intellij/openapi/vfs/impl/AddonlyKeylessHash.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2000-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.openapi.vfs.impl;
+
+import gnu.trove.PrimeFinder;
+
+/**
+ * IdentityHashMap that have the (immutable) keys stored in values (2 times more compact).
+ */
+final class AddonlyKeylessHash<K, V> {
+  private int size;
+  private Object[] entries;
+  private final KeyValueMapper<K, V> keyValueMapper;
+
+  public AddonlyKeylessHash(KeyValueMapper<K, V> _keyValueMapper) {
+    this(4, _keyValueMapper);
+  }
+
+  public AddonlyKeylessHash(int expectedSize, KeyValueMapper<K, V> _keyValueMapper) {
+    int i = PrimeFinder.nextPrime(5 * expectedSize / 4 + 1);
+    entries = new Object[i];
+    keyValueMapper = _keyValueMapper;
+  }
+
+  public int size() {
+    return size;
+  }
+
+  public void add(V item) {
+    if (size >= (4 * entries.length) / 5) rehash();
+    V v = doPut(entries, item);
+    if (v == null) size++;
+  }
+
+  private V doPut(Object[] a, V o) {
+    K key = keyValueMapper.key(o);
+    int index = hashIndex(a, key);
+    V obj = (V)a[index];
+    a[index] = o;
+    return obj;
+  }
+
+  private int hashIndex(Object[] a, K key) {
+    int hash = keyValueMapper.hash(key) & 0x7fffffff;
+    int index = hash % a.length;
+    V candidate = (V)a[index];
+    if (candidate == null || keyValueMapper.valueHasKey(candidate, key)) {
+      return index;
+    }
+
+    final int probe = 1 + (hash % (a.length - 2));
+
+    do {
+      index -= probe;
+      if (index < 0) index += a.length;
+
+      candidate = (V)a[index];
+    }
+    while (candidate != null && !keyValueMapper.valueHasKey(candidate, key));
+
+    return index;
+  }
+
+  private void rehash() {
+    Object[] b = new Object[PrimeFinder.nextPrime(entries.length * 2)];
+    for (int i = entries.length; --i >= 0; ) {
+      V ns = (V)entries[i];
+      if (ns != null) doPut(b, ns);
+    }
+    entries = b;
+  }
+
+  public V get(K key) {
+    return (V)entries[hashIndex(entries, key)];
+  }
+
+  public abstract static class KeyValueMapper<K, V> {
+    public abstract int hash(K k);
+
+    public abstract K key(V v);
+
+    public boolean valueHasKey(V value, K key) {
+      return key == key(value); // identity
+    }
+
+    protected boolean isIdentity() { return true; }
+  }
+}

platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java

@@ -21,8 +21,10 @@ import com.intellij.openapi.util.io.FileAttributes;
 import com.intellij.openapi.util.io.FileSystemUtil;
 import com.intellij.reference.SoftReference;
 import com.intellij.util.ArrayUtil;
+import com.intellij.util.SmartList;
 import com.intellij.util.text.ByteArrayCharSequence;
-import gnu.trove.THashSet;
+import gnu.trove.THashMap;
+import gnu.trove.TObjectObjectProcedure;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -30,8 +32,8 @@ import java.io.File;
 import java.io.IOException;
 import java.lang.ref.Reference;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 public abstract class ArchiveHandler {
   public static final long DEFAULT_LENGTH = 0L;
@@ -61,6 +63,7 @@ public abstract class ArchiveHandler {
   private final File myPath;
   private final Object myLock = new Object();
   private volatile Reference<Map<String, EntryInfo>> myEntries = new SoftReference<Map<String, EntryInfo>>(null);
+  private volatile Reference<AddonlyKeylessHash<EntryInfo, Object>> myChildrenEntries = new SoftReference<AddonlyKeylessHash<EntryInfo, Object>>(null);
   private boolean myCorrupted;
 
   protected ArchiveHandler(@NotNull String path) {
@@ -89,17 +92,84 @@ public abstract class ArchiveHandler {
     EntryInfo entry = getEntryInfo(relativePath);
     if (entry == null || !entry.isDirectory) return ArrayUtil.EMPTY_STRING_ARRAY;
 
-    Set<String> names = new THashSet<String>();
-    for (EntryInfo info : getEntriesMap().values()) {
-      if (info.parent == entry) {
-        names.add(info.shortName.toString());
+    AddonlyKeylessHash<EntryInfo, Object> result = getParentChildrenMap();
+
+    Object o = result.get(entry);
+    if (o == null) {
+      return ArrayUtil.EMPTY_STRING_ARRAY; // directories without children
+    }
+    if (o instanceof EntryInfo) {
+      return new String[] {((EntryInfo)o).shortName.toString()};
+    }
+    EntryInfo[] infos = (EntryInfo[])o;
+
+    String[] names = new String[infos.length];
+    for (int i = 0; i < infos.length; ++i) {
+      names[i] = infos[i].shortName.toString();
+    }
+    return names;
+  }
+
+  @NotNull
+  private AddonlyKeylessHash<EntryInfo, Object> getParentChildrenMap() {
+    AddonlyKeylessHash<EntryInfo, Object> map = SoftReference.dereference(myChildrenEntries);
+    if (map == null) {
+      synchronized (myLock) {
+        map = SoftReference.dereference(myChildrenEntries);
+
+        if (map == null) {
+          if (myCorrupted) {
+            map = new AddonlyKeylessHash<EntryInfo, Object>(ourKeyValueMapper);
+          }
+          else {
+            try {
+              map = createParentChildrenMap();
+            }
+            catch (Exception e) {
+              myCorrupted = true;
+              Logger.getInstance(getClass()).warn(e.getMessage() + ": " + myPath, e);
+              map = new AddonlyKeylessHash<EntryInfo, Object>(ourKeyValueMapper);
+            }
+          }
+
+          myChildrenEntries = new SoftReference<AddonlyKeylessHash<EntryInfo, Object>>(map);
+        }
       }
     }
-    return ArrayUtil.toStringArray(names);
+    return map;
+  }
+
+  private AddonlyKeylessHash<EntryInfo, Object> createParentChildrenMap() {
+    THashMap<EntryInfo, List<EntryInfo>> map = new THashMap<EntryInfo, List<EntryInfo>>();
+    for (EntryInfo info : getEntriesMap().values()) {
+      if (info.isDirectory && !map.containsKey(info)) map.put(info, new SmartList<EntryInfo>());
+      if (info.parent != null) {
+        List<EntryInfo> parentChildren = map.get(info.parent);
+        if (parentChildren == null) map.put(info.parent, parentChildren = new SmartList<EntryInfo>());
+        parentChildren.add(info);
+      }
+    }
+
+    final AddonlyKeylessHash<EntryInfo, Object> result = new AddonlyKeylessHash<EntryInfo, Object>(map.size(), ourKeyValueMapper);
+    map.forEachEntry(new TObjectObjectProcedure<EntryInfo, List<EntryInfo>>() {
+      @Override
+      public boolean execute(EntryInfo a, List<EntryInfo> b) {
+        int numberOfChildren = b.size();
+        if (numberOfChildren == 1) {
+          result.add(b.get(0));
+        }
+        else if (numberOfChildren > 1) {
+          result.add(b.toArray(new EntryInfo[numberOfChildren]));
+        }
+        return true;
+      }
+    });
+    return result;
   }
 
   public void dispose() {
     myEntries.clear();
+    myChildrenEntries.clear();
   }
 
   @Nullable
@@ -167,4 +237,17 @@ public abstract class ArchiveHandler {
 
   @NotNull
   public abstract byte[] contentsToByteArray(@NotNull String relativePath) throws IOException;
+
+  private static final AddonlyKeylessHash.KeyValueMapper<EntryInfo, Object> ourKeyValueMapper = new AddonlyKeylessHash.KeyValueMapper<EntryInfo, Object>() {
+    @Override
+    public int hash(EntryInfo info) {
+      return System.identityHashCode(info);
+    }
+
+    @Override
+    public EntryInfo key(Object o) {
+      if (o instanceof EntryInfo) return ((EntryInfo)o).parent;
+      return ((EntryInfo[])o)[0].parent;
+    }
+  };
 }