ZipHandler memory optimisation: do not store relative paths to entries, compress entry short names

platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java

@@ -38,12 +38,13 @@ public abstract class ArchiveHandler {
 
   protected static class EntryInfo {
     public final EntryInfo parent;
-    public final String shortName;
+    @NotNull
+    public final CharSequence shortName;
     public final boolean isDirectory;
     public final long length;
     public final long timestamp;
 
-    public EntryInfo(EntryInfo parent, @NotNull String shortName, boolean isDirectory, long length, long timestamp) {
+    public EntryInfo(EntryInfo parent, @NotNull CharSequence shortName, boolean isDirectory, long length, long timestamp) {
       this.parent = parent;
       this.shortName = shortName;
       this.isDirectory = isDirectory;
@@ -52,18 +53,19 @@ public abstract class ArchiveHandler {
     }
   }
 
-  private final String myPath;
+  @NotNull
+  private final File myPath;
   private final Object myLock = new Object();
   private volatile Reference<Map<String, EntryInfo>> myEntries = new SoftReference<Map<String, EntryInfo>>(null);
-  private boolean myCorrupted = false;
+  private boolean myCorrupted;
 
   protected ArchiveHandler(@NotNull String path) {
-    myPath = path;
+    myPath = new File(path);
   }
 
   @NotNull
   public File getFile() {
-    return new File(myPath);
+    return myPath;
   }
 
   @Nullable
@@ -86,7 +88,7 @@ public abstract class ArchiveHandler {
     Set<String> names = new HashSet<String>();
     for (EntryInfo info : getEntriesMap().values()) {
       if (info.parent == entry) {
-        names.add(info.shortName);
+        names.add(info.shortName.toString());
       }
     }
     return ArrayUtil.toStringArray(names);
@@ -114,7 +116,7 @@ public abstract class ArchiveHandler {
           }
           else {
             try {
-              map = Collections.unmodifiableMap(createEntriesMap());
+              map = createEntriesMap();
             }
             catch (Exception e) {
               myCorrupted = true;

platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipEntryMap.java

@@ -0,0 +1,219 @@
+/*
+ * Copyright 2000-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.openapi.vfs.impl;
+
+import com.intellij.openapi.util.Condition;
+import com.intellij.util.Function;
+import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.text.CharArrayUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.*;
+
+/**
+ * Map of relativePath => ArchiveHandler.EntryInfo optimised for memory:
+ * - it does not store keys (may be recovered from the ArchiveHandler.EntryInfo)
+ */
+class ZipEntryMap extends AbstractMap<String, ArchiveHandler.EntryInfo> {
+  private static final ArchiveHandler.EntryInfo REMOVED = new ArchiveHandler.EntryInfo(null, "", false, 0, 0);
+
+  private ArchiveHandler.EntryInfo[] entries;
+  private int size;
+
+  ZipEntryMap() {
+    clear();
+  }
+
+  @Override
+  public ArchiveHandler.EntryInfo get(@NotNull Object key) {
+    String relativePath = (String)key;
+    int index = index(relativePath, entries);
+    ArchiveHandler.EntryInfo entry;
+    int i = index;
+    while (true) {
+      entry = entries[i];
+      if (entry == null || isTheOne(entry, relativePath)) break;
+      if (++i == entries.length) {
+        i = 0;
+      }
+      if (i == index) {
+        entry = null;
+        break;
+      }
+    }
+    return entry;
+  }
+
+  private static int index(@NotNull String relativePath, @NotNull ArchiveHandler.EntryInfo[] entries) {
+    return (relativePath.hashCode() & 0x7fffffff) % entries.length;
+  }
+
+  @Override
+  public ArchiveHandler.EntryInfo put(String relativePath, ArchiveHandler.EntryInfo value) {
+    if (size >= 0.8 * entries.length) {
+      rehash();
+    }
+
+    ArchiveHandler.EntryInfo old = put(relativePath, value, entries);
+    if (old == null){
+      size++;
+    }
+    return old;
+  }
+
+  @Nullable
+  private static ArchiveHandler.EntryInfo put(@NotNull String relativePath,
+                                              @NotNull ArchiveHandler.EntryInfo value,
+                                              @NotNull ArchiveHandler.EntryInfo[] entries) {
+    int index = index(relativePath, entries);
+    ArchiveHandler.EntryInfo entry;
+    int i = index;
+    while (true) {
+      entry = entries[i];
+      if (entry == null || entry == REMOVED || isTheOne(entry, relativePath)) {
+        entries[i] = value;
+        break;
+      }
+      if (++i == entries.length) {
+        i = 0;
+      }
+    }
+    return entry == REMOVED ? null : entry;
+  }
+
+  private static boolean isTheOne(@NotNull ArchiveHandler.EntryInfo entry, @NotNull String relativePath) {
+    int endIndex = relativePath.length();
+    for (ArchiveHandler.EntryInfo e = entry; e != null; e = e.parent) {
+      CharSequence shortName = e.shortName;
+      if (!CharArrayUtil.regionMatches(relativePath,endIndex-shortName.length(), relativePath.length(), shortName)) {
+        return false;
+      }
+
+      endIndex -= shortName.length();
+      if (e.parent != null && e.parent.shortName.length() != 0 && endIndex != 0) {
+        // match "/"
+        if (relativePath.charAt(endIndex-1) == '/') {
+          endIndex -= 1;
+        }
+        else {
+          return false;
+        }
+      }
+
+    }
+    return endIndex==0;
+  }
+
+  @NotNull
+  private ArchiveHandler.EntryInfo[] rehash() {
+    ArchiveHandler.EntryInfo[] newEntries = new ArchiveHandler.EntryInfo[entries.length < 1000 ? entries.length  * 2 : entries.length * 3/2];
+    for (ArchiveHandler.EntryInfo entry : entries) {
+      if (entry != null) {
+        put(getRelativePath(entry), entry, newEntries);
+      }
+    }
+    entries = newEntries;
+    return newEntries;
+  }
+
+  @NotNull
+  private static String getRelativePath(@NotNull ArchiveHandler.EntryInfo entry) {
+    if (entry.parent == null) {
+      return entry.shortName.toString();
+    }
+    String parentPath = getRelativePath(entry.parent);
+    return (parentPath.isEmpty() ? "" : parentPath + "/") + entry.shortName;
+  }
+
+  @Override
+  public ArchiveHandler.EntryInfo remove(@NotNull Object key) {
+    ArchiveHandler.EntryInfo old = put((String)key, REMOVED);
+    if (old != null) {
+      size--;
+    }
+    return old;
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public void clear() {
+    size = 0;
+    entries = new ArchiveHandler.EntryInfo[10];
+  }
+
+  private EntrySet entrySet;
+  @NotNull
+  @Override
+  public EntrySet entrySet() {
+    EntrySet es;
+    return (es = entrySet) == null ? (entrySet = new EntrySet()) : es;
+  }
+
+  private final class EntrySet extends AbstractSet<Entry<String, ArchiveHandler.EntryInfo>> {
+    @Override
+    public final int size() {
+      return ZipEntryMap.this.size();
+    }
+
+    @Override
+    public final void clear() {
+      ZipEntryMap.this.clear();
+    }
+
+    @Override
+    public final Iterator<Entry<String, ArchiveHandler.EntryInfo>> iterator() {
+      return ContainerUtil.mapIterator(ContainerUtil.iterate(entries, Condition.NOT_NULL).iterator(),
+                                       new Function<ArchiveHandler.EntryInfo, Entry<String, ArchiveHandler.EntryInfo>>() {
+                                         @Override
+                                         public Entry<String, ArchiveHandler.EntryInfo> fun(ArchiveHandler.EntryInfo entry) {
+                                           return new SimpleEntry<String, ArchiveHandler.EntryInfo>(getRelativePath(entry), entry);
+                                         }
+                                       });
+    }
+
+    @Override
+    public final boolean contains(Object o) {
+      if (!(o instanceof Map.Entry)) {
+        return false;
+      }
+      Map.Entry<?, ?> e = (Map.Entry<?, ?>)o;
+      String key = (String)e.getKey();
+      ArchiveHandler.EntryInfo value = (ArchiveHandler.EntryInfo)e.getValue();
+      return value.equals(get(key));
+    }
+
+    @Override
+    public final boolean remove(Object o) {
+      if (o instanceof Map.Entry) {
+        Map.Entry<?, ?> e = (Map.Entry<?, ?>)o;
+        Object key = e.getKey();
+        return contains(o) && ZipEntryMap.this.remove(key) != null;
+      }
+      return false;
+    }
+  }
+
+  @NotNull
+  @Override
+  public Collection<ArchiveHandler.EntryInfo> values() {
+    return ContainerUtil.mapNotNull(entries, Function.ID);
+  }
+}

platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java

@@ -21,8 +21,9 @@ import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.ArrayUtil;
 import com.intellij.util.io.FileAccessorCache;
-import gnu.trove.THashMap;
+import com.intellij.util.text.ByteArrayCharSequence;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.Closeable;
 import java.io.File;
@@ -96,7 +97,7 @@ public class ZipHandler extends ArchiveHandler {
   @NotNull
   @Override
   protected Map<String, EntryInfo> createEntriesMap() throws IOException {
-    Map<String, EntryInfo> map = new THashMap<String, EntryInfo>();
+    Map<String, EntryInfo> map = new ZipEntryMap();
     map.put("", createRootEntry());
 
     FileAccessorCache.Handle<ZipFile> zipRef = ourZipFileFileAccessorCache.get(this);
@@ -126,7 +127,7 @@ public class ZipHandler extends ArchiveHandler {
   }
 
   @NotNull
-  private EntryInfo getOrCreate(ZipEntry entry, Map<String, EntryInfo> map, ZipFile zip) {
+  private EntryInfo getOrCreate(@NotNull ZipEntry entry, @NotNull Map<String, EntryInfo> map, @NotNull ZipFile zip) {
     boolean isDirectory = entry.isDirectory();
     String entryName = entry.getName();
     if (StringUtil.endsWithChar(entryName, '/')) {
@@ -142,13 +143,26 @@ public class ZipHandler extends ArchiveHandler {
     if (".".equals(path.second)) {
       return parentInfo;
     }
-    info = new EntryInfo(parentInfo, path.second, isDirectory, entry.getSize(), entry.getTime());
+    info = store(map, parentInfo, path.second, isDirectory, entry.getSize(), entry.getTime(), entryName);
+    return info;
+  }
+
+  @NotNull
+  private static EntryInfo store(@NotNull Map<String, EntryInfo> map,
+                                 @Nullable EntryInfo parentInfo,
+                                 @NotNull CharSequence shortName,
+                                 boolean isDirectory,
+                                 long size,
+                                 long time,
+                                 @NotNull String entryName) {
+    CharSequence sequence = shortName instanceof ByteArrayCharSequence ? shortName : ByteArrayCharSequence.convertToBytesIfAsciiString(shortName);
+    EntryInfo info = new EntryInfo(parentInfo, sequence, isDirectory, size, time);
     map.put(entryName, info);
     return info;
   }
 
   @NotNull
-  private EntryInfo getOrCreate(String entryName, Map<String, EntryInfo> map, ZipFile zip) {
+  private EntryInfo getOrCreate(@NotNull String entryName, Map<String, EntryInfo> map, @NotNull ZipFile zip) {
     EntryInfo info = map.get(entryName);
 
     if (info == null) {
@@ -159,14 +173,12 @@ public class ZipHandler extends ArchiveHandler {
 
       Pair<String, String> path = splitPath(entryName);
       EntryInfo parentInfo = getOrCreate(path.first, map, zip);
-      info = new EntryInfo(parentInfo, path.second, true, DEFAULT_LENGTH, DEFAULT_TIMESTAMP);
-      map.put(entryName, info);
+      info = store(map, parentInfo, path.second, true, DEFAULT_LENGTH, DEFAULT_TIMESTAMP, entryName);
     }
 
     if (!info.isDirectory) {
       Logger.getInstance(getClass()).info(zip.getName() + ": " + entryName + " should be a directory");
-      info = new EntryInfo(info.parent, info.shortName, true, info.length, info.timestamp);
-      map.put(entryName, info);
+      info = store(map, info.parent, info.shortName, true, info.length, info.timestamp, entryName);
     }
 
     return info;

platform/core-impl/src/com/intellij/openapi/vfs/impl/jar/CoreJarHandler.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2014 JetBrains s.r.o.
+ * Copyright 2000-2015 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -52,7 +52,7 @@ public class CoreJarHandler extends ZipHandler {
     if (file == null) {
       FileAttributes attributes = new FileAttributes(info.isDirectory, false, false, false, info.length, info.timestamp, false);
       EntryInfo parent = info.parent;
-      file = new CoreJarVirtualFile(this, info.shortName, attributes, parent != null ? getOrCreateFile(parent, entries) : null);
+      file = new CoreJarVirtualFile(this, info.shortName.toString(), attributes, parent != null ? getOrCreateFile(parent, entries) : null);
       entries.put(info, file);
     }
     return file;

platform/platform-impl/src/com/intellij/openapi/vfs/impl/jar/JarHandler.java

@@ -65,7 +65,12 @@ public class JarHandler extends ZipHandler {
   protected File getFileToUse() {
     File fileWithMirrorResolved = myFileWithMirrorResolved;
     if (fileWithMirrorResolved == null) {
-      myFileWithMirrorResolved = fileWithMirrorResolved = getMirrorFile(getFile());
+      File file = getFile();
+      fileWithMirrorResolved = getMirrorFile(file);
+      if (FileUtil.compareFiles(file, fileWithMirrorResolved) == 0) {
+        fileWithMirrorResolved = file;
+      }
+      myFileWithMirrorResolved = fileWithMirrorResolved;
     }
     return fileWithMirrorResolved;
   }
@@ -118,12 +123,11 @@ public class JarHandler extends ZipHandler {
         try {
           FileInputStream is = new FileInputStream(originalFile);
           try {
-            byte[] buffer = new byte[20 * 1024];
-
             sha1 = MessageDigest.getInstance("SHA1");
             sha1.update(String.valueOf(originalAttributes.length).getBytes(Charset.defaultCharset()));
             sha1.update((byte)0);
 
+            byte[] buffer = new byte[20 * 1024];
             while (true) {
               int read = is.read(buffer);
               if (read < 0) break;

platform/util/src/com/intellij/util/containers/ContainerUtil.java

@@ -1513,6 +1513,27 @@ public class ContainerUtil extends ContainerUtilRt {
     return res;
   }
 
+  @NotNull
+  @Contract(pure=true)
+  public static <T,U> Iterator<U> mapIterator(@NotNull final Iterator<T> iterator, @NotNull final Function<T,U> mapper) {
+    return new Iterator<U>() {
+      @Override
+      public boolean hasNext() {
+        return iterator.hasNext();
+      }
+
+      @Override
+      public U next() {
+        return mapper.fun(iterator.next());
+      }
+
+      @Override
+      public void remove() {
+        iterator.remove();
+      }
+    };
+  }
+
   @Nullable
   @Contract(pure=true)
   public static <T, L extends List<T>> T getLastItem(@Nullable L list, @Nullable T def) {

platform/util/src/com/intellij/util/io/IOUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2009 JetBrains s.r.o.
+ * Copyright 2000-2015 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -156,6 +156,9 @@ public class IOUtil {
   }
 
   public static boolean isAscii(@NotNull String str) {
+    return isAscii((CharSequence)str);
+  }
+  public static boolean isAscii(@NotNull CharSequence str) {
     for (int i = 0, length = str.length(); i < length; ++ i) {
       if (str.charAt(i) >= 128) return false;
     }

platform/util/src/com/intellij/util/text/ByteArrayCharSequence.java

@@ -15,7 +15,6 @@
  */
 package com.intellij.util.text;
 
-import com.intellij.util.io.IOUtil;
 import org.jetbrains.annotations.NotNull;
 
 public class ByteArrayCharSequence implements CharSequence {
@@ -49,16 +48,22 @@ public class ByteArrayCharSequence implements CharSequence {
 
   @NotNull
   public static CharSequence convertToBytesIfAsciiString(@NotNull String name) {
+    return convertToBytesIfAsciiString((CharSequence)name);
+  }
+
+  @NotNull
+  public static CharSequence convertToBytesIfAsciiString(@NotNull CharSequence name) {
     int length = name.length();
     if (length == 0) return "";
 
-    if (!IOUtil.isAscii(name)) {
-      return new String(name); // So we don't hold whole char[] buffer of a lengthy path on JDK 6
-    }
-
     byte[] bytes = new byte[length];
     for (int i = 0; i < length; i++) {
-      bytes[i] = (byte)name.charAt(i);
+      char c = name.charAt(i);
+      if (c >= 128) {
+        //noinspection RedundantStringConstructorCall
+        return new String(name.toString()); // So we don't hold whole char[] buffer of a lengthy path on JDK 6
+      }
+      bytes[i] = (byte)c;
     }
     return new ByteArrayCharSequence(bytes);
   }