From d7a2cfdb75685839d13294a1e09196c79607c7d4 Mon Sep 17 00:00:00 2001 From: Alexey Kudravtsev Date: Wed, 20 May 2015 13:12:32 +0300 Subject: [PATCH] ZipHandler memory optimisation: do not store relative paths to entries, compress entry short names --- .../openapi/vfs/impl/ArchiveHandler.java | 18 +- .../openapi/vfs/impl/ZipEntryMap.java | 219 ++++++++++++++++++ .../intellij/openapi/vfs/impl/ZipHandler.java | 30 ++- .../openapi/vfs/impl/jar/CoreJarHandler.java | 4 +- .../openapi/vfs/impl/jar/JarHandler.java | 10 +- .../util/containers/ContainerUtil.java | 21 ++ .../util/src/com/intellij/util/io/IOUtil.java | 5 +- .../util/text/ByteArrayCharSequence.java | 17 +- 8 files changed, 295 insertions(+), 29 deletions(-) create mode 100644 platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipEntryMap.java diff --git a/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java b/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java index 909e20fdfdb4..985b2416678f 100644 --- a/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java +++ b/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java @@ -38,12 +38,13 @@ public abstract class ArchiveHandler { protected static class EntryInfo { public final EntryInfo parent; - public final String shortName; + @NotNull + public final CharSequence shortName; public final boolean isDirectory; public final long length; public final long timestamp; - public EntryInfo(EntryInfo parent, @NotNull String shortName, boolean isDirectory, long length, long timestamp) { + public EntryInfo(EntryInfo parent, @NotNull CharSequence shortName, boolean isDirectory, long length, long timestamp) { this.parent = parent; this.shortName = shortName; this.isDirectory = isDirectory; @@ -52,18 +53,19 @@ public abstract class ArchiveHandler { } } - private final String myPath; + @NotNull + private final File myPath; private final Object myLock = new Object(); private volatile Reference> myEntries = new SoftReference>(null); - private boolean myCorrupted = false; + private boolean myCorrupted; protected ArchiveHandler(@NotNull String path) { - myPath = path; + myPath = new File(path); } @NotNull public File getFile() { - return new File(myPath); + return myPath; } @Nullable @@ -86,7 +88,7 @@ public abstract class ArchiveHandler { Set names = new HashSet(); for (EntryInfo info : getEntriesMap().values()) { if (info.parent == entry) { - names.add(info.shortName); + names.add(info.shortName.toString()); } } return ArrayUtil.toStringArray(names); @@ -114,7 +116,7 @@ public abstract class ArchiveHandler { } else { try { - map = Collections.unmodifiableMap(createEntriesMap()); + map = createEntriesMap(); } catch (Exception e) { myCorrupted = true; diff --git a/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipEntryMap.java b/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipEntryMap.java new file mode 100644 index 000000000000..3502f1973bdc --- /dev/null +++ b/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipEntryMap.java @@ -0,0 +1,219 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.openapi.vfs.impl; + +import com.intellij.openapi.util.Condition; +import com.intellij.util.Function; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.text.CharArrayUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.*; + +/** + * Map of relativePath => ArchiveHandler.EntryInfo optimised for memory: + * - it does not store keys (may be recovered from the ArchiveHandler.EntryInfo) + */ +class ZipEntryMap extends AbstractMap { + private static final ArchiveHandler.EntryInfo REMOVED = new ArchiveHandler.EntryInfo(null, "", false, 0, 0); + + private ArchiveHandler.EntryInfo[] entries; + private int size; + + ZipEntryMap() { + clear(); + } + + @Override + public ArchiveHandler.EntryInfo get(@NotNull Object key) { + String relativePath = (String)key; + int index = index(relativePath, entries); + ArchiveHandler.EntryInfo entry; + int i = index; + while (true) { + entry = entries[i]; + if (entry == null || isTheOne(entry, relativePath)) break; + if (++i == entries.length) { + i = 0; + } + if (i == index) { + entry = null; + break; + } + } + return entry; + } + + private static int index(@NotNull String relativePath, @NotNull ArchiveHandler.EntryInfo[] entries) { + return (relativePath.hashCode() & 0x7fffffff) % entries.length; + } + + @Override + public ArchiveHandler.EntryInfo put(String relativePath, ArchiveHandler.EntryInfo value) { + if (size >= 0.8 * entries.length) { + rehash(); + } + + ArchiveHandler.EntryInfo old = put(relativePath, value, entries); + if (old == null){ + size++; + } + return old; + } + + @Nullable + private static ArchiveHandler.EntryInfo put(@NotNull String relativePath, + @NotNull ArchiveHandler.EntryInfo value, + @NotNull ArchiveHandler.EntryInfo[] entries) { + int index = index(relativePath, entries); + ArchiveHandler.EntryInfo entry; + int i = index; + while (true) { + entry = entries[i]; + if (entry == null || entry == REMOVED || isTheOne(entry, relativePath)) { + entries[i] = value; + break; + } + if (++i == entries.length) { + i = 0; + } + } + return entry == REMOVED ? null : entry; + } + + private static boolean isTheOne(@NotNull ArchiveHandler.EntryInfo entry, @NotNull String relativePath) { + int endIndex = relativePath.length(); + for (ArchiveHandler.EntryInfo e = entry; e != null; e = e.parent) { + CharSequence shortName = e.shortName; + if (!CharArrayUtil.regionMatches(relativePath,endIndex-shortName.length(), relativePath.length(), shortName)) { + return false; + } + + endIndex -= shortName.length(); + if (e.parent != null && e.parent.shortName.length() != 0 && endIndex != 0) { + // match "/" + if (relativePath.charAt(endIndex-1) == '/') { + endIndex -= 1; + } + else { + return false; + } + } + + } + return endIndex==0; + } + + @NotNull + private ArchiveHandler.EntryInfo[] rehash() { + ArchiveHandler.EntryInfo[] newEntries = new ArchiveHandler.EntryInfo[entries.length < 1000 ? entries.length * 2 : entries.length * 3/2]; + for (ArchiveHandler.EntryInfo entry : entries) { + if (entry != null) { + put(getRelativePath(entry), entry, newEntries); + } + } + entries = newEntries; + return newEntries; + } + + @NotNull + private static String getRelativePath(@NotNull ArchiveHandler.EntryInfo entry) { + if (entry.parent == null) { + return entry.shortName.toString(); + } + String parentPath = getRelativePath(entry.parent); + return (parentPath.isEmpty() ? "" : parentPath + "/") + entry.shortName; + } + + @Override + public ArchiveHandler.EntryInfo remove(@NotNull Object key) { + ArchiveHandler.EntryInfo old = put((String)key, REMOVED); + if (old != null) { + size--; + } + return old; + } + + @Override + public int size() { + return size; + } + + @Override + public void clear() { + size = 0; + entries = new ArchiveHandler.EntryInfo[10]; + } + + private EntrySet entrySet; + @NotNull + @Override + public EntrySet entrySet() { + EntrySet es; + return (es = entrySet) == null ? (entrySet = new EntrySet()) : es; + } + + private final class EntrySet extends AbstractSet> { + @Override + public final int size() { + return ZipEntryMap.this.size(); + } + + @Override + public final void clear() { + ZipEntryMap.this.clear(); + } + + @Override + public final Iterator> iterator() { + return ContainerUtil.mapIterator(ContainerUtil.iterate(entries, Condition.NOT_NULL).iterator(), + new Function>() { + @Override + public Entry fun(ArchiveHandler.EntryInfo entry) { + return new SimpleEntry(getRelativePath(entry), entry); + } + }); + } + + @Override + public final boolean contains(Object o) { + if (!(o instanceof Map.Entry)) { + return false; + } + Map.Entry e = (Map.Entry)o; + String key = (String)e.getKey(); + ArchiveHandler.EntryInfo value = (ArchiveHandler.EntryInfo)e.getValue(); + return value.equals(get(key)); + } + + @Override + public final boolean remove(Object o) { + if (o instanceof Map.Entry) { + Map.Entry e = (Map.Entry)o; + Object key = e.getKey(); + return contains(o) && ZipEntryMap.this.remove(key) != null; + } + return false; + } + } + + @NotNull + @Override + public Collection values() { + return ContainerUtil.mapNotNull(entries, Function.ID); + } +} diff --git a/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java b/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java index ce5fe008eda9..f97fd7fc8693 100644 --- a/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java +++ b/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java @@ -21,8 +21,9 @@ import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.ArrayUtil; import com.intellij.util.io.FileAccessorCache; -import gnu.trove.THashMap; +import com.intellij.util.text.ByteArrayCharSequence; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.Closeable; import java.io.File; @@ -96,7 +97,7 @@ public class ZipHandler extends ArchiveHandler { @NotNull @Override protected Map createEntriesMap() throws IOException { - Map map = new THashMap(); + Map map = new ZipEntryMap(); map.put("", createRootEntry()); FileAccessorCache.Handle zipRef = ourZipFileFileAccessorCache.get(this); @@ -126,7 +127,7 @@ public class ZipHandler extends ArchiveHandler { } @NotNull - private EntryInfo getOrCreate(ZipEntry entry, Map map, ZipFile zip) { + private EntryInfo getOrCreate(@NotNull ZipEntry entry, @NotNull Map map, @NotNull ZipFile zip) { boolean isDirectory = entry.isDirectory(); String entryName = entry.getName(); if (StringUtil.endsWithChar(entryName, '/')) { @@ -142,13 +143,26 @@ public class ZipHandler extends ArchiveHandler { if (".".equals(path.second)) { return parentInfo; } - info = new EntryInfo(parentInfo, path.second, isDirectory, entry.getSize(), entry.getTime()); + info = store(map, parentInfo, path.second, isDirectory, entry.getSize(), entry.getTime(), entryName); + return info; + } + + @NotNull + private static EntryInfo store(@NotNull Map map, + @Nullable EntryInfo parentInfo, + @NotNull CharSequence shortName, + boolean isDirectory, + long size, + long time, + @NotNull String entryName) { + CharSequence sequence = shortName instanceof ByteArrayCharSequence ? shortName : ByteArrayCharSequence.convertToBytesIfAsciiString(shortName); + EntryInfo info = new EntryInfo(parentInfo, sequence, isDirectory, size, time); map.put(entryName, info); return info; } @NotNull - private EntryInfo getOrCreate(String entryName, Map map, ZipFile zip) { + private EntryInfo getOrCreate(@NotNull String entryName, Map map, @NotNull ZipFile zip) { EntryInfo info = map.get(entryName); if (info == null) { @@ -159,14 +173,12 @@ public class ZipHandler extends ArchiveHandler { Pair path = splitPath(entryName); EntryInfo parentInfo = getOrCreate(path.first, map, zip); - info = new EntryInfo(parentInfo, path.second, true, DEFAULT_LENGTH, DEFAULT_TIMESTAMP); - map.put(entryName, info); + info = store(map, parentInfo, path.second, true, DEFAULT_LENGTH, DEFAULT_TIMESTAMP, entryName); } if (!info.isDirectory) { Logger.getInstance(getClass()).info(zip.getName() + ": " + entryName + " should be a directory"); - info = new EntryInfo(info.parent, info.shortName, true, info.length, info.timestamp); - map.put(entryName, info); + info = store(map, info.parent, info.shortName, true, info.length, info.timestamp, entryName); } return info; diff --git a/platform/core-impl/src/com/intellij/openapi/vfs/impl/jar/CoreJarHandler.java b/platform/core-impl/src/com/intellij/openapi/vfs/impl/jar/CoreJarHandler.java index 2e6ae4c393cb..cffe197d3038 100644 --- a/platform/core-impl/src/com/intellij/openapi/vfs/impl/jar/CoreJarHandler.java +++ b/platform/core-impl/src/com/intellij/openapi/vfs/impl/jar/CoreJarHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2014 JetBrains s.r.o. + * Copyright 2000-2015 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ public class CoreJarHandler extends ZipHandler { if (file == null) { FileAttributes attributes = new FileAttributes(info.isDirectory, false, false, false, info.length, info.timestamp, false); EntryInfo parent = info.parent; - file = new CoreJarVirtualFile(this, info.shortName, attributes, parent != null ? getOrCreateFile(parent, entries) : null); + file = new CoreJarVirtualFile(this, info.shortName.toString(), attributes, parent != null ? getOrCreateFile(parent, entries) : null); entries.put(info, file); } return file; diff --git a/platform/platform-impl/src/com/intellij/openapi/vfs/impl/jar/JarHandler.java b/platform/platform-impl/src/com/intellij/openapi/vfs/impl/jar/JarHandler.java index 2e641b0d9707..07a58cf5d25c 100644 --- a/platform/platform-impl/src/com/intellij/openapi/vfs/impl/jar/JarHandler.java +++ b/platform/platform-impl/src/com/intellij/openapi/vfs/impl/jar/JarHandler.java @@ -65,7 +65,12 @@ public class JarHandler extends ZipHandler { protected File getFileToUse() { File fileWithMirrorResolved = myFileWithMirrorResolved; if (fileWithMirrorResolved == null) { - myFileWithMirrorResolved = fileWithMirrorResolved = getMirrorFile(getFile()); + File file = getFile(); + fileWithMirrorResolved = getMirrorFile(file); + if (FileUtil.compareFiles(file, fileWithMirrorResolved) == 0) { + fileWithMirrorResolved = file; + } + myFileWithMirrorResolved = fileWithMirrorResolved; } return fileWithMirrorResolved; } @@ -118,12 +123,11 @@ public class JarHandler extends ZipHandler { try { FileInputStream is = new FileInputStream(originalFile); try { - byte[] buffer = new byte[20 * 1024]; - sha1 = MessageDigest.getInstance("SHA1"); sha1.update(String.valueOf(originalAttributes.length).getBytes(Charset.defaultCharset())); sha1.update((byte)0); + byte[] buffer = new byte[20 * 1024]; while (true) { int read = is.read(buffer); if (read < 0) break; diff --git a/platform/util/src/com/intellij/util/containers/ContainerUtil.java b/platform/util/src/com/intellij/util/containers/ContainerUtil.java index 634e81ad8990..52ece82f3685 100644 --- a/platform/util/src/com/intellij/util/containers/ContainerUtil.java +++ b/platform/util/src/com/intellij/util/containers/ContainerUtil.java @@ -1513,6 +1513,27 @@ public class ContainerUtil extends ContainerUtilRt { return res; } + @NotNull + @Contract(pure=true) + public static Iterator mapIterator(@NotNull final Iterator iterator, @NotNull final Function mapper) { + return new Iterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public U next() { + return mapper.fun(iterator.next()); + } + + @Override + public void remove() { + iterator.remove(); + } + }; + } + @Nullable @Contract(pure=true) public static > T getLastItem(@Nullable L list, @Nullable T def) { diff --git a/platform/util/src/com/intellij/util/io/IOUtil.java b/platform/util/src/com/intellij/util/io/IOUtil.java index 11a7be8e333d..c8d8527e205b 100644 --- a/platform/util/src/com/intellij/util/io/IOUtil.java +++ b/platform/util/src/com/intellij/util/io/IOUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2009 JetBrains s.r.o. + * Copyright 2000-2015 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -156,6 +156,9 @@ public class IOUtil { } public static boolean isAscii(@NotNull String str) { + return isAscii((CharSequence)str); + } + public static boolean isAscii(@NotNull CharSequence str) { for (int i = 0, length = str.length(); i < length; ++ i) { if (str.charAt(i) >= 128) return false; } diff --git a/platform/util/src/com/intellij/util/text/ByteArrayCharSequence.java b/platform/util/src/com/intellij/util/text/ByteArrayCharSequence.java index 843d0664f127..fcf7ea4f1dbf 100644 --- a/platform/util/src/com/intellij/util/text/ByteArrayCharSequence.java +++ b/platform/util/src/com/intellij/util/text/ByteArrayCharSequence.java @@ -15,7 +15,6 @@ */ package com.intellij.util.text; -import com.intellij.util.io.IOUtil; import org.jetbrains.annotations.NotNull; public class ByteArrayCharSequence implements CharSequence { @@ -49,16 +48,22 @@ public class ByteArrayCharSequence implements CharSequence { @NotNull public static CharSequence convertToBytesIfAsciiString(@NotNull String name) { + return convertToBytesIfAsciiString((CharSequence)name); + } + + @NotNull + public static CharSequence convertToBytesIfAsciiString(@NotNull CharSequence name) { int length = name.length(); if (length == 0) return ""; - if (!IOUtil.isAscii(name)) { - return new String(name); // So we don't hold whole char[] buffer of a lengthy path on JDK 6 - } - byte[] bytes = new byte[length]; for (int i = 0; i < length; i++) { - bytes[i] = (byte)name.charAt(i); + char c = name.charAt(i); + if (c >= 128) { + //noinspection RedundantStringConstructorCall + return new String(name.toString()); // So we don't hold whole char[] buffer of a lengthy path on JDK 6 + } + bytes[i] = (byte)c; } return new ByteArrayCharSequence(bytes); }