From d1b900d16197043c5f4be15467df1492e0679cb8 Mon Sep 17 00:00:00 2001 From: Roman Shevchenko Date: Fri, 10 Feb 2017 18:22:54 +0100 Subject: [PATCH] [vfs] prevent OOME on reading enormous files from archives (IDEA-167934) --- .../openapi/vfs/impl/ArchiveHandler.java | 9 +- .../intellij/openapi/vfs/impl/ZipHandler.java | 136 +++++++++++++----- .../openapi/vfs/newvfs/ArchiveFileSystem.java | 5 +- 3 files changed, 114 insertions(+), 36 deletions(-) diff --git a/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java b/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java index 26fcff811c8f..4f4e22914e37 100644 --- a/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java +++ b/platform/core-api/src/com/intellij/openapi/vfs/impl/ArchiveHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2016 JetBrains s.r.o. + * Copyright 2000-2017 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.intellij.openapi.vfs.impl; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.io.BufferExposingByteArrayInputStream; import com.intellij.openapi.util.io.FileAttributes; import com.intellij.openapi.util.io.FileSystemUtil; import com.intellij.reference.SoftReference; @@ -30,6 +31,7 @@ import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.lang.ref.Reference; import java.util.Collections; import java.util.List; @@ -233,6 +235,11 @@ public abstract class ArchiveHandler { @NotNull public abstract byte[] contentsToByteArray(@NotNull String relativePath) throws IOException; + @NotNull + public InputStream getInputStream(@NotNull String relativePath) throws IOException { + return new BufferExposingByteArrayInputStream(contentsToByteArray(relativePath)); + } + private static final AddonlyKeylessHash.KeyValueMapper ourKeyValueMapper = new AddonlyKeylessHash.KeyValueMapper() { @Override public int hash(EntryInfo info) { diff --git a/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java b/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java index 0cce9b25af70..5578dd2b6dd3 100644 --- a/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java +++ b/platform/core-impl/src/com/intellij/openapi/vfs/impl/ZipHandler.java @@ -17,9 +17,7 @@ package com.intellij.openapi.vfs.impl; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.io.FileAttributes; -import com.intellij.openapi.util.io.FileSystemUtil; -import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.io.*; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.io.FileAccessorCache; import com.intellij.util.text.ByteArrayCharSequence; @@ -33,6 +31,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Enumeration; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -108,27 +107,34 @@ public class ZipHandler extends ArchiveHandler { } @Contract("true -> !null") - protected FileAccessorCache.Handle getCachedZipFileHandle(boolean createHandleIfNeeded) throws IOException { - FileAccessorCache.Handle handle = createHandleIfNeeded ? ourZipFileFileAccessorCache.get(this) : ourZipFileFileAccessorCache.getIfCached(this); + protected FileAccessorCache.Handle getCachedZipFileHandle(boolean createIfNeeded) throws IOException { + try { + FileAccessorCache.Handle handle = createIfNeeded ? ourZipFileFileAccessorCache.get(this) : ourZipFileFileAccessorCache.getIfCached(this); - // check handle is valid - if (handle != null && getFile() == getFileToUse()) { // files are canonicalized - // IDEA-148458, http://bugs.java.com/view_bug.do?bug_id=4425695, JVM crashes on use of opened ZipFile after it was updated - // Reopen file if the file has been changed - FileAttributes attributes = FileSystemUtil.getAttributes(getCanonicalPathToZip()); - if (attributes == null) { - throw new FileNotFoundException(getCanonicalPathToZip()); + // check handle is valid + if (handle != null && getFile() == getFileToUse()) { // files are canonicalized + // IDEA-148458, http://bugs.java.com/view_bug.do?bug_id=4425695, JVM crashes on use of opened ZipFile after it was updated + // Reopen file if the file has been changed + FileAttributes attributes = FileSystemUtil.getAttributes(getCanonicalPathToZip()); + if (attributes == null) { + throw new FileNotFoundException(getCanonicalPathToZip()); + } + + if (attributes.lastModified == myFileStamp && attributes.length == myFileLength) return handle; + + // Note that zip_util.c#ZIP_Get_From_Cache will allow us to have duplicated ZipFile instances without a problem + removeZipHandlerFromCache(); + handle.release(); + handle = ourZipFileFileAccessorCache.get(this); } - if (attributes.lastModified == myFileStamp && attributes.length == myFileLength) return handle; - - // Note that zip_util.c#ZIP_Get_From_Cache will allow us to have duplicated ZipFile instances without a problem - removeZipHandlerFromCache(); - handle.release(); - handle = ourZipFileFileAccessorCache.get(this); + return handle; + } + catch (RuntimeException e) { + Throwable cause = e.getCause(); + if (cause instanceof IOException) throw (IOException)cause; + throw e; } - - return handle; } private void removeZipHandlerFromCache() { @@ -207,26 +213,20 @@ public class ZipHandler extends ArchiveHandler { @NotNull @Override public byte[] contentsToByteArray(@NotNull String relativePath) throws IOException { - FileAccessorCache.Handle zipRef; - - try { - zipRef = getCachedZipFileHandle(true); - } - catch (RuntimeException ex) { - Throwable cause = ex.getCause(); - if (cause instanceof IOException) throw (IOException)cause; - throw ex; - } - + FileAccessorCache.Handle zipRef = getCachedZipFileHandle(true); try { ZipFile zip = zipRef.get(); ZipEntry entry = zip.getEntry(relativePath); if (entry != null) { + long length = entry.getSize(); + if (FileUtilRt.isTooLarge(length)) { + throw new FileTooBigException(getFile() + "!/" + relativePath); + } InputStream stream = zip.getInputStream(entry); if (stream != null) { // ZipFile.c#Java_java_util_zip_ZipFile_read reads data in 8K (stack allocated) blocks - no sense to create BufferedInputStream try { - return FileUtil.loadBytes(stream, (int)entry.getSize()); + return FileUtil.loadBytes(stream, (int)length); } finally { stream.close(); @@ -241,6 +241,78 @@ public class ZipHandler extends ArchiveHandler { throw new FileNotFoundException(getFile() + "!/" + relativePath); } + @NotNull + @Override + public InputStream getInputStream(@NotNull String relativePath) throws IOException { + boolean release = true; + final FileAccessorCache.Handle zipRef = getCachedZipFileHandle(true); + try { + ZipFile zip = zipRef.get(); + ZipEntry entry = zip.getEntry(relativePath); + if (entry != null) { + InputStream stream = zip.getInputStream(entry); + if (stream != null) { + long length = entry.getSize(); + if (!FileUtilRt.isTooLarge(length)) { + try { + return new BufferExposingByteArrayInputStream(FileUtil.loadBytes(stream, (int)length)); + } + finally { + stream.close(); + } + } + else { + release = false; + return new InputStreamWrapper(stream, zipRef); + } + } + } + } + finally { + if (release) zipRef.release(); + } + + throw new FileNotFoundException(getFile() + "!/" + relativePath); + } + + private static class InputStreamWrapper extends InputStream { + private final InputStream myStream; + private final FileAccessorCache.Handle myZipRef; + private final AtomicBoolean closed = new AtomicBoolean(false); + + public InputStreamWrapper(InputStream stream, FileAccessorCache.Handle zipRef) { + myStream = stream; + myZipRef = zipRef; + } + + @Override + public int read() throws IOException { + return myStream.read(); + } + + @Override + public int read(@NotNull byte[] b, int off, int len) throws IOException { + return myStream.read(b, off, len); + } + + @Override + public int available() throws IOException { + return myStream.available(); + } + + @Override + public void close() throws IOException { + if (!closed.getAndSet(true)) { + try { + myStream.close(); + } + finally { + myZipRef.release(); + } + } + } + } + // also used in Kotlin public static void clearFileAccessorCache() { ourZipFileFileAccessorCache.clear(); diff --git a/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/ArchiveFileSystem.java b/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/ArchiveFileSystem.java index b0777554ecaf..c59c11003910 100644 --- a/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/ArchiveFileSystem.java +++ b/platform/platform-api/src/com/intellij/openapi/vfs/newvfs/ArchiveFileSystem.java @@ -1,5 +1,5 @@ /* - * Copyright 2000-2016 JetBrains s.r.o. + * Copyright 2000-2017 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ package com.intellij.openapi.vfs.newvfs; import com.intellij.openapi.fileTypes.FileTypeRegistry; import com.intellij.openapi.fileTypes.FileTypes; import com.intellij.openapi.util.Key; -import com.intellij.openapi.util.io.BufferExposingByteArrayInputStream; import com.intellij.openapi.util.io.FileAttributes; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.*; @@ -215,7 +214,7 @@ public abstract class ArchiveFileSystem extends NewVirtualFileSystem { @NotNull @Override public InputStream getInputStream(@NotNull VirtualFile file) throws IOException { - return new BufferExposingByteArrayInputStream(contentsToByteArray(file)); + return getHandler(file).getInputStream(getRelativePath(file)); } @Override