diff --git a/java/java-tests/testSrc/com/intellij/util/indexing/IndexPackTest.java b/java/java-tests/testSrc/com/intellij/util/indexing/IndexPackTest.java new file mode 100644 index 000000000000..a785eff35419 --- /dev/null +++ b/java/java-tests/testSrc/com/intellij/util/indexing/IndexPackTest.java @@ -0,0 +1,305 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing; + +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.testFramework.PlatformTestUtil; +import com.intellij.util.containers.MultiMap; +import com.intellij.util.indexing.impl.MapIndexStorage; +import com.intellij.util.indexing.impl.MapReduceIndex; +import com.intellij.util.indexing.impl.ReadOnlyIndexPack; +import com.intellij.util.indexing.impl.forward.KeyCollectionForwardIndexAccessor; +import com.intellij.util.indexing.impl.forward.PersistentMapBasedForwardIndex; +import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem; +import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider; +import com.intellij.util.io.DataExternalizer; +import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.StringEnumeratorTest; +import com.intellij.util.io.zip.JBZipFile; +import gnu.trove.TIntHashSet; +import junit.framework.AssertionFailedError; +import junit.framework.TestCase; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +public class IndexPackTest extends TestCase { + private static final Logger LOG = Logger.getInstance(IndexPackTest.class); + + public void testSimpleIndexPack() throws IOException, StorageException { + File dir = FileUtil.createTempDirectory("persistent-map", "packs"); + + MapReduceIndex m1 = createSplitStringIndex(dir.toPath().resolve("index1").resolve("index"), false); + MapReduceIndex m2 = createSplitStringIndex(dir.toPath().resolve("index2").resolve("index"), false); + + assertTrue(m1.update(1, "key/value").compute()); + assertTrue(m1.update(2, "key2/value2").compute()); + assertTrue(m1.update(3, "key/value2").compute()); + + assertTrue(m2.update(1, "key/value").compute()); + assertTrue(m2.update(2, "key2/value2").compute()); + assertTrue(m2.update(3, "key/value2").compute()); + + m1.dispose(); + m2.dispose(); + + File pack = new File(dir, "pack.zip"); + try (JBZipFile file = new JBZipFile(pack)) { + for (String index : Arrays.asList("index1", "index2")) { + for (Path path : Files.newDirectoryStream(dir.toPath().resolve(index))) { + file.getOrCreateEntry(index + "/" + path.getFileName().toString()).setDataFromFile(path.toFile()); + } + } + } + + MultiMap> indexMap1 = new MultiMap<>(); + indexMap1.putValue("key", Pair.create("value", 1)); + indexMap1.putValue("key", Pair.create("value2", 3)); + indexMap1.putValue("key2", Pair.create("value2", 2)); + + MultiMap> indexMap2 = new MultiMap<>(); + indexMap2.putValue("key", Pair.create("value", 1)); + indexMap2.putValue("key", Pair.create("value2", 3)); + indexMap2.putValue("key2", Pair.create("value2", 2)); + + try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) { + MapReduceIndex mm1 = createSplitStringIndex(fs.getPath("index1").resolve("index"), true); + MapReduceIndex mm2 = createSplitStringIndex(fs.getPath("index2").resolve("index"), true); + + for (String key : new ArrayList<>(indexMap1.keySet())) { + mm1.getData(key).forEach((id, value) -> { + indexMap1.remove(key, Pair.create(value, id)); + return true; + }); + } + + for (String key : new ArrayList<>(indexMap2.keySet())) { + mm2.getData(key).forEach((id, value) -> { + indexMap2.remove(key, Pair.create(value, id)); + return true; + }); + } + + mm1.dispose(); + mm2.dispose(); + } + + assertTrue(indexMap1.isEmpty()); + assertTrue(indexMap2.isEmpty()); + } + + private static final int sampleCount = 1000; + + public void testIndexAccessPerformance() throws IOException { + File dir = FileUtil.createTempDirectory("persistent-map", "packs"); + + TIntHashSet keys = new TIntHashSet(); + InvertedIndex index = createStringLengthIndex(dir.toPath().resolve("index"), false); + for (int i = 1; i <= sampleCount; ++i) { + final String string = generateString(); + keys.add(string.length()); + index.update(i, string).compute(); + } + index.dispose(); + + PlatformTestUtil.startPerformanceTest("read", 2000, () -> { + InvertedIndex readIndex = createStringLengthIndex(dir.toPath().resolve("index"), true); + LongAdder recordCount = new LongAdder(); + for (int key : keys.toArray()) { + readIndex.getData(String.valueOf(key)).forEach((id, value) -> { + recordCount.increment(); + return true; + }); + } + assertEquals(sampleCount, recordCount.intValue()); + readIndex.dispose(); + }).attempts(5).ioBound().assertTiming(); + } + + public void testTrivialPackAccessPerformance() throws IOException { + File dir = FileUtil.createTempDirectory("persistent-map", "packs"); + + TIntHashSet keys = new TIntHashSet(); + InvertedIndex index = createStringLengthIndex(dir.toPath().resolve("index").resolve("index"), false); + for (int i = 1; i <= sampleCount; ++i) { + final String string = generateString(); + keys.add(string.length()); + index.update(i, string).compute(); + } + index.dispose(); + + File pack = new File(dir, "pack.zip"); + try (JBZipFile file = new JBZipFile(pack)) { + for (Path path : Files.newDirectoryStream(dir.toPath().resolve("index"))) { + file.getOrCreateEntry("index/" + path.getFileName().toString()).setDataFromFile(path.toFile()); + } + } + + try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) { + PlatformTestUtil.startPerformanceTest("read", 2000, () -> { + InvertedIndex readIndex = createStringLengthIndex(fs.getPath("index", "index"), true); + LongAdder recordCount = new LongAdder(); + for (int key : keys.toArray()) { + readIndex.getData(String.valueOf(key)).forEach((id, value) -> { + recordCount.increment(); + return true; + }); + } + assertEquals(sampleCount, recordCount.intValue()); + readIndex.dispose(); + }).attempts(5).ioBound().assertTiming(); + } + } + + public void test1000PackAccessPerformance() throws IOException { + File dir = FileUtil.createTempDirectory("persistent-map", "packs"); + + TIntHashSet keys = new TIntHashSet(); + int packSize = 1000; + List> indexes = generateIndexNames(packSize) + .map(name -> createStringLengthIndex(dir.toPath().resolve(name).resolve("index"), false)) + .collect(Collectors.toList()); + for (int i = 1; i <= sampleCount; ++i) { + final String string = generateString(); + keys.add(string.length()); + indexes.get(Math.abs(string.hashCode() % packSize)).update(i, string).compute(); + } + indexes.forEach(InvertedIndex::dispose); + + File pack = new File(dir, "pack.zip"); + try (JBZipFile file = new JBZipFile(pack)) { + generateIndexNames(packSize).map(name -> dir.toPath().resolve(name)).forEach(p -> { + try { + for (Path path : Files.newDirectoryStream(p)) { + file.getOrCreateEntry(path.getParent().getFileName().toString() + "/" + path.getFileName().toString()).setDataFromFile(path.toFile()); + } + } + catch (IOException e) { + LOG.error(e); + throw new AssertionFailedError(e.getMessage()); + } + }); + } + + try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) { + PlatformTestUtil.startPerformanceTest("read", 4000, () -> { + + ReadOnlyIndexPack indexPack = new ReadOnlyIndexPack<>(generateIndexNames(packSize) + .map(name -> createStringLengthIndex(fs.getPath(name, "index"), true)) + .collect(Collectors.toList())); + + LongAdder recordCount = new LongAdder(); + for (int key : keys.toArray()) { + indexPack.getData(String.valueOf(key)).forEach((id, value) -> { + recordCount.increment(); + return true; + }); + } + assertEquals(sampleCount, recordCount.intValue()); + indexPack.dispose(); + }).attempts(5).ioBound().assertTiming(); + } + } + + @NotNull + private static Stream generateIndexNames(int packSize) { + return IntStream + .rangeClosed(1, packSize) + .mapToObj(idx -> "index" + idx); + } + + @NotNull + private static String generateString() { + return StringUtil.repeat(StringEnumeratorTest.createRandomString(), 30); + } + + private static MapReduceIndex createSplitStringIndex(@NotNull Path path, boolean readOnly) throws IOException { + return createIndex(path, readOnly, new DataIndexer() { + @NotNull + @Override + public Map map(@NotNull String inputData) { + String[] split = inputData.split("/"); + return Collections.singletonMap(split[0], split[1]); + } + }); + } + + + private static InvertedIndex createStringLengthIndex(@NotNull Path path, boolean readOnly) { + try { + return createIndex(path, readOnly, new DataIndexer() { + @NotNull + @Override + public Map map(@NotNull String inputData) { + return Collections.singletonMap(String.valueOf(inputData.length()), inputData + "_value"); + } + }); + } + catch (IOException e) { + LOG.error(e); + throw new AssertionFailedError(e.getMessage()); + } + } + + private static MapReduceIndex createIndex(@NotNull Path path, boolean readOnly, DataIndexer indexer) throws IOException { + IndexExtension extension = new IndexExtension() { + @NotNull + @Override + public IndexId getName() { + return IndexId.create("AnIndex"); + } + + @NotNull + @Override + public DataIndexer getIndexer() { + return indexer; + } + + @NotNull + @Override + public KeyDescriptor getKeyDescriptor() { + return EnumeratorStringDescriptor.INSTANCE; + } + + @NotNull + @Override + public DataExternalizer getValueExternalizer() { + return EnumeratorStringDescriptor.INSTANCE; + } + + @Override + public int getVersion() { + return 0; + } + }; + return new MapReduceIndex(extension, new MapIndexStorage(path.getParent().resolve(path.getFileName() + ".storage"), EnumeratorStringDescriptor.INSTANCE, + EnumeratorStringDescriptor.INSTANCE, 1024, false, true, readOnly, null) { + @Override + protected void checkCanceled() { + + } + }, new PersistentMapBasedForwardIndex(path.getParent().resolve(path.getFileName() + ".forward"), readOnly), new KeyCollectionForwardIndexAccessor<>(extension)) { + @Override + public void checkCanceled() { + + } + + @Override + protected void requestRebuild(@NotNull Throwable e) { + e.printStackTrace(); + fail(); + } + }; + } +} diff --git a/platform/lang-impl/src/com/intellij/codeInsight/TargetElementUtil.java b/platform/lang-impl/src/com/intellij/codeInsight/TargetElementUtil.java index 5246a8c2949a..3463a1222c23 100644 --- a/platform/lang-impl/src/com/intellij/codeInsight/TargetElementUtil.java +++ b/platform/lang-impl/src/com/intellij/codeInsight/TargetElementUtil.java @@ -30,6 +30,8 @@ import com.intellij.psi.util.PsiUtilCore; import com.intellij.util.BitUtil; import com.intellij.util.Consumer; import com.intellij.util.ThreeState; +import com.intellij.util.indexing.DumbModeAccessType; +import com.intellij.util.indexing.FileBasedIndex; import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -332,14 +334,23 @@ public class TargetElementUtil { PsiReference ref = findReference(editor, offset); if (ref == null) return null; - final Language language = ref.getElement().getLanguage(); - TargetElementEvaluator evaluator = TargetElementUtilBase.TARGET_ELEMENT_EVALUATOR.forLanguage(language); - if (evaluator != null) { - final PsiElement element = evaluator.getElementByReference(ref, flags); - if (element != null) return element; - } + Project project = editor.getProject(); + if (project == null) return null; + PsiElement[] result = {null}; + FileBasedIndex.getInstance().ignoreDumbMode(() -> { + final Language language = ref.getElement().getLanguage(); + TargetElementEvaluator evaluator = TargetElementUtilBase.TARGET_ELEMENT_EVALUATOR.forLanguage(language); + if (evaluator != null) { + final PsiElement element = evaluator.getElementByReference(ref, flags); + if (element != null) { + result[0] = element; + return; + } + } + result[0] = ref.resolve(); + }, project, DumbModeAccessType.RELIABLE_DATA_ONLY); - return ref.resolve(); + return result[0]; } @NotNull diff --git a/platform/lang-impl/src/com/intellij/util/indexing/zipFs/FileBlockReadOnlyFileChannel.java b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/FileBlockReadOnlyFileChannel.java new file mode 100644 index 000000000000..1468eee3d7cf --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/FileBlockReadOnlyFileChannel.java @@ -0,0 +1,134 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.zipFs; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +public class FileBlockReadOnlyFileChannel extends FileChannel { + @NotNull + private final FileChannel myUnderlying; + private final long myStartOffset; + private final long myEndOffset; + private final long mySize; + + // TODO duplicates? + private volatile long myGlobalPosition; + private volatile long myLocalPosition; + + public FileBlockReadOnlyFileChannel(@NotNull FileChannel underlying, + long startOffset, + long size) { + myUnderlying = underlying; + + myStartOffset = startOffset; + myEndOffset = startOffset + size; + mySize = size; + + myGlobalPosition = myStartOffset; + myLocalPosition = 0; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + int read = read(dst, myLocalPosition); + if (read != -1 && read != 0) { + position(myLocalPosition + read); + } + return read; + } + + @Override + public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { + throw new UnsupportedOperationException("Implement it!"); + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long write(ByteBuffer[] srcs, int offset, int length) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long position() { + return myGlobalPosition - myStartOffset; + } + + @Override + public FileChannel position(long newPosition) { + myGlobalPosition = myStartOffset + newPosition; + myLocalPosition = newPosition; + assert myGlobalPosition <= myEndOffset; + assert myLocalPosition <= mySize; + return this; + } + + @Override + public long size() { + return mySize; + } + + @Override + public FileChannel truncate(long size){ + throw new UnsupportedOperationException(); + } + + @Override + public void force(boolean metaData) { + // do nothing + } + + @Override + public long transferTo(long position, long count, WritableByteChannel target) { + throw new UnsupportedOperationException("Implement it!"); + } + + @Override + public long transferFrom(ReadableByteChannel src, long position, long count) { + throw new UnsupportedOperationException(); + } + + @Override + public int read(ByteBuffer dst, long position) throws IOException { + long globalPosition = myStartOffset + position; + if (position >= mySize) { + return -1; + } + return myUnderlying.read(dst, globalPosition); + } + + @Override + public int write(ByteBuffer src, long position) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public MappedByteBuffer map(MapMode mode, long position, long size) { + throw new UnsupportedOperationException(); + } + + @Override + public FileLock lock(long position, long size, boolean shared) { + return null; + } + + @Override + public FileLock tryLock(long position, long size, boolean shared) { + return null; + } + + @Override + protected void implCloseChannel() { + // do nothing + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipEntryFileAttributes.java b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipEntryFileAttributes.java new file mode 100644 index 000000000000..ae86b76e8d76 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipEntryFileAttributes.java @@ -0,0 +1,63 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.zipFs; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; + +public class UncompressedZipEntryFileAttributes implements BasicFileAttributes { + private final UncompressedZipPath myPath; + private final UncompressedZipFileSystem.ZipTreeNode myNode; + + public UncompressedZipEntryFileAttributes(@NotNull UncompressedZipPath path) throws IOException { + myPath = path; + myNode = UncompressedZipFileSystemProvider.find(path); + } + + @Override + public FileTime lastModifiedTime() { + throw new UnsupportedOperationException(); + } + + @Override + public FileTime lastAccessTime() { + throw new UnsupportedOperationException(); + } + + @Override + public FileTime creationTime() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isRegularFile() { + return !isDirectory(); + } + + @Override + public boolean isDirectory() { + return myNode.isDirectory(); + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public long size() { + return myNode.getEntry().getCompressedSize(); + } + + @Override + public Object fileKey() { + throw new UnsupportedOperationException(); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileStore.java b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileStore.java new file mode 100644 index 000000000000..a60575f750de --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileStore.java @@ -0,0 +1,68 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.zipFs; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.file.FileStore; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.FileStoreAttributeView; + +class UncompressedZipFileStore extends FileStore { + @NotNull + private final UncompressedZipFileSystem mySystem; + + UncompressedZipFileStore(@NotNull UncompressedZipFileSystem system) { + mySystem = system; + } + + @Override + public String name() { + return "zip0"; + } + + @Override + public String type() { + return "zip0"; + } + + @Override + public boolean isReadOnly() { + return true; + } + + @Override + public long getTotalSpace() throws IOException { + return mySystem.getChannel().size(); + } + + @Override + public long getUsableSpace() throws IOException { + return mySystem.getChannel().size(); + } + + @Override + public long getUnallocatedSpace() { + return 0; + } + + @Override + public boolean supportsFileAttributeView(Class type) { + return false; + } + + @Override + public boolean supportsFileAttributeView(String name) { + return false; + } + + @Override + public V getFileStoreAttributeView(Class type) { + return null; + } + + @Override + public Object getAttribute(String attribute) { + throw new UnsupportedOperationException(); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileSystem.java b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileSystem.java new file mode 100644 index 000000000000..b7043ab8090d --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileSystem.java @@ -0,0 +1,198 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.zipFs; + +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.ArrayUtil; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.io.zip.JBZipEntry; +import com.intellij.util.io.zip.JBZipFile; +import gnu.trove.THashMap; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.*; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.nio.file.spi.FileSystemProvider; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +public class UncompressedZipFileSystem extends FileSystem { + @NotNull + private final JBZipFile myUncompressedZip; + @NotNull + private final FileChannel myChannel; + @NotNull + private final Path myUncompressedZipPath; + private final UncompressedZipFileSystemProvider myProvider; + private final ZipTreeNode myRoot = new ZipTreeNode(); + // probably should be eliminated + private final Set myOpenFiles = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + public UncompressedZipFileSystem(@NotNull Path uncompressedZip, @NotNull UncompressedZipFileSystemProvider provider) throws IOException { + myUncompressedZipPath = uncompressedZip; + myProvider = provider; + assert uncompressedZip.getFileSystem() == FileSystems.getDefault(); + myUncompressedZip = new JBZipFile(uncompressedZip.toFile()); + myChannel = FileChannel.open(uncompressedZip, StandardOpenOption.READ); + buildTree(); + } + + @NotNull + FileChannel openChannel(@NotNull JBZipEntry entry) throws IOException { + FileBlockReadOnlyFileChannel channel = new FileBlockReadOnlyFileChannel(myChannel, entry.calcDataOffset(), entry.getSize()) { + @Override + protected void implCloseChannel() { + myOpenFiles.remove(this); + } + }; + myOpenFiles.add(channel); + return channel; + } + + private void buildTree() { + for (JBZipEntry entry : myUncompressedZip.getEntries()) { + List names = StringUtil.split(entry.getName(), getSeparator()); + ZipTreeNode current = myRoot; + for (int i = 0; i < names.size(); i++) { + if (i == names.size() - 1) { + current.createEntryChild(names.get(i), entry); + } else { + current = current.createDirChild(names.get(i)); + } + } + } + } + + @Override + public FileSystemProvider provider() { + return myProvider; + } + + @Override + public void close() throws IOException { + try { + myChannel.close(); + } finally { + myUncompressedZip.close(); + } + } + + @NotNull + Path getUncompressedZipPath() { + return myUncompressedZipPath; + } + + @NotNull + FileChannel getChannel() { + return myChannel; + } + + @Override + public boolean isOpen() { + return myChannel.isOpen(); + } + + @Override + public boolean isReadOnly() { + return true; + } + + @Override + public String getSeparator() { + return "/"; + } + + @Override + public Iterable getRootDirectories() { + return Collections.singleton(new UncompressedZipPath(this, ArrayUtil.EMPTY_STRING_ARRAY, true)); + } + + @Override + public Iterable getFileStores() { + return Collections.singleton(new UncompressedZipFileStore(this)); + } + + @Override + public Set supportedFileAttributeViews() { + return Collections.singleton("basic"); + } + + @NotNull + @Override + public Path getPath(@NotNull String first, @NotNull String... more) { + // should it be absolute + String[] nameElements = ArrayUtil.toStringArray(ContainerUtil.concat(Collections.singletonList(first), Arrays.asList(more))); + return new UncompressedZipPath(this, nameElements, true); + } + + @Override + public PathMatcher getPathMatcher(String syntaxAndPattern) { + throw new UnsupportedOperationException(); + } + + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() { + throw new UnsupportedOperationException(); + } + + @Override + public WatchService newWatchService() { + throw new UnsupportedOperationException(); + } + + ZipTreeNode getRoot() { + return myRoot; + } + + static class ZipTreeNode { + @Nullable + private final Map myChildren; + @Nullable + private final JBZipEntry myEntry; + + public boolean isDirectory() { + return myChildren != null; + } + + @NotNull + JBZipEntry getEntry() { + assert myEntry != null; + return myEntry; + } + + Set getChildNames() { + assert myChildren != null; + return myChildren.keySet(); + } + + ZipTreeNode(@NotNull JBZipEntry entry) { + myEntry = entry; + myChildren = null; + } + + ZipTreeNode() { + myEntry = null; + myChildren = new THashMap<>(); + } + + @Nullable + ZipTreeNode getChild(@NotNull String childName) { + assert myChildren != null; + return myChildren.get(childName); + } + + @NotNull + ZipTreeNode createDirChild(@NotNull String childName) { + assert myChildren != null; + return myChildren.computeIfAbsent(childName, __ -> new ZipTreeNode()); + } + + void createEntryChild(@NotNull String childName, @NotNull JBZipEntry entry) { + assert myChildren != null; + ZipTreeNode previous = myChildren.put(childName, new ZipTreeNode(entry)); + assert previous == null; + } + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileSystemProvider.java b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileSystemProvider.java new file mode 100644 index 000000000000..febe9606a0f8 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipFileSystemProvider.java @@ -0,0 +1,172 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.zipFs; + +import com.intellij.util.ArrayUtil; +import com.intellij.util.io.zip.JBZipEntry; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.spi.FileSystemProvider; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +public class UncompressedZipFileSystemProvider extends FileSystemProvider { + @Override + public String getScheme() { + return "zip0"; + } + + @Override + public FileSystem newFileSystem(URI uri, Map env) { + return null; + } + + @Override + public FileSystem getFileSystem(URI uri) { + throw new UnsupportedOperationException(); + } + + @NotNull + @Override + public Path getPath(@NotNull URI uri) { + throw new UnsupportedOperationException(); + } + + @Override + public SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) throws IOException { + return newFileChannel(path, options, attrs); + } + + @Override + public FileChannel newFileChannel(Path path, Set options, FileAttribute... attrs) throws IOException { + UncompressedZipFileSystem.ZipTreeNode node = find(path); + if (node.isDirectory()) { + throw new IllegalArgumentException(path.toString()); + } + JBZipEntry entry = node.getEntry(); + return ((UncompressedZipFileSystem)path.getFileSystem()).openChannel(entry); + } + + @Override + public DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) throws IOException { + UncompressedZipFileSystem.ZipTreeNode element = find(dir); + if (!element.isDirectory()) { + throw new NotDirectoryException(dir.toString()); + } + return new DirectoryStream() { + @Override + public Iterator iterator() { + return element.getChildNames().stream().map(n -> dir.resolve(n)).filter(p -> { + try { + return filter.accept(p); + } + catch (IOException e) { + throw new RuntimeException(e); + } + }).iterator(); + } + + @Override + public void close() { } + }; + } + + @Override + public void createDirectory(Path dir, FileAttribute... attrs) { + throw new UnsupportedOperationException(); + } + + @Override + public void delete(Path path) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void copy(Path source, Path target, CopyOption... options) { + throw new UnsupportedOperationException(); + } + + @Override + public void move(Path source, Path target, CopyOption... options) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isSameFile(Path path1, Path path2) throws IOException { + if (path1.getFileSystem().provider() != path2.getFileSystem().provider()) { + return false; + } + UncompressedZipFileSystem system1 = (UncompressedZipFileSystem)path1.getFileSystem(); + UncompressedZipFileSystem system2 = (UncompressedZipFileSystem)path2.getFileSystem(); + if (!Files.isSameFile(system1.getUncompressedZipPath(), system2.getUncompressedZipPath())) { + return false; + } + UncompressedZipPath absolutePath1 = (UncompressedZipPath)path1.toAbsolutePath(); + UncompressedZipPath absolutePath2 = (UncompressedZipPath)path2.toAbsolutePath(); + return Arrays.equals(absolutePath1.getNameElements(), absolutePath2.getNameElements()); + } + + @Override + public boolean isHidden(Path path) { + return false; + } + + @Override + public FileStore getFileStore(Path path) { + return new UncompressedZipFileStore(((UncompressedZipFileSystem)path.getFileSystem())); + } + + @Override + public void checkAccess(Path path, AccessMode... modes) throws IOException { + if (ArrayUtil.contains(AccessMode.WRITE, path) || ArrayUtil.contains(AccessMode.EXECUTE, path)) { + throw new UnsupportedOperationException(); + } + find(path); + } + + @Override + public V getFileAttributeView(Path path, Class type, LinkOption... options) { + throw new UnsupportedOperationException(); + } + + @Override + public A readAttributes(Path path, Class type, LinkOption... options) throws IOException { + if (type == BasicFileAttributes.class) { + return (A)new UncompressedZipEntryFileAttributes(((UncompressedZipPath)path)); + } + throw new UnsupportedOperationException(); + } + + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) { + throw new UnsupportedOperationException(); + } + + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) { + throw new UnsupportedOperationException(); + } + + @NotNull + static UncompressedZipFileSystem.ZipTreeNode find(@NotNull Path dir) throws IOException { + Path absoluteDir = dir.toAbsolutePath(); + String[] elements = ((UncompressedZipPath)absoluteDir).getNameElements(); + UncompressedZipFileSystem.ZipTreeNode currentElement = ((UncompressedZipFileSystem)absoluteDir.getFileSystem()).getRoot(); + for (String element : elements) { + currentElement = currentElement.getChild(element); + if (currentElement == null) { + throw new NotDirectoryException(dir.toString()); + } + } + return currentElement; + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipPath.java b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipPath.java new file mode 100644 index 000000000000..7743a39b6c96 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/zipFs/UncompressedZipPath.java @@ -0,0 +1,198 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.zipFs; + +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.ArrayUtil; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.net.URI; +import java.nio.file.*; +import java.util.Iterator; +import java.util.stream.Stream; + +public class UncompressedZipPath implements Path { + @NotNull + private final UncompressedZipFileSystem myFileSystem; + @NotNull + private final String[] myNameElements; + private final boolean myAbsolute; + + public UncompressedZipPath(@NotNull UncompressedZipFileSystem system, + @NotNull String[] elements, + boolean absolute) { + myFileSystem = system; + myNameElements = elements; + myAbsolute = absolute; + } + + @NotNull + String[] getNameElements() { + return myNameElements; + } + + @NotNull + @Override + public FileSystem getFileSystem() { + return myFileSystem; + } + + @Override + public boolean isAbsolute() { + return myAbsolute; + } + + @Override + public Path getRoot() { + return isAbsolute() ? new UncompressedZipPath(myFileSystem, ArrayUtil.EMPTY_STRING_ARRAY, true) : null; + } + + @Override + public Path getFileName() { + return getName(getNameCount() - 1); + } + + @Override + public Path getParent() { + if (myNameElements.length == 0) { + throw new AssertionError(); + } + String[] parentElements = ArrayUtil.remove(myNameElements, myNameElements.length - 1, ArrayUtil.STRING_ARRAY_FACTORY); + return new UncompressedZipPath(myFileSystem, parentElements, myAbsolute); + } + + @Override + public int getNameCount() { + return myNameElements.length; + } + + @NotNull + @Override + public Path getName(int index) { + return new UncompressedZipPath(myFileSystem, new String[] {myNameElements[index]}, false); + } + + @NotNull + @Override + public Path subpath(int beginIndex, int endIndex) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean startsWith(@NotNull Path other) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean startsWith(@NotNull String other) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean endsWith(@NotNull Path other) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean endsWith(@NotNull String other) { + throw new UnsupportedOperationException(); + } + + @Override + public Path normalize() { + String separator = myFileSystem.getSeparator(); + String path = FileUtil.toCanonicalPath(StringUtil.join(myNameElements, separator), separator.charAt(0)); + return new UncompressedZipPath(myFileSystem, ArrayUtil.toStringArray(StringUtil.split(path, separator)), myAbsolute); + } + + @NotNull + @Override + public Path resolve(@NotNull Path other) { + if (other.isAbsolute()) return other; + String[] toAppend = ((UncompressedZipPath)other).myNameElements; + return new UncompressedZipPath(myFileSystem, ArrayUtil.mergeArrays(myNameElements,toAppend), myAbsolute); + } + + @NotNull + @Override + public Path resolve(@NotNull String other) { + String[] toAppend = ArrayUtil.toStringArray(StringUtil.split(other, myFileSystem.getSeparator())); + return new UncompressedZipPath(myFileSystem, ArrayUtil.mergeArrays(myNameElements,toAppend), myAbsolute); + } + + @NotNull + @Override + public Path resolveSibling(@NotNull Path other) { + return getParent().resolve(other); + } + + @NotNull + @Override + public Path resolveSibling(@NotNull String other) { + return getParent().resolve(other); + } + + @NotNull + @Override + public Path relativize(@NotNull Path other) { + throw new UnsupportedOperationException(); + } + + @NotNull + @Override + public URI toUri() { + throw new UnsupportedOperationException(); + } + + @NotNull + @Override + public Path toAbsolutePath() { + String separator = myFileSystem.getSeparator(); + String path = FileUtil.toCanonicalPath(StringUtil.join(myNameElements, separator), separator.charAt(0)); + return new UncompressedZipPath(myFileSystem, ArrayUtil.toStringArray(StringUtil.split(path, separator)), true); + } + + @NotNull + @Override + public Path toRealPath(@NotNull LinkOption... options) { + if (!myAbsolute) { + UncompressedZipPath absolutePath = (UncompressedZipPath)toAbsolutePath(); + return absolutePath.toRealPath(options); + } else { + return this; + } + } + + @NotNull + @Override + public File toFile() { + throw new UnsupportedOperationException(); + } + + @Override + public WatchKey register(WatchService watcher, WatchEvent.Kind[] events, WatchEvent.Modifier... modifiers) { + throw new UnsupportedOperationException(); + } + + @Override + public WatchKey register(WatchService watcher, WatchEvent.Kind... events) { + throw new UnsupportedOperationException(); + } + + @NotNull + @Override + public Iterator iterator() { + return Stream.of(myNameElements).map(n -> (Path)new UncompressedZipPath(myFileSystem, new String[] { n }, false)).iterator(); + } + + @Override + public int compareTo(Path other) { + return ArrayUtil.lexicographicCompare(myNameElements, ((UncompressedZipPath)other).myNameElements); + } + + @Override + public String toString() { + return StringUtil.join(myNameElements, myFileSystem.getSeparator()); + } +} diff --git a/platform/lang-impl/testSources/com/intellij/util/indexing/PersistentMapPacksTest.java b/platform/lang-impl/testSources/com/intellij/util/indexing/PersistentMapPacksTest.java new file mode 100644 index 000000000000..3109005ec7f3 --- /dev/null +++ b/platform/lang-impl/testSources/com/intellij/util/indexing/PersistentMapPacksTest.java @@ -0,0 +1,63 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing; + +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem; +import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider; +import com.intellij.util.io.EnumeratorStringDescriptor; +import com.intellij.util.io.PersistentHashMap; +import com.intellij.util.io.zip.JBZipEntry; +import com.intellij.util.io.zip.JBZipFile; +import junit.framework.TestCase; + +import java.io.File; +import java.io.IOException; +import java.util.zip.ZipEntry; + +public class PersistentMapPacksTest extends TestCase { + + public void testPersistentHashMapPack() throws IOException { + File dir = FileUtil.createTempDirectory("persistent-map", "packs"); + + EnumeratorStringDescriptor descriptor = EnumeratorStringDescriptor.INSTANCE; + + try (PersistentHashMap map1 = new PersistentHashMap<>(dir.toPath().resolve("map1"), descriptor, descriptor); + PersistentHashMap map2 = new PersistentHashMap<>(dir.toPath().resolve("map2"), descriptor, descriptor)) { + map1.put("XXX", "III"); + map1.put("YYY", "JJJ"); + + map2.put("IntelliJ", "IDEA"); + map2.put("IDEA", "IntelliJ"); + } + + File pack = new File(dir, "pack.zip"); + try (JBZipFile file = new JBZipFile(pack)) { + for (File f : dir.listFiles((__, name) -> name.startsWith("map"))) { + JBZipEntry entry = file.getOrCreateEntry(f.getName()); + entry.setMethod(ZipEntry.STORED); + entry.setDataFromFile(f); + } + } + + try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) { + try (PersistentHashMap map1 = new PersistentHashMap(fs.getPath("map1"), descriptor, descriptor) { + @Override + protected boolean isReadOnly() { + return true; + } + }; + PersistentHashMap map2 = new PersistentHashMap(fs.getPath("map2"), descriptor, descriptor) { + @Override + protected boolean isReadOnly() { + return true; + } + }) { + assertEquals("III", map1.get("XXX")); + assertEquals("JJJ", map1.get("YYY")); + + assertEquals("IDEA", map2.get("IntelliJ")); + assertEquals("IntelliJ", map2.get("IDEA")); + } + } + } +} diff --git a/platform/lang-impl/testSources/com/intellij/util/indexing/UncompressedZipTest.kt b/platform/lang-impl/testSources/com/intellij/util/indexing/UncompressedZipTest.kt new file mode 100644 index 000000000000..156c8da4c547 --- /dev/null +++ b/platform/lang-impl/testSources/com/intellij/util/indexing/UncompressedZipTest.kt @@ -0,0 +1,62 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing + +import com.intellij.openapi.util.io.FileUtil +import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem +import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider +import com.intellij.util.io.zip.JBZipFile +import junit.framework.TestCase + +import java.io.File +import java.nio.file.Files + +class UncompressedZipTest : TestCase() { + + @Throws(Exception::class) + override fun setUp() { + } + + @Throws(Exception::class) + override fun tearDown() { + } + + fun testFsStructure() { + val dir = FileUtil.createTempDirectory("zip0-fs-structure-dir", null) + + val file = File(dir, "a.zip") + val zip = JBZipFile(file) + val str = "Hello" + val helloBytes = str.toByteArray(Charsets.UTF_8) + zip.use { + zip.getOrCreateEntry("b.txt").data = helloBytes + zip.getOrCreateEntry("a/b.txt").data = helloBytes + zip.getOrCreateEntry("a/c.txt").data = helloBytes + } + + val fs = UncompressedZipFileSystem(file.toPath(), UncompressedZipFileSystemProvider()) + val file1 = fs.getPath("b.txt") + val file2 = fs.getPath("a", "b.txt") + val file3 = fs.getPath("a", "c.txt") + + assertTrue(Files.exists(file1)) + assertTrue(Files.exists(file2)) + assertTrue(Files.exists(file3)) + + assertTrue(helloBytes.contentEquals(Files.readAllBytes(file1))) + assertTrue(helloBytes.contentEquals(Files.readAllBytes(file2))) + assertTrue(helloBytes.contentEquals(Files.readAllBytes(file3))) + + val parent2 = file2.parent + val parent3 = file3.parent + assertTrue(Files.isSameFile(parent2, parent3)) + + val pp2 = parent2.parent + val pp3 = parent3.parent + val pp1 = file1.parent + assertTrue(Files.isSameFile(pp1, pp2)) + assertTrue(Files.isSameFile(pp1, pp3)) + assertTrue(Files.isSameFile(pp2, pp3)) + + fs.close() + } +} diff --git a/platform/platform-tests/testSrc/com/intellij/util/io/StringEnumeratorTest.java b/platform/platform-tests/testSrc/com/intellij/util/io/StringEnumeratorTest.java index b0c7d3a6cc86..4986aab70ba2 100644 --- a/platform/platform-tests/testSrc/com/intellij/util/io/StringEnumeratorTest.java +++ b/platform/platform-tests/testSrc/com/intellij/util/io/StringEnumeratorTest.java @@ -180,7 +180,7 @@ public class StringEnumeratorTest extends TestCase { private static final StringBuilder builder = new StringBuilder(100); private static final Random random = new Random(2_71828); - static String createRandomString() { + public static String createRandomString() { return createRandomString(random); } diff --git a/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java b/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java new file mode 100644 index 000000000000..b238a1577670 --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/MergedValueContainer.java @@ -0,0 +1,79 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.impl; + +import com.intellij.util.indexing.ValueContainer; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Arrays; +import java.util.List; + +public class MergedValueContainer extends ValueContainer { + private final List> myContainers; + private int mySize; + + @NotNull + public static ValueContainer merge(@NotNull ValueContainer container1, @NotNull ValueContainer container2) { + if (container1.size() == 0) return container2; + if (container2.size() == 0) return container1; + return new MergedValueContainer<>(Arrays.asList(container1, container2)); + } + + public MergedValueContainer(@NotNull List> containers) { + if (containers.isEmpty()) { + throw new IllegalArgumentException(); + } + myContainers = containers; + } + + @NotNull + @Override + public InvertedIndexValueIterator getValueIterator() { + return new InvertedIndexValueIterator() { + int myNextId = 1; + ValueIterator myCurrent = myContainers.get(0).getValueIterator(); + + @NotNull + @Override + public IntIterator getInputIdsIterator() { + return myCurrent.getInputIdsIterator(); + } + + @Nullable + @Override + public IntPredicate getValueAssociationPredicate() { + return myCurrent.getValueAssociationPredicate(); + } + + @Override + public Object getFileSetObject() { + return null; + } + + @Override + public boolean hasNext() { + while (true) { + if (myCurrent.hasNext()) return true; + if (myNextId < myContainers.size()) { + myCurrent = myContainers.get(myNextId++).getValueIterator(); + } else { + return false; + } + } + } + + @Override + public Value next() { + return myCurrent.next(); + } + }; + } + + @Override + public int size() { + if (mySize == 0) { + mySize = myContainers.stream().mapToInt(c -> c.size()).sum(); + } + return mySize; + } +} diff --git a/platform/util/src/com/intellij/util/indexing/impl/ReadOnlyIndexPack.java b/platform/util/src/com/intellij/util/indexing/impl/ReadOnlyIndexPack.java new file mode 100644 index 000000000000..7210ef6247af --- /dev/null +++ b/platform/util/src/com/intellij/util/indexing/impl/ReadOnlyIndexPack.java @@ -0,0 +1,75 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing.impl; + +import com.intellij.openapi.util.Computable; +import com.intellij.util.SmartList; +import com.intellij.util.indexing.InvertedIndex; +import com.intellij.util.indexing.StorageException; +import com.intellij.util.indexing.ValueContainer; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +public class ReadOnlyIndexPack> implements InvertedIndex { + @NotNull + private final List myIndexes; + + public ReadOnlyIndexPack(@NotNull List indexes) {myIndexes = indexes;} + + @NotNull + @Override + public ValueContainer getData(@NotNull K k) throws StorageException { + List> result = new SmartList<>(); + for (InvertedIndex index : myIndexes) { + ValueContainer currentData = index.getData(k); + if (currentData.size() != 0) { + result.add(currentData); + } + } + return result.isEmpty() ? new ValueContainerImpl<>() : new MergedValueContainer<>(result); + } + + @NotNull + @Override + public Computable update(int inputId, @Nullable Input content) { + throw new UnsupportedOperationException("index pack is read-only"); + } + + @Override + public void flush() throws StorageException { + List exceptions = new SmartList<>(); + for (InvertedIndex index : myIndexes) { + try { + index.flush(); + } + catch (StorageException e) { + exceptions.add(e); + } + } + if (!exceptions.isEmpty()) { + throw exceptions.get(0); + } + } + + @Override + public void clear() throws StorageException { + throw new UnsupportedOperationException("index pack is read-only"); + } + + @Override + public void dispose() { + List exceptions = new SmartList<>(); + for (InvertedIndex index : myIndexes) { + try { + index.dispose(); + } + catch (RuntimeException e) { + exceptions.add(e); + } + } + if (!exceptions.isEmpty()) { + throw exceptions.get(0); + } + } +} diff --git a/platform/util/src/com/intellij/util/io/zip/JBZipEntry.java b/platform/util/src/com/intellij/util/io/zip/JBZipEntry.java index b9bbe4971199..afef05a11434 100644 --- a/platform/util/src/com/intellij/util/io/zip/JBZipEntry.java +++ b/platform/util/src/com/intellij/util/io/zip/JBZipEntry.java @@ -457,7 +457,7 @@ public class JBZipEntry implements Cloneable { } } - private long calcDataOffset() throws IOException { + public long calcDataOffset() throws IOException { long offset = getHeaderOffset(); myFile.archive.seek(offset + JBZipFile.LFH_OFFSET_FOR_FILENAME_LENGTH); byte[] b = new byte[JBZipFile.WORD];