mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-09-27 10:03:11 +07:00
zipped index
GitOrigin-RevId: 41fffd09abf4659708fc7ca4029dbcc9245feeec
This commit is contained in:
Dmitry Batkovich
committed by
intellij-monorepo-bot
intellij-monorepo-bot
parent
f587b42bf7
commit
fd32720544
@@ -0,0 +1,305 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing;
|
||||
|
||||
import com.intellij.openapi.diagnostic.Logger;
|
||||
import com.intellij.openapi.util.Pair;
|
||||
import com.intellij.openapi.util.io.FileUtil;
|
||||
import com.intellij.openapi.util.text.StringUtil;
|
||||
import com.intellij.testFramework.PlatformTestUtil;
|
||||
import com.intellij.util.containers.MultiMap;
|
||||
import com.intellij.util.indexing.impl.MapIndexStorage;
|
||||
import com.intellij.util.indexing.impl.MapReduceIndex;
|
||||
import com.intellij.util.indexing.impl.ReadOnlyIndexPack;
|
||||
import com.intellij.util.indexing.impl.forward.KeyCollectionForwardIndexAccessor;
|
||||
import com.intellij.util.indexing.impl.forward.PersistentMapBasedForwardIndex;
|
||||
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem;
|
||||
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider;
|
||||
import com.intellij.util.io.DataExternalizer;
|
||||
import com.intellij.util.io.EnumeratorStringDescriptor;
|
||||
import com.intellij.util.io.KeyDescriptor;
|
||||
import com.intellij.util.io.StringEnumeratorTest;
|
||||
import com.intellij.util.io.zip.JBZipFile;
|
||||
import gnu.trove.TIntHashSet;
|
||||
import junit.framework.AssertionFailedError;
|
||||
import junit.framework.TestCase;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class IndexPackTest extends TestCase {
|
||||
private static final Logger LOG = Logger.getInstance(IndexPackTest.class);
|
||||
|
||||
public void testSimpleIndexPack() throws IOException, StorageException {
|
||||
File dir = FileUtil.createTempDirectory("persistent-map", "packs");
|
||||
|
||||
MapReduceIndex<String, String, String> m1 = createSplitStringIndex(dir.toPath().resolve("index1").resolve("index"), false);
|
||||
MapReduceIndex<String, String, String> m2 = createSplitStringIndex(dir.toPath().resolve("index2").resolve("index"), false);
|
||||
|
||||
assertTrue(m1.update(1, "key/value").compute());
|
||||
assertTrue(m1.update(2, "key2/value2").compute());
|
||||
assertTrue(m1.update(3, "key/value2").compute());
|
||||
|
||||
assertTrue(m2.update(1, "key/value").compute());
|
||||
assertTrue(m2.update(2, "key2/value2").compute());
|
||||
assertTrue(m2.update(3, "key/value2").compute());
|
||||
|
||||
m1.dispose();
|
||||
m2.dispose();
|
||||
|
||||
File pack = new File(dir, "pack.zip");
|
||||
try (JBZipFile file = new JBZipFile(pack)) {
|
||||
for (String index : Arrays.asList("index1", "index2")) {
|
||||
for (Path path : Files.newDirectoryStream(dir.toPath().resolve(index))) {
|
||||
file.getOrCreateEntry(index + "/" + path.getFileName().toString()).setDataFromFile(path.toFile());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MultiMap<String, Pair<String, Integer>> indexMap1 = new MultiMap<>();
|
||||
indexMap1.putValue("key", Pair.create("value", 1));
|
||||
indexMap1.putValue("key", Pair.create("value2", 3));
|
||||
indexMap1.putValue("key2", Pair.create("value2", 2));
|
||||
|
||||
MultiMap<String, Pair<String, Integer>> indexMap2 = new MultiMap<>();
|
||||
indexMap2.putValue("key", Pair.create("value", 1));
|
||||
indexMap2.putValue("key", Pair.create("value2", 3));
|
||||
indexMap2.putValue("key2", Pair.create("value2", 2));
|
||||
|
||||
try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) {
|
||||
MapReduceIndex<String, String, String> mm1 = createSplitStringIndex(fs.getPath("index1").resolve("index"), true);
|
||||
MapReduceIndex<String, String, String> mm2 = createSplitStringIndex(fs.getPath("index2").resolve("index"), true);
|
||||
|
||||
for (String key : new ArrayList<>(indexMap1.keySet())) {
|
||||
mm1.getData(key).forEach((id, value) -> {
|
||||
indexMap1.remove(key, Pair.create(value, id));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
for (String key : new ArrayList<>(indexMap2.keySet())) {
|
||||
mm2.getData(key).forEach((id, value) -> {
|
||||
indexMap2.remove(key, Pair.create(value, id));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
mm1.dispose();
|
||||
mm2.dispose();
|
||||
}
|
||||
|
||||
assertTrue(indexMap1.isEmpty());
|
||||
assertTrue(indexMap2.isEmpty());
|
||||
}
|
||||
|
||||
private static final int sampleCount = 1000;
|
||||
|
||||
public void testIndexAccessPerformance() throws IOException {
|
||||
File dir = FileUtil.createTempDirectory("persistent-map", "packs");
|
||||
|
||||
TIntHashSet keys = new TIntHashSet();
|
||||
InvertedIndex<String, String, String> index = createStringLengthIndex(dir.toPath().resolve("index"), false);
|
||||
for (int i = 1; i <= sampleCount; ++i) {
|
||||
final String string = generateString();
|
||||
keys.add(string.length());
|
||||
index.update(i, string).compute();
|
||||
}
|
||||
index.dispose();
|
||||
|
||||
PlatformTestUtil.startPerformanceTest("read", 2000, () -> {
|
||||
InvertedIndex<String, String, String> readIndex = createStringLengthIndex(dir.toPath().resolve("index"), true);
|
||||
LongAdder recordCount = new LongAdder();
|
||||
for (int key : keys.toArray()) {
|
||||
readIndex.getData(String.valueOf(key)).forEach((id, value) -> {
|
||||
recordCount.increment();
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assertEquals(sampleCount, recordCount.intValue());
|
||||
readIndex.dispose();
|
||||
}).attempts(5).ioBound().assertTiming();
|
||||
}
|
||||
|
||||
public void testTrivialPackAccessPerformance() throws IOException {
|
||||
File dir = FileUtil.createTempDirectory("persistent-map", "packs");
|
||||
|
||||
TIntHashSet keys = new TIntHashSet();
|
||||
InvertedIndex<String, String, String> index = createStringLengthIndex(dir.toPath().resolve("index").resolve("index"), false);
|
||||
for (int i = 1; i <= sampleCount; ++i) {
|
||||
final String string = generateString();
|
||||
keys.add(string.length());
|
||||
index.update(i, string).compute();
|
||||
}
|
||||
index.dispose();
|
||||
|
||||
File pack = new File(dir, "pack.zip");
|
||||
try (JBZipFile file = new JBZipFile(pack)) {
|
||||
for (Path path : Files.newDirectoryStream(dir.toPath().resolve("index"))) {
|
||||
file.getOrCreateEntry("index/" + path.getFileName().toString()).setDataFromFile(path.toFile());
|
||||
}
|
||||
}
|
||||
|
||||
try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) {
|
||||
PlatformTestUtil.startPerformanceTest("read", 2000, () -> {
|
||||
InvertedIndex<String, String, String> readIndex = createStringLengthIndex(fs.getPath("index", "index"), true);
|
||||
LongAdder recordCount = new LongAdder();
|
||||
for (int key : keys.toArray()) {
|
||||
readIndex.getData(String.valueOf(key)).forEach((id, value) -> {
|
||||
recordCount.increment();
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assertEquals(sampleCount, recordCount.intValue());
|
||||
readIndex.dispose();
|
||||
}).attempts(5).ioBound().assertTiming();
|
||||
}
|
||||
}
|
||||
|
||||
public void test1000PackAccessPerformance() throws IOException {
|
||||
File dir = FileUtil.createTempDirectory("persistent-map", "packs");
|
||||
|
||||
TIntHashSet keys = new TIntHashSet();
|
||||
int packSize = 1000;
|
||||
List<InvertedIndex<String, String, String>> indexes = generateIndexNames(packSize)
|
||||
.map(name -> createStringLengthIndex(dir.toPath().resolve(name).resolve("index"), false))
|
||||
.collect(Collectors.toList());
|
||||
for (int i = 1; i <= sampleCount; ++i) {
|
||||
final String string = generateString();
|
||||
keys.add(string.length());
|
||||
indexes.get(Math.abs(string.hashCode() % packSize)).update(i, string).compute();
|
||||
}
|
||||
indexes.forEach(InvertedIndex::dispose);
|
||||
|
||||
File pack = new File(dir, "pack.zip");
|
||||
try (JBZipFile file = new JBZipFile(pack)) {
|
||||
generateIndexNames(packSize).map(name -> dir.toPath().resolve(name)).forEach(p -> {
|
||||
try {
|
||||
for (Path path : Files.newDirectoryStream(p)) {
|
||||
file.getOrCreateEntry(path.getParent().getFileName().toString() + "/" + path.getFileName().toString()).setDataFromFile(path.toFile());
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
LOG.error(e);
|
||||
throw new AssertionFailedError(e.getMessage());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) {
|
||||
PlatformTestUtil.startPerformanceTest("read", 4000, () -> {
|
||||
|
||||
ReadOnlyIndexPack<String, String, String, ?> indexPack = new ReadOnlyIndexPack<>(generateIndexNames(packSize)
|
||||
.map(name -> createStringLengthIndex(fs.getPath(name, "index"), true))
|
||||
.collect(Collectors.toList()));
|
||||
|
||||
LongAdder recordCount = new LongAdder();
|
||||
for (int key : keys.toArray()) {
|
||||
indexPack.getData(String.valueOf(key)).forEach((id, value) -> {
|
||||
recordCount.increment();
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assertEquals(sampleCount, recordCount.intValue());
|
||||
indexPack.dispose();
|
||||
}).attempts(5).ioBound().assertTiming();
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static Stream<String> generateIndexNames(int packSize) {
|
||||
return IntStream
|
||||
.rangeClosed(1, packSize)
|
||||
.mapToObj(idx -> "index" + idx);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static String generateString() {
|
||||
return StringUtil.repeat(StringEnumeratorTest.createRandomString(), 30);
|
||||
}
|
||||
|
||||
private static MapReduceIndex<String, String, String> createSplitStringIndex(@NotNull Path path, boolean readOnly) throws IOException {
|
||||
return createIndex(path, readOnly, new DataIndexer<String, String, String>() {
|
||||
@NotNull
|
||||
@Override
|
||||
public Map<String, String> map(@NotNull String inputData) {
|
||||
String[] split = inputData.split("/");
|
||||
return Collections.singletonMap(split[0], split[1]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private static InvertedIndex<String, String, String> createStringLengthIndex(@NotNull Path path, boolean readOnly) {
|
||||
try {
|
||||
return createIndex(path, readOnly, new DataIndexer<String, String, String>() {
|
||||
@NotNull
|
||||
@Override
|
||||
public Map<String, String> map(@NotNull String inputData) {
|
||||
return Collections.singletonMap(String.valueOf(inputData.length()), inputData + "_value");
|
||||
}
|
||||
});
|
||||
}
|
||||
catch (IOException e) {
|
||||
LOG.error(e);
|
||||
throw new AssertionFailedError(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static MapReduceIndex<String, String, String> createIndex(@NotNull Path path, boolean readOnly, DataIndexer<String, String, String> indexer) throws IOException {
|
||||
IndexExtension<String, String, String> extension = new IndexExtension<String, String, String>() {
|
||||
@NotNull
|
||||
@Override
|
||||
public IndexId<String, String> getName() {
|
||||
return IndexId.create("AnIndex");
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public DataIndexer<String, String, String> getIndexer() {
|
||||
return indexer;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public KeyDescriptor<String> getKeyDescriptor() {
|
||||
return EnumeratorStringDescriptor.INSTANCE;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public DataExternalizer<String> getValueExternalizer() {
|
||||
return EnumeratorStringDescriptor.INSTANCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getVersion() {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
return new MapReduceIndex<String, String, String>(extension, new MapIndexStorage<String, String>(path.getParent().resolve(path.getFileName() + ".storage"), EnumeratorStringDescriptor.INSTANCE,
|
||||
EnumeratorStringDescriptor.INSTANCE, 1024, false, true, readOnly, null) {
|
||||
@Override
|
||||
protected void checkCanceled() {
|
||||
|
||||
}
|
||||
}, new PersistentMapBasedForwardIndex(path.getParent().resolve(path.getFileName() + ".forward"), readOnly), new KeyCollectionForwardIndexAccessor<>(extension)) {
|
||||
@Override
|
||||
public void checkCanceled() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void requestRebuild(@NotNull Throwable e) {
|
||||
e.printStackTrace();
|
||||
fail();
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,8 @@ import com.intellij.psi.util.PsiUtilCore;
|
||||
import com.intellij.util.BitUtil;
|
||||
import com.intellij.util.Consumer;
|
||||
import com.intellij.util.ThreeState;
|
||||
import com.intellij.util.indexing.DumbModeAccessType;
|
||||
import com.intellij.util.indexing.FileBasedIndex;
|
||||
import org.jetbrains.annotations.Contract;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
@@ -332,14 +334,23 @@ public class TargetElementUtil {
|
||||
PsiReference ref = findReference(editor, offset);
|
||||
if (ref == null) return null;
|
||||
|
||||
final Language language = ref.getElement().getLanguage();
|
||||
TargetElementEvaluator evaluator = TargetElementUtilBase.TARGET_ELEMENT_EVALUATOR.forLanguage(language);
|
||||
if (evaluator != null) {
|
||||
final PsiElement element = evaluator.getElementByReference(ref, flags);
|
||||
if (element != null) return element;
|
||||
}
|
||||
Project project = editor.getProject();
|
||||
if (project == null) return null;
|
||||
PsiElement[] result = {null};
|
||||
FileBasedIndex.getInstance().ignoreDumbMode(() -> {
|
||||
final Language language = ref.getElement().getLanguage();
|
||||
TargetElementEvaluator evaluator = TargetElementUtilBase.TARGET_ELEMENT_EVALUATOR.forLanguage(language);
|
||||
if (evaluator != null) {
|
||||
final PsiElement element = evaluator.getElementByReference(ref, flags);
|
||||
if (element != null) {
|
||||
result[0] = element;
|
||||
return;
|
||||
}
|
||||
}
|
||||
result[0] = ref.resolve();
|
||||
}, project, DumbModeAccessType.RELIABLE_DATA_ONLY);
|
||||
|
||||
return ref.resolve();
|
||||
return result[0];
|
||||
}
|
||||
|
||||
@NotNull
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.zipFs;
|
||||
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.MappedByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.FileLock;
|
||||
import java.nio.channels.ReadableByteChannel;
|
||||
import java.nio.channels.WritableByteChannel;
|
||||
|
||||
public class FileBlockReadOnlyFileChannel extends FileChannel {
|
||||
@NotNull
|
||||
private final FileChannel myUnderlying;
|
||||
private final long myStartOffset;
|
||||
private final long myEndOffset;
|
||||
private final long mySize;
|
||||
|
||||
// TODO duplicates?
|
||||
private volatile long myGlobalPosition;
|
||||
private volatile long myLocalPosition;
|
||||
|
||||
public FileBlockReadOnlyFileChannel(@NotNull FileChannel underlying,
|
||||
long startOffset,
|
||||
long size) {
|
||||
myUnderlying = underlying;
|
||||
|
||||
myStartOffset = startOffset;
|
||||
myEndOffset = startOffset + size;
|
||||
mySize = size;
|
||||
|
||||
myGlobalPosition = myStartOffset;
|
||||
myLocalPosition = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(ByteBuffer dst) throws IOException {
|
||||
int read = read(dst, myLocalPosition);
|
||||
if (read != -1 && read != 0) {
|
||||
position(myLocalPosition + read);
|
||||
}
|
||||
return read;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long read(ByteBuffer[] dsts, int offset, int length) throws IOException {
|
||||
throw new UnsupportedOperationException("Implement it!");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int write(ByteBuffer src) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long write(ByteBuffer[] srcs, int offset, int length) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long position() {
|
||||
return myGlobalPosition - myStartOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileChannel position(long newPosition) {
|
||||
myGlobalPosition = myStartOffset + newPosition;
|
||||
myLocalPosition = newPosition;
|
||||
assert myGlobalPosition <= myEndOffset;
|
||||
assert myLocalPosition <= mySize;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size() {
|
||||
return mySize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileChannel truncate(long size){
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void force(boolean metaData) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
@Override
|
||||
public long transferTo(long position, long count, WritableByteChannel target) {
|
||||
throw new UnsupportedOperationException("Implement it!");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long transferFrom(ReadableByteChannel src, long position, long count) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(ByteBuffer dst, long position) throws IOException {
|
||||
long globalPosition = myStartOffset + position;
|
||||
if (position >= mySize) {
|
||||
return -1;
|
||||
}
|
||||
return myUnderlying.read(dst, globalPosition);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int write(ByteBuffer src, long position) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public MappedByteBuffer map(MapMode mode, long position, long size) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileLock lock(long position, long size, boolean shared) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileLock tryLock(long position, long size, boolean shared) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void implCloseChannel() {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.zipFs;
|
||||
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
|
||||
public class UncompressedZipEntryFileAttributes implements BasicFileAttributes {
|
||||
private final UncompressedZipPath myPath;
|
||||
private final UncompressedZipFileSystem.ZipTreeNode myNode;
|
||||
|
||||
public UncompressedZipEntryFileAttributes(@NotNull UncompressedZipPath path) throws IOException {
|
||||
myPath = path;
|
||||
myNode = UncompressedZipFileSystemProvider.find(path);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileTime lastModifiedTime() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileTime lastAccessTime() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileTime creationTime() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isRegularFile() {
|
||||
return !isDirectory();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDirectory() {
|
||||
return myNode.isDirectory();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSymbolicLink() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isOther() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size() {
|
||||
return myNode.getEntry().getCompressedSize();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object fileKey() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.zipFs;
|
||||
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileStore;
|
||||
import java.nio.file.attribute.FileAttributeView;
|
||||
import java.nio.file.attribute.FileStoreAttributeView;
|
||||
|
||||
class UncompressedZipFileStore extends FileStore {
|
||||
@NotNull
|
||||
private final UncompressedZipFileSystem mySystem;
|
||||
|
||||
UncompressedZipFileStore(@NotNull UncompressedZipFileSystem system) {
|
||||
mySystem = system;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "zip0";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return "zip0";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isReadOnly() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalSpace() throws IOException {
|
||||
return mySystem.getChannel().size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getUsableSpace() throws IOException {
|
||||
return mySystem.getChannel().size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getUnallocatedSpace() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supportsFileAttributeView(String name) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getAttribute(String attribute) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.zipFs;
|
||||
|
||||
import com.intellij.openapi.util.text.StringUtil;
|
||||
import com.intellij.util.ArrayUtil;
|
||||
import com.intellij.util.containers.ContainerUtil;
|
||||
import com.intellij.util.io.zip.JBZipEntry;
|
||||
import com.intellij.util.io.zip.JBZipFile;
|
||||
import gnu.trove.THashMap;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.UserPrincipalLookupService;
|
||||
import java.nio.file.spi.FileSystemProvider;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
public class UncompressedZipFileSystem extends FileSystem {
|
||||
@NotNull
|
||||
private final JBZipFile myUncompressedZip;
|
||||
@NotNull
|
||||
private final FileChannel myChannel;
|
||||
@NotNull
|
||||
private final Path myUncompressedZipPath;
|
||||
private final UncompressedZipFileSystemProvider myProvider;
|
||||
private final ZipTreeNode myRoot = new ZipTreeNode();
|
||||
// probably should be eliminated
|
||||
private final Set<FileChannel> myOpenFiles = Collections.newSetFromMap(new ConcurrentHashMap<>());
|
||||
|
||||
public UncompressedZipFileSystem(@NotNull Path uncompressedZip, @NotNull UncompressedZipFileSystemProvider provider) throws IOException {
|
||||
myUncompressedZipPath = uncompressedZip;
|
||||
myProvider = provider;
|
||||
assert uncompressedZip.getFileSystem() == FileSystems.getDefault();
|
||||
myUncompressedZip = new JBZipFile(uncompressedZip.toFile());
|
||||
myChannel = FileChannel.open(uncompressedZip, StandardOpenOption.READ);
|
||||
buildTree();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
FileChannel openChannel(@NotNull JBZipEntry entry) throws IOException {
|
||||
FileBlockReadOnlyFileChannel channel = new FileBlockReadOnlyFileChannel(myChannel, entry.calcDataOffset(), entry.getSize()) {
|
||||
@Override
|
||||
protected void implCloseChannel() {
|
||||
myOpenFiles.remove(this);
|
||||
}
|
||||
};
|
||||
myOpenFiles.add(channel);
|
||||
return channel;
|
||||
}
|
||||
|
||||
private void buildTree() {
|
||||
for (JBZipEntry entry : myUncompressedZip.getEntries()) {
|
||||
List<String> names = StringUtil.split(entry.getName(), getSeparator());
|
||||
ZipTreeNode current = myRoot;
|
||||
for (int i = 0; i < names.size(); i++) {
|
||||
if (i == names.size() - 1) {
|
||||
current.createEntryChild(names.get(i), entry);
|
||||
} else {
|
||||
current = current.createDirChild(names.get(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileSystemProvider provider() {
|
||||
return myProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
try {
|
||||
myChannel.close();
|
||||
} finally {
|
||||
myUncompressedZip.close();
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
Path getUncompressedZipPath() {
|
||||
return myUncompressedZipPath;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
FileChannel getChannel() {
|
||||
return myChannel;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isOpen() {
|
||||
return myChannel.isOpen();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isReadOnly() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSeparator() {
|
||||
return "/";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterable<Path> getRootDirectories() {
|
||||
return Collections.singleton(new UncompressedZipPath(this, ArrayUtil.EMPTY_STRING_ARRAY, true));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterable<FileStore> getFileStores() {
|
||||
return Collections.singleton(new UncompressedZipFileStore(this));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> supportedFileAttributeViews() {
|
||||
return Collections.singleton("basic");
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path getPath(@NotNull String first, @NotNull String... more) {
|
||||
// should it be absolute
|
||||
String[] nameElements = ArrayUtil.toStringArray(ContainerUtil.concat(Collections.singletonList(first), Arrays.asList(more)));
|
||||
return new UncompressedZipPath(this, nameElements, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PathMatcher getPathMatcher(String syntaxAndPattern) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public UserPrincipalLookupService getUserPrincipalLookupService() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public WatchService newWatchService() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
ZipTreeNode getRoot() {
|
||||
return myRoot;
|
||||
}
|
||||
|
||||
static class ZipTreeNode {
|
||||
@Nullable
|
||||
private final Map<String, ZipTreeNode> myChildren;
|
||||
@Nullable
|
||||
private final JBZipEntry myEntry;
|
||||
|
||||
public boolean isDirectory() {
|
||||
return myChildren != null;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
JBZipEntry getEntry() {
|
||||
assert myEntry != null;
|
||||
return myEntry;
|
||||
}
|
||||
|
||||
Set<String> getChildNames() {
|
||||
assert myChildren != null;
|
||||
return myChildren.keySet();
|
||||
}
|
||||
|
||||
ZipTreeNode(@NotNull JBZipEntry entry) {
|
||||
myEntry = entry;
|
||||
myChildren = null;
|
||||
}
|
||||
|
||||
ZipTreeNode() {
|
||||
myEntry = null;
|
||||
myChildren = new THashMap<>();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
ZipTreeNode getChild(@NotNull String childName) {
|
||||
assert myChildren != null;
|
||||
return myChildren.get(childName);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
ZipTreeNode createDirChild(@NotNull String childName) {
|
||||
assert myChildren != null;
|
||||
return myChildren.computeIfAbsent(childName, __ -> new ZipTreeNode());
|
||||
}
|
||||
|
||||
void createEntryChild(@NotNull String childName, @NotNull JBZipEntry entry) {
|
||||
assert myChildren != null;
|
||||
ZipTreeNode previous = myChildren.put(childName, new ZipTreeNode(entry));
|
||||
assert previous == null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.zipFs;
|
||||
|
||||
import com.intellij.util.ArrayUtil;
|
||||
import com.intellij.util.io.zip.JBZipEntry;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.nio.file.attribute.FileAttribute;
|
||||
import java.nio.file.attribute.FileAttributeView;
|
||||
import java.nio.file.spi.FileSystemProvider;
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public class UncompressedZipFileSystemProvider extends FileSystemProvider {
|
||||
@Override
|
||||
public String getScheme() {
|
||||
return "zip0";
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileSystem newFileSystem(URI uri, Map<String, ?> env) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileSystem getFileSystem(URI uri) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path getPath(@NotNull URI uri) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
|
||||
return newFileChannel(path, options, attrs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
|
||||
UncompressedZipFileSystem.ZipTreeNode node = find(path);
|
||||
if (node.isDirectory()) {
|
||||
throw new IllegalArgumentException(path.toString());
|
||||
}
|
||||
JBZipEntry entry = node.getEntry();
|
||||
return ((UncompressedZipFileSystem)path.getFileSystem()).openChannel(entry);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException {
|
||||
UncompressedZipFileSystem.ZipTreeNode element = find(dir);
|
||||
if (!element.isDirectory()) {
|
||||
throw new NotDirectoryException(dir.toString());
|
||||
}
|
||||
return new DirectoryStream<Path>() {
|
||||
@Override
|
||||
public Iterator<Path> iterator() {
|
||||
return element.getChildNames().stream().map(n -> dir.resolve(n)).filter(p -> {
|
||||
try {
|
||||
return filter.accept(p);
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}).iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() { }
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createDirectory(Path dir, FileAttribute<?>... attrs) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(Path path) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(Path source, Path target, CopyOption... options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void move(Path source, Path target, CopyOption... options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSameFile(Path path1, Path path2) throws IOException {
|
||||
if (path1.getFileSystem().provider() != path2.getFileSystem().provider()) {
|
||||
return false;
|
||||
}
|
||||
UncompressedZipFileSystem system1 = (UncompressedZipFileSystem)path1.getFileSystem();
|
||||
UncompressedZipFileSystem system2 = (UncompressedZipFileSystem)path2.getFileSystem();
|
||||
if (!Files.isSameFile(system1.getUncompressedZipPath(), system2.getUncompressedZipPath())) {
|
||||
return false;
|
||||
}
|
||||
UncompressedZipPath absolutePath1 = (UncompressedZipPath)path1.toAbsolutePath();
|
||||
UncompressedZipPath absolutePath2 = (UncompressedZipPath)path2.toAbsolutePath();
|
||||
return Arrays.equals(absolutePath1.getNameElements(), absolutePath2.getNameElements());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isHidden(Path path) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileStore getFileStore(Path path) {
|
||||
return new UncompressedZipFileStore(((UncompressedZipFileSystem)path.getFileSystem()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkAccess(Path path, AccessMode... modes) throws IOException {
|
||||
if (ArrayUtil.contains(AccessMode.WRITE, path) || ArrayUtil.contains(AccessMode.EXECUTE, path)) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
find(path);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
|
||||
if (type == BasicFileAttributes.class) {
|
||||
return (A)new UncompressedZipEntryFileAttributes(((UncompressedZipPath)path));
|
||||
}
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAttribute(Path path, String attribute, Object value, LinkOption... options) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
static UncompressedZipFileSystem.ZipTreeNode find(@NotNull Path dir) throws IOException {
|
||||
Path absoluteDir = dir.toAbsolutePath();
|
||||
String[] elements = ((UncompressedZipPath)absoluteDir).getNameElements();
|
||||
UncompressedZipFileSystem.ZipTreeNode currentElement = ((UncompressedZipFileSystem)absoluteDir.getFileSystem()).getRoot();
|
||||
for (String element : elements) {
|
||||
currentElement = currentElement.getChild(element);
|
||||
if (currentElement == null) {
|
||||
throw new NotDirectoryException(dir.toString());
|
||||
}
|
||||
}
|
||||
return currentElement;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.zipFs;
|
||||
|
||||
import com.intellij.openapi.util.io.FileUtil;
|
||||
import com.intellij.openapi.util.text.StringUtil;
|
||||
import com.intellij.util.ArrayUtil;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URI;
|
||||
import java.nio.file.*;
|
||||
import java.util.Iterator;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class UncompressedZipPath implements Path {
|
||||
@NotNull
|
||||
private final UncompressedZipFileSystem myFileSystem;
|
||||
@NotNull
|
||||
private final String[] myNameElements;
|
||||
private final boolean myAbsolute;
|
||||
|
||||
public UncompressedZipPath(@NotNull UncompressedZipFileSystem system,
|
||||
@NotNull String[] elements,
|
||||
boolean absolute) {
|
||||
myFileSystem = system;
|
||||
myNameElements = elements;
|
||||
myAbsolute = absolute;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
String[] getNameElements() {
|
||||
return myNameElements;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public FileSystem getFileSystem() {
|
||||
return myFileSystem;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAbsolute() {
|
||||
return myAbsolute;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getRoot() {
|
||||
return isAbsolute() ? new UncompressedZipPath(myFileSystem, ArrayUtil.EMPTY_STRING_ARRAY, true) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getFileName() {
|
||||
return getName(getNameCount() - 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getParent() {
|
||||
if (myNameElements.length == 0) {
|
||||
throw new AssertionError();
|
||||
}
|
||||
String[] parentElements = ArrayUtil.remove(myNameElements, myNameElements.length - 1, ArrayUtil.STRING_ARRAY_FACTORY);
|
||||
return new UncompressedZipPath(myFileSystem, parentElements, myAbsolute);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNameCount() {
|
||||
return myNameElements.length;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path getName(int index) {
|
||||
return new UncompressedZipPath(myFileSystem, new String[] {myNameElements[index]}, false);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path subpath(int beginIndex, int endIndex) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startsWith(@NotNull Path other) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startsWith(@NotNull String other) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endsWith(@NotNull Path other) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endsWith(@NotNull String other) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path normalize() {
|
||||
String separator = myFileSystem.getSeparator();
|
||||
String path = FileUtil.toCanonicalPath(StringUtil.join(myNameElements, separator), separator.charAt(0));
|
||||
return new UncompressedZipPath(myFileSystem, ArrayUtil.toStringArray(StringUtil.split(path, separator)), myAbsolute);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path resolve(@NotNull Path other) {
|
||||
if (other.isAbsolute()) return other;
|
||||
String[] toAppend = ((UncompressedZipPath)other).myNameElements;
|
||||
return new UncompressedZipPath(myFileSystem, ArrayUtil.mergeArrays(myNameElements,toAppend), myAbsolute);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path resolve(@NotNull String other) {
|
||||
String[] toAppend = ArrayUtil.toStringArray(StringUtil.split(other, myFileSystem.getSeparator()));
|
||||
return new UncompressedZipPath(myFileSystem, ArrayUtil.mergeArrays(myNameElements,toAppend), myAbsolute);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path resolveSibling(@NotNull Path other) {
|
||||
return getParent().resolve(other);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path resolveSibling(@NotNull String other) {
|
||||
return getParent().resolve(other);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path relativize(@NotNull Path other) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public URI toUri() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path toAbsolutePath() {
|
||||
String separator = myFileSystem.getSeparator();
|
||||
String path = FileUtil.toCanonicalPath(StringUtil.join(myNameElements, separator), separator.charAt(0));
|
||||
return new UncompressedZipPath(myFileSystem, ArrayUtil.toStringArray(StringUtil.split(path, separator)), true);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Path toRealPath(@NotNull LinkOption... options) {
|
||||
if (!myAbsolute) {
|
||||
UncompressedZipPath absolutePath = (UncompressedZipPath)toAbsolutePath();
|
||||
return absolutePath.toRealPath(options);
|
||||
} else {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public File toFile() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public WatchKey register(WatchService watcher, WatchEvent.Kind<?>... events) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Iterator<Path> iterator() {
|
||||
return Stream.of(myNameElements).map(n -> (Path)new UncompressedZipPath(myFileSystem, new String[] { n }, false)).iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Path other) {
|
||||
return ArrayUtil.lexicographicCompare(myNameElements, ((UncompressedZipPath)other).myNameElements);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return StringUtil.join(myNameElements, myFileSystem.getSeparator());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing;
|
||||
|
||||
import com.intellij.openapi.util.io.FileUtil;
|
||||
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem;
|
||||
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider;
|
||||
import com.intellij.util.io.EnumeratorStringDescriptor;
|
||||
import com.intellij.util.io.PersistentHashMap;
|
||||
import com.intellij.util.io.zip.JBZipEntry;
|
||||
import com.intellij.util.io.zip.JBZipFile;
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.zip.ZipEntry;
|
||||
|
||||
public class PersistentMapPacksTest extends TestCase {
|
||||
|
||||
public void testPersistentHashMapPack() throws IOException {
|
||||
File dir = FileUtil.createTempDirectory("persistent-map", "packs");
|
||||
|
||||
EnumeratorStringDescriptor descriptor = EnumeratorStringDescriptor.INSTANCE;
|
||||
|
||||
try (PersistentHashMap<String, String> map1 = new PersistentHashMap<>(dir.toPath().resolve("map1"), descriptor, descriptor);
|
||||
PersistentHashMap<String, String> map2 = new PersistentHashMap<>(dir.toPath().resolve("map2"), descriptor, descriptor)) {
|
||||
map1.put("XXX", "III");
|
||||
map1.put("YYY", "JJJ");
|
||||
|
||||
map2.put("IntelliJ", "IDEA");
|
||||
map2.put("IDEA", "IntelliJ");
|
||||
}
|
||||
|
||||
File pack = new File(dir, "pack.zip");
|
||||
try (JBZipFile file = new JBZipFile(pack)) {
|
||||
for (File f : dir.listFiles((__, name) -> name.startsWith("map"))) {
|
||||
JBZipEntry entry = file.getOrCreateEntry(f.getName());
|
||||
entry.setMethod(ZipEntry.STORED);
|
||||
entry.setDataFromFile(f);
|
||||
}
|
||||
}
|
||||
|
||||
try (UncompressedZipFileSystem fs = new UncompressedZipFileSystem(pack.toPath(), new UncompressedZipFileSystemProvider())) {
|
||||
try (PersistentHashMap<String, String> map1 = new PersistentHashMap<String, String>(fs.getPath("map1"), descriptor, descriptor) {
|
||||
@Override
|
||||
protected boolean isReadOnly() {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
PersistentHashMap<String, String> map2 = new PersistentHashMap<String, String>(fs.getPath("map2"), descriptor, descriptor) {
|
||||
@Override
|
||||
protected boolean isReadOnly() {
|
||||
return true;
|
||||
}
|
||||
}) {
|
||||
assertEquals("III", map1.get("XXX"));
|
||||
assertEquals("JJJ", map1.get("YYY"));
|
||||
|
||||
assertEquals("IDEA", map2.get("IntelliJ"));
|
||||
assertEquals("IntelliJ", map2.get("IDEA"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing
|
||||
|
||||
import com.intellij.openapi.util.io.FileUtil
|
||||
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystem
|
||||
import com.intellij.util.indexing.zipFs.UncompressedZipFileSystemProvider
|
||||
import com.intellij.util.io.zip.JBZipFile
|
||||
import junit.framework.TestCase
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Files
|
||||
|
||||
class UncompressedZipTest : TestCase() {
|
||||
|
||||
@Throws(Exception::class)
|
||||
override fun setUp() {
|
||||
}
|
||||
|
||||
@Throws(Exception::class)
|
||||
override fun tearDown() {
|
||||
}
|
||||
|
||||
fun testFsStructure() {
|
||||
val dir = FileUtil.createTempDirectory("zip0-fs-structure-dir", null)
|
||||
|
||||
val file = File(dir, "a.zip")
|
||||
val zip = JBZipFile(file)
|
||||
val str = "Hello"
|
||||
val helloBytes = str.toByteArray(Charsets.UTF_8)
|
||||
zip.use {
|
||||
zip.getOrCreateEntry("b.txt").data = helloBytes
|
||||
zip.getOrCreateEntry("a/b.txt").data = helloBytes
|
||||
zip.getOrCreateEntry("a/c.txt").data = helloBytes
|
||||
}
|
||||
|
||||
val fs = UncompressedZipFileSystem(file.toPath(), UncompressedZipFileSystemProvider())
|
||||
val file1 = fs.getPath("b.txt")
|
||||
val file2 = fs.getPath("a", "b.txt")
|
||||
val file3 = fs.getPath("a", "c.txt")
|
||||
|
||||
assertTrue(Files.exists(file1))
|
||||
assertTrue(Files.exists(file2))
|
||||
assertTrue(Files.exists(file3))
|
||||
|
||||
assertTrue(helloBytes.contentEquals(Files.readAllBytes(file1)))
|
||||
assertTrue(helloBytes.contentEquals(Files.readAllBytes(file2)))
|
||||
assertTrue(helloBytes.contentEquals(Files.readAllBytes(file3)))
|
||||
|
||||
val parent2 = file2.parent
|
||||
val parent3 = file3.parent
|
||||
assertTrue(Files.isSameFile(parent2, parent3))
|
||||
|
||||
val pp2 = parent2.parent
|
||||
val pp3 = parent3.parent
|
||||
val pp1 = file1.parent
|
||||
assertTrue(Files.isSameFile(pp1, pp2))
|
||||
assertTrue(Files.isSameFile(pp1, pp3))
|
||||
assertTrue(Files.isSameFile(pp2, pp3))
|
||||
|
||||
fs.close()
|
||||
}
|
||||
}
|
||||
@@ -180,7 +180,7 @@ public class StringEnumeratorTest extends TestCase {
|
||||
private static final StringBuilder builder = new StringBuilder(100);
|
||||
private static final Random random = new Random(2_71828);
|
||||
|
||||
static String createRandomString() {
|
||||
public static String createRandomString() {
|
||||
return createRandomString(random);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.impl;
|
||||
|
||||
import com.intellij.util.indexing.ValueContainer;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class MergedValueContainer<Value> extends ValueContainer<Value> {
|
||||
private final List<ValueContainer<Value>> myContainers;
|
||||
private int mySize;
|
||||
|
||||
@NotNull
|
||||
public static <Value> ValueContainer<Value> merge(@NotNull ValueContainer<Value> container1, @NotNull ValueContainer<Value> container2) {
|
||||
if (container1.size() == 0) return container2;
|
||||
if (container2.size() == 0) return container1;
|
||||
return new MergedValueContainer<>(Arrays.asList(container1, container2));
|
||||
}
|
||||
|
||||
public MergedValueContainer(@NotNull List<ValueContainer<Value>> containers) {
|
||||
if (containers.isEmpty()) {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
myContainers = containers;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public InvertedIndexValueIterator<Value> getValueIterator() {
|
||||
return new InvertedIndexValueIterator<Value>() {
|
||||
int myNextId = 1;
|
||||
ValueIterator<Value> myCurrent = myContainers.get(0).getValueIterator();
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public IntIterator getInputIdsIterator() {
|
||||
return myCurrent.getInputIdsIterator();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public IntPredicate getValueAssociationPredicate() {
|
||||
return myCurrent.getValueAssociationPredicate();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getFileSetObject() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
while (true) {
|
||||
if (myCurrent.hasNext()) return true;
|
||||
if (myNextId < myContainers.size()) {
|
||||
myCurrent = myContainers.get(myNextId++).getValueIterator();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Value next() {
|
||||
return myCurrent.next();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
if (mySize == 0) {
|
||||
mySize = myContainers.stream().mapToInt(c -> c.size()).sum();
|
||||
}
|
||||
return mySize;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
|
||||
package com.intellij.util.indexing.impl;
|
||||
|
||||
import com.intellij.openapi.util.Computable;
|
||||
import com.intellij.util.SmartList;
|
||||
import com.intellij.util.indexing.InvertedIndex;
|
||||
import com.intellij.util.indexing.StorageException;
|
||||
import com.intellij.util.indexing.ValueContainer;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class ReadOnlyIndexPack<K, V, Input, Index extends InvertedIndex<K, V, Input>> implements InvertedIndex<K, V, Input> {
|
||||
@NotNull
|
||||
private final List<Index> myIndexes;
|
||||
|
||||
public ReadOnlyIndexPack(@NotNull List<Index> indexes) {myIndexes = indexes;}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public ValueContainer<V> getData(@NotNull K k) throws StorageException {
|
||||
List<ValueContainer<V>> result = new SmartList<>();
|
||||
for (InvertedIndex<K, V, Input> index : myIndexes) {
|
||||
ValueContainer<V> currentData = index.getData(k);
|
||||
if (currentData.size() != 0) {
|
||||
result.add(currentData);
|
||||
}
|
||||
}
|
||||
return result.isEmpty() ? new ValueContainerImpl<>() : new MergedValueContainer<>(result);
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public Computable<Boolean> update(int inputId, @Nullable Input content) {
|
||||
throw new UnsupportedOperationException("index pack is read-only");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws StorageException {
|
||||
List<StorageException> exceptions = new SmartList<>();
|
||||
for (InvertedIndex<K, V, Input> index : myIndexes) {
|
||||
try {
|
||||
index.flush();
|
||||
}
|
||||
catch (StorageException e) {
|
||||
exceptions.add(e);
|
||||
}
|
||||
}
|
||||
if (!exceptions.isEmpty()) {
|
||||
throw exceptions.get(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() throws StorageException {
|
||||
throw new UnsupportedOperationException("index pack is read-only");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dispose() {
|
||||
List<RuntimeException> exceptions = new SmartList<>();
|
||||
for (InvertedIndex<K, V, Input> index : myIndexes) {
|
||||
try {
|
||||
index.dispose();
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
exceptions.add(e);
|
||||
}
|
||||
}
|
||||
if (!exceptions.isEmpty()) {
|
||||
throw exceptions.get(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -457,7 +457,7 @@ public class JBZipEntry implements Cloneable {
|
||||
}
|
||||
}
|
||||
|
||||
private long calcDataOffset() throws IOException {
|
||||
public long calcDataOffset() throws IOException {
|
||||
long offset = getHeaderOffset();
|
||||
myFile.archive.seek(offset + JBZipFile.LFH_OFFSET_FOR_FILENAME_LENGTH);
|
||||
byte[] b = new byte[JBZipFile.WORD];
|
||||
|
||||
Reference in New Issue
Block a user