MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

read-only index storage

Browse Source
This commit is contained in:
Dmitry Batkovich
2017-03-06 16:07:01 +03:00
parent c1bb394630
commit 27214b011e
9 changed files with 73 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/compiler/impl/src/com/intellij/compiler/backwardRefs/CompilerReferenceReader.java
+1 -1
View File
@@ -48,7 +48,7 @@ class CompilerReferenceReader {
private final File myBuildDir;
private CompilerReferenceReader(File buildDir) {
myIndex = new CompilerBackwardReferenceIndex(buildDir);
myIndex = new CompilerBackwardReferenceIndex(buildDir, true);
myBuildDir = buildDir;
}
java/java-tests/testSrc/com/intellij/index/IndexTest.groovy
+15 -5
View File
@@ -64,6 +64,7 @@ import com.intellij.testFramework.SkipSlowTestLocally
import com.intellij.testFramework.exceptionCases.IllegalArgumentExceptionCase
import com.intellij.testFramework.fixtures.JavaCodeInsightFixtureTestCase
import com.intellij.util.FileContentUtil
import com.intellij.util.IncorrectOperationException
import com.intellij.util.Processor
import com.intellij.util.indexing.*
import com.intellij.util.io.*
@@ -86,7 +87,7 @@ class IndexTest extends JavaCodeInsightFixtureTestCase {
}
void testUpdate() throws StorageException, IOException {
StringIndex index = createIndex(getTestName(false), new EnumeratorStringDescriptor())
StringIndex index = createIndex(getTestName(false), new EnumeratorStringDescriptor(), false)
try {
// build index
@@ -133,7 +134,7 @@ class IndexTest extends JavaCodeInsightFixtureTestCase {
}
void testUpdateWithCustomEqualityPolicy() {
def index = createIndex(getTestName(false), new CaseInsensitiveEnumeratorStringDescriptor())
def index = createIndex(getTestName(false), new CaseInsensitiveEnumeratorStringDescriptor(), false)
try {
index.update("a.java", "x", null)
assertDataEquals(index.getFilesByWord("x"), "a.java")
@@ -151,12 +152,12 @@ class IndexTest extends JavaCodeInsightFixtureTestCase {
}
}
private static StringIndex createIndex(String testName, EnumeratorStringDescriptor keyDescriptor) {
private static StringIndex createIndex(String testName, EnumeratorStringDescriptor keyDescriptor, boolean readOnly) {
final File storageFile = FileUtil.createTempFile("index_test", "storage")
final File metaIndexFile = FileUtil.createTempFile("index_test_inputs", "storage")
PersistentHashMap<Integer, Collection<String>> index = createMetaIndex(metaIndexFile)
final VfsAwareMapIndexStorage indexStorage = new VfsAwareMapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024)
return new StringIndex(testName, indexStorage, index)
final VfsAwareMapIndexStorage indexStorage = new VfsAwareMapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024, readOnly)
return new StringIndex(testName, indexStorage, index, !readOnly)
}
private static PersistentHashMap<Integer, Collection<String>> createMetaIndex(File metaIndexFile) throws IOException {
@@ -665,6 +666,15 @@ class IndexTest extends JavaCodeInsightFixtureTestCase {
assert JavaFileElementType.isInSourceContent(myFixture.tempDirFixture.getFile('another/doo/A.java'))
}
void "test read-only index access"() {
StringIndex index = createIndex(getTestName(false), new EnumeratorStringDescriptor(), true)
assertFalse(index.update("qwe/asd", "some_string", null))
def rebuildException = index.getRebuildException()
assertInstanceOf(rebuildException, StorageException.class)
def rebuildCause = rebuildException.getCause()
assertInstanceOf(rebuildCause, IncorrectOperationException.class)
}
@CompileStatic
void "test Vfs Events Processing Performance"() {
def filename = 'A.java'
java/java-tests/testSrc/com/intellij/index/StringIndex.java
+16 -5
View File
@@ -40,8 +40,11 @@ import java.util.Map;
*/
public class StringIndex {
private final MapReduceIndex<String, String, PathContentPair> myIndex;
public StringIndex(String testName, final IndexStorage<String, String> storage, final PersistentHashMap<Integer, Collection<String>> inputIndex)
private volatile Exception myRebuildException;
public StringIndex(String testName,
final IndexStorage<String, String> storage,
final PersistentHashMap<Integer, Collection<String>> inputIndex,
boolean failOnRebuildRequest)
throws IOException {
ID<String, String> id = ID.create(testName + "string_index");
IndexExtension<String, String, PathContentPair> extension = new IndexExtension<String, String, PathContentPair>() {
@@ -83,7 +86,11 @@ public class StringIndex {
}) {
@Override
public void requestRebuild(@NotNull Exception ex) {
Assert.fail();
if (failOnRebuildRequest) {
Assert.fail();
} else {
myRebuildException = ex;
}
}
};
@@ -93,8 +100,8 @@ public class StringIndex {
return ContainerUtil.collect(myIndex.getData(word).getValueIterator());
}
public void update(final String path, @Nullable String content, @Nullable String oldContent) throws StorageException {
myIndex.update(Math.abs(path.hashCode()), toInput(path, content)).compute();
public boolean update(final String path, @Nullable String content, @Nullable String oldContent) throws StorageException {
return myIndex.update(Math.abs(path.hashCode()), toInput(path, content)).compute();
}
public void flush() throws StorageException {
@@ -110,6 +117,10 @@ public class StringIndex {
return content != null ? new PathContentPair(path, content) : null;
}
public Exception getRebuildException() {
return myRebuildException;
}
private static class Indexer implements DataIndexer<String, String, PathContentPair> {
@Override
@NotNull
jps/jps-builders/src/org/jetbrains/jps/backwardRefs/BackwardReferenceIndexWriter.java
+1 -1
View File
@@ -95,7 +95,7 @@ public class BackwardReferenceIndexWriter {
}
if (CompilerBackwardReferenceIndex.exist(buildDir) || isRebuild) {
ourInstance = new BackwardReferenceIndexWriter(new CompilerBackwardReferenceIndex(buildDir));
ourInstance = new BackwardReferenceIndexWriter(new CompilerBackwardReferenceIndex(buildDir, false));
}
} else {
CompilerBackwardReferenceIndex.removeIndexFiles(buildDir);
jps/jps-builders/src/org/jetbrains/jps/backwardRefs/CompilerBackwardReferenceIndex.java
+11 -7
View File
@@ -65,7 +65,7 @@ public class CompilerBackwardReferenceIndex {
});
private volatile Exception myRebuildRequestCause;
public CompilerBackwardReferenceIndex(File buildDir) {
public CompilerBackwardReferenceIndex(File buildDir, boolean readOnly) {
myIndicesDir = getIndexDir(buildDir);
if (!myIndicesDir.exists() && !myIndicesDir.mkdirs()) {
throw new RuntimeException("Can't create dir: " + buildDir.getAbsolutePath());
@@ -84,7 +84,7 @@ public class CompilerBackwardReferenceIndex {
myIndices = new HashMap<>();
for (IndexExtension<LightRef, ?, CompiledFileData> indexExtension : CompilerIndices.getIndices()) {
//noinspection unchecked
myIndices.put(indexExtension.getName(), new CompilerMapReduceIndex(indexExtension, myIndicesDir));
myIndices.put(indexExtension.getName(), new CompilerMapReduceIndex(indexExtension, myIndicesDir, readOnly));
}
myNameEnumerator = new NameEnumerator(new File(myIndicesDir, NAME_ENUM_TAB));
@@ -222,11 +222,12 @@ public class CompilerBackwardReferenceIndex {
class CompilerMapReduceIndex<Key, Value> extends MapReduceIndex<Key, Value, CompiledFileData> {
public CompilerMapReduceIndex(@NotNull final IndexExtension<Key, Value, CompiledFileData> extension,
@NotNull final File indexDir)
@NotNull final File indexDir,
boolean readOnly)
throws IOException {
super(extension,
createIndexStorage(extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getName(), indexDir),
new MapBasedForwardIndex<Key, Value>(extension) {
createIndexStorage(extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getName(), indexDir, readOnly),
readOnly ? null : new MapBasedForwardIndex<Key, Value>(extension) {
@NotNull
@Override
public PersistentHashMap<Integer, Collection<Key>> createMap() throws IOException {
@@ -253,12 +254,15 @@ public class CompilerBackwardReferenceIndex {
private static <Key, Value> IndexStorage<Key, Value> createIndexStorage(@NotNull KeyDescriptor<Key> keyDescriptor,
@NotNull DataExternalizer<Value> valueExternalizer,
@NotNull ID<Key, Value> indexId,
@NotNull File indexDir) throws IOException {
@NotNull File indexDir,
boolean readOnly) throws IOException {
return new MapIndexStorage<Key, Value>(new File(indexDir, indexId.toString()),
keyDescriptor,
valueExternalizer,
16 * 1024,
false) {
false,
true,
readOnly) {
@Override
public void checkCanceled() {
//TODO
jps/jps-builders/testSrc/org/jetbrains/references/ReferenceIndexTestBase.kt
+1 -1
View File
@@ -78,7 +78,7 @@ abstract class ReferenceIndexTestBase : JpsBuildTestCase() {
val pd = createProjectDescriptor(BuildLoggingManager(TestProjectBuilderLogger()))
val manager = pd.dataManager
val buildDir = manager.dataPaths.dataStorageRoot
val index = CompilerBackwardReferenceIndex(buildDir)
val index = CompilerBackwardReferenceIndex(buildDir, true)
try {
val fileEnumerator = index.filePathEnumerator
platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java
+7 -3
View File
@@ -36,6 +36,7 @@ import gnu.trove.TIntHashSet;
import gnu.trove.TIntProcedure;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
import java.io.*;
@@ -52,12 +53,15 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
private static final ConcurrentIntObjectMap<Boolean> ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap();
@TestOnly
public VfsAwareMapIndexStorage(@NotNull File storageFile,
@NotNull KeyDescriptor<Key> keyDescriptor,
@NotNull DataExternalizer<Value> valueExternalizer,
final int cacheSize
final int cacheSize,
final boolean readOnly
) throws IOException {
this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false);
super(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, true, readOnly);
myBuildKeyHashToVirtualFileMapping = false;
}
public VfsAwareMapIndexStorage(@NotNull File storageFile,
@@ -66,7 +70,7 @@ public final class VfsAwareMapIndexStorage<Key, Value> extends MapIndexStorage<K
final int cacheSize,
boolean keyIsUniqueForIndexedFile,
boolean buildKeyHashToVirtualFileMapping) throws IOException {
super(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, false);
super(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, false, false);
myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping;
initMapAndCache();
}
platform/util/src/com/intellij/util/indexing/impl/MapIndexStorage.java
+12 -5
View File
@@ -16,6 +16,7 @@
package com.intellij.util.indexing.impl;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.Processor;
import com.intellij.util.containers.SLRUCache;
import com.intellij.util.indexing.StorageException;
@@ -42,13 +43,14 @@ public abstract class MapIndexStorage<Key, Value> implements IndexStorage<Key, V
protected final Lock l = new ReentrantLock();
private final DataExternalizer<Value> myDataExternalizer;
private final boolean myKeyIsUniqueForIndexedFile;
private final boolean myReadOnly;
public MapIndexStorage(@NotNull File storageFile,
protected MapIndexStorage(@NotNull File storageFile,
@NotNull KeyDescriptor<Key> keyDescriptor,
@NotNull DataExternalizer<Value> valueExternalizer,
final int cacheSize,
boolean keyIsUniqueForIndexedFile) throws IOException {
this(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, true);
this(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, true, false);
}
protected MapIndexStorage(@NotNull File storageFile,
@@ -56,12 +58,14 @@ public abstract class MapIndexStorage<Key, Value> implements IndexStorage<Key, V
@NotNull DataExternalizer<Value> valueExternalizer,
final int cacheSize,
boolean keyIsUniqueForIndexedFile,
boolean initialize) throws IOException {
boolean initialize,
boolean readOnly) throws IOException {
myBaseStorageFile = storageFile;
myKeyDescriptor = keyDescriptor;
myCacheSize = cacheSize;
myDataExternalizer = valueExternalizer;
myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
myReadOnly = readOnly;
if (initialize) initMapAndCache();
}
@@ -76,7 +80,7 @@ public abstract class MapIndexStorage<Key, Value> implements IndexStorage<Key, V
});
PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(Boolean.TRUE);
try {
map = new ValueContainerMap<Key, Value>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile);
map = new ValueContainerMap<Key, Value>(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile, myReadOnly);
} finally {
PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null);
PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null);
@@ -112,7 +116,7 @@ public abstract class MapIndexStorage<Key, Value> implements IndexStorage<Key, V
@Override
protected void onDropFromCache(final Key key, @NotNull final ChangeTrackingValueContainer<Value> valueContainer) {
if (valueContainer.isDirty()) {
if (!myReadOnly && valueContainer.isDirty()) {
try {
map.put(key, valueContainer);
}
@@ -201,6 +205,9 @@ public abstract class MapIndexStorage<Key, Value> implements IndexStorage<Key, V
@Override
public void addValue(final Key key, final int inputId, final Value value) throws StorageException {
if (myReadOnly) {
throw new IncorrectOperationException("Index storage is read-only");
}
try {
myMap.markDirty();
if (!myKeyIsUniqueForIndexedFile) {
platform/util/src/com/intellij/util/indexing/impl/ValueContainerMap.java
+9 -1
View File
@@ -29,15 +29,18 @@ import java.io.*;
class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, UpdatableValueContainer<Value>> {
@NotNull private final DataExternalizer<Value> myValueExternalizer;
private final boolean myKeyIsUniqueForIndexedFile;
private final boolean myReadOnly;
ValueContainerMap(@NotNull final File file,
@NotNull KeyDescriptor<Key> keyKeyDescriptor,
@NotNull DataExternalizer<Value> valueExternalizer,
boolean keyIsUniqueForIndexedFile
boolean keyIsUniqueForIndexedFile,
boolean readOnly
) throws IOException {
super(file, keyKeyDescriptor, new ValueContainerExternalizer<Value>(valueExternalizer));
myValueExternalizer = valueExternalizer;
myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile;
myReadOnly = readOnly;
}
@NotNull
@@ -68,6 +71,11 @@ class ValueContainerMap<Key, Value> extends PersistentHashMap<Key, UpdatableValu
}
}
@Override
protected boolean isReadOnly() {
return myReadOnly;
}
private static final class ValueContainerExternalizer<T> implements DataExternalizer<UpdatableValueContainer<T>> {
@NotNull private final DataExternalizer<T> myValueExternalizer;