From 27214b011ed3c13adc8db20fab99f57b8d9d4ac2 Mon Sep 17 00:00:00 2001 From: Dmitry Batkovich Date: Mon, 6 Mar 2017 11:31:29 +0300 Subject: [PATCH] read-only index storage --- .../backwardRefs/CompilerReferenceReader.java | 2 +- .../com/intellij/index/IndexTest.groovy | 20 +++++++++++++----- .../com/intellij/index/StringIndex.java | 21 ++++++++++++++----- .../BackwardReferenceIndexWriter.java | 2 +- .../CompilerBackwardReferenceIndex.java | 18 +++++++++------- .../references/ReferenceIndexTestBase.kt | 2 +- .../indexing/VfsAwareMapIndexStorage.java | 10 ++++++--- .../util/indexing/impl/MapIndexStorage.java | 17 ++++++++++----- .../util/indexing/impl/ValueContainerMap.java | 10 ++++++++- 9 files changed, 73 insertions(+), 29 deletions(-) diff --git a/java/compiler/impl/src/com/intellij/compiler/backwardRefs/CompilerReferenceReader.java b/java/compiler/impl/src/com/intellij/compiler/backwardRefs/CompilerReferenceReader.java index 49ab7d6ccb70..e08f2027b541 100644 --- a/java/compiler/impl/src/com/intellij/compiler/backwardRefs/CompilerReferenceReader.java +++ b/java/compiler/impl/src/com/intellij/compiler/backwardRefs/CompilerReferenceReader.java @@ -48,7 +48,7 @@ class CompilerReferenceReader { private final File myBuildDir; private CompilerReferenceReader(File buildDir) { - myIndex = new CompilerBackwardReferenceIndex(buildDir); + myIndex = new CompilerBackwardReferenceIndex(buildDir, true); myBuildDir = buildDir; } diff --git a/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy b/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy index ead6888afb47..39e911d3e60d 100644 --- a/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy +++ b/java/java-tests/testSrc/com/intellij/index/IndexTest.groovy @@ -64,6 +64,7 @@ import com.intellij.testFramework.SkipSlowTestLocally import com.intellij.testFramework.exceptionCases.IllegalArgumentExceptionCase import com.intellij.testFramework.fixtures.JavaCodeInsightFixtureTestCase import com.intellij.util.FileContentUtil +import com.intellij.util.IncorrectOperationException import com.intellij.util.Processor import com.intellij.util.indexing.* import com.intellij.util.io.* @@ -86,7 +87,7 @@ class IndexTest extends JavaCodeInsightFixtureTestCase { } void testUpdate() throws StorageException, IOException { - StringIndex index = createIndex(getTestName(false), new EnumeratorStringDescriptor()) + StringIndex index = createIndex(getTestName(false), new EnumeratorStringDescriptor(), false) try { // build index @@ -133,7 +134,7 @@ class IndexTest extends JavaCodeInsightFixtureTestCase { } void testUpdateWithCustomEqualityPolicy() { - def index = createIndex(getTestName(false), new CaseInsensitiveEnumeratorStringDescriptor()) + def index = createIndex(getTestName(false), new CaseInsensitiveEnumeratorStringDescriptor(), false) try { index.update("a.java", "x", null) assertDataEquals(index.getFilesByWord("x"), "a.java") @@ -151,12 +152,12 @@ class IndexTest extends JavaCodeInsightFixtureTestCase { } } - private static StringIndex createIndex(String testName, EnumeratorStringDescriptor keyDescriptor) { + private static StringIndex createIndex(String testName, EnumeratorStringDescriptor keyDescriptor, boolean readOnly) { final File storageFile = FileUtil.createTempFile("index_test", "storage") final File metaIndexFile = FileUtil.createTempFile("index_test_inputs", "storage") PersistentHashMap> index = createMetaIndex(metaIndexFile) - final VfsAwareMapIndexStorage indexStorage = new VfsAwareMapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024) - return new StringIndex(testName, indexStorage, index) + final VfsAwareMapIndexStorage indexStorage = new VfsAwareMapIndexStorage(storageFile, keyDescriptor, new EnumeratorStringDescriptor(), 16 * 1024, readOnly) + return new StringIndex(testName, indexStorage, index, !readOnly) } private static PersistentHashMap> createMetaIndex(File metaIndexFile) throws IOException { @@ -665,6 +666,15 @@ class IndexTest extends JavaCodeInsightFixtureTestCase { assert JavaFileElementType.isInSourceContent(myFixture.tempDirFixture.getFile('another/doo/A.java')) } + void "test read-only index access"() { + StringIndex index = createIndex(getTestName(false), new EnumeratorStringDescriptor(), true) + assertFalse(index.update("qwe/asd", "some_string", null)) + def rebuildException = index.getRebuildException() + assertInstanceOf(rebuildException, StorageException.class) + def rebuildCause = rebuildException.getCause() + assertInstanceOf(rebuildCause, IncorrectOperationException.class) + } + @CompileStatic void "test Vfs Events Processing Performance"() { def filename = 'A.java' diff --git a/java/java-tests/testSrc/com/intellij/index/StringIndex.java b/java/java-tests/testSrc/com/intellij/index/StringIndex.java index 460044d3cfc2..d279242b0589 100644 --- a/java/java-tests/testSrc/com/intellij/index/StringIndex.java +++ b/java/java-tests/testSrc/com/intellij/index/StringIndex.java @@ -40,8 +40,11 @@ import java.util.Map; */ public class StringIndex { private final MapReduceIndex myIndex; - - public StringIndex(String testName, final IndexStorage storage, final PersistentHashMap> inputIndex) + private volatile Exception myRebuildException; + public StringIndex(String testName, + final IndexStorage storage, + final PersistentHashMap> inputIndex, + boolean failOnRebuildRequest) throws IOException { ID id = ID.create(testName + "string_index"); IndexExtension extension = new IndexExtension() { @@ -83,7 +86,11 @@ public class StringIndex { }) { @Override public void requestRebuild(@NotNull Exception ex) { - Assert.fail(); + if (failOnRebuildRequest) { + Assert.fail(); + } else { + myRebuildException = ex; + } } }; @@ -93,8 +100,8 @@ public class StringIndex { return ContainerUtil.collect(myIndex.getData(word).getValueIterator()); } - public void update(final String path, @Nullable String content, @Nullable String oldContent) throws StorageException { - myIndex.update(Math.abs(path.hashCode()), toInput(path, content)).compute(); + public boolean update(final String path, @Nullable String content, @Nullable String oldContent) throws StorageException { + return myIndex.update(Math.abs(path.hashCode()), toInput(path, content)).compute(); } public void flush() throws StorageException { @@ -110,6 +117,10 @@ public class StringIndex { return content != null ? new PathContentPair(path, content) : null; } + public Exception getRebuildException() { + return myRebuildException; + } + private static class Indexer implements DataIndexer { @Override @NotNull diff --git a/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/BackwardReferenceIndexWriter.java b/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/BackwardReferenceIndexWriter.java index f03304ed6db7..5cc44a7cce74 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/BackwardReferenceIndexWriter.java +++ b/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/BackwardReferenceIndexWriter.java @@ -95,7 +95,7 @@ public class BackwardReferenceIndexWriter { } if (CompilerBackwardReferenceIndex.exist(buildDir) || isRebuild) { - ourInstance = new BackwardReferenceIndexWriter(new CompilerBackwardReferenceIndex(buildDir)); + ourInstance = new BackwardReferenceIndexWriter(new CompilerBackwardReferenceIndex(buildDir, false)); } } else { CompilerBackwardReferenceIndex.removeIndexFiles(buildDir); diff --git a/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/CompilerBackwardReferenceIndex.java b/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/CompilerBackwardReferenceIndex.java index b3e724e0e239..900165450b62 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/CompilerBackwardReferenceIndex.java +++ b/jps/jps-builders/src/org/jetbrains/jps/backwardRefs/CompilerBackwardReferenceIndex.java @@ -65,7 +65,7 @@ public class CompilerBackwardReferenceIndex { }); private volatile Exception myRebuildRequestCause; - public CompilerBackwardReferenceIndex(File buildDir) { + public CompilerBackwardReferenceIndex(File buildDir, boolean readOnly) { myIndicesDir = getIndexDir(buildDir); if (!myIndicesDir.exists() && !myIndicesDir.mkdirs()) { throw new RuntimeException("Can't create dir: " + buildDir.getAbsolutePath()); @@ -84,7 +84,7 @@ public class CompilerBackwardReferenceIndex { myIndices = new HashMap<>(); for (IndexExtension indexExtension : CompilerIndices.getIndices()) { //noinspection unchecked - myIndices.put(indexExtension.getName(), new CompilerMapReduceIndex(indexExtension, myIndicesDir)); + myIndices.put(indexExtension.getName(), new CompilerMapReduceIndex(indexExtension, myIndicesDir, readOnly)); } myNameEnumerator = new NameEnumerator(new File(myIndicesDir, NAME_ENUM_TAB)); @@ -222,11 +222,12 @@ public class CompilerBackwardReferenceIndex { class CompilerMapReduceIndex extends MapReduceIndex { public CompilerMapReduceIndex(@NotNull final IndexExtension extension, - @NotNull final File indexDir) + @NotNull final File indexDir, + boolean readOnly) throws IOException { super(extension, - createIndexStorage(extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getName(), indexDir), - new MapBasedForwardIndex(extension) { + createIndexStorage(extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getName(), indexDir, readOnly), + readOnly ? null : new MapBasedForwardIndex(extension) { @NotNull @Override public PersistentHashMap> createMap() throws IOException { @@ -253,12 +254,15 @@ public class CompilerBackwardReferenceIndex { private static IndexStorage createIndexStorage(@NotNull KeyDescriptor keyDescriptor, @NotNull DataExternalizer valueExternalizer, @NotNull ID indexId, - @NotNull File indexDir) throws IOException { + @NotNull File indexDir, + boolean readOnly) throws IOException { return new MapIndexStorage(new File(indexDir, indexId.toString()), keyDescriptor, valueExternalizer, 16 * 1024, - false) { + false, + true, + readOnly) { @Override public void checkCanceled() { //TODO diff --git a/jps/jps-builders/testSrc/org/jetbrains/references/ReferenceIndexTestBase.kt b/jps/jps-builders/testSrc/org/jetbrains/references/ReferenceIndexTestBase.kt index 0eb61668bb63..460dc2d69438 100644 --- a/jps/jps-builders/testSrc/org/jetbrains/references/ReferenceIndexTestBase.kt +++ b/jps/jps-builders/testSrc/org/jetbrains/references/ReferenceIndexTestBase.kt @@ -78,7 +78,7 @@ abstract class ReferenceIndexTestBase : JpsBuildTestCase() { val pd = createProjectDescriptor(BuildLoggingManager(TestProjectBuilderLogger())) val manager = pd.dataManager val buildDir = manager.dataPaths.dataStorageRoot - val index = CompilerBackwardReferenceIndex(buildDir) + val index = CompilerBackwardReferenceIndex(buildDir, true) try { val fileEnumerator = index.filePathEnumerator diff --git a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java index 77aecf010694..3f0569dddfba 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapIndexStorage.java @@ -36,6 +36,7 @@ import gnu.trove.TIntHashSet; import gnu.trove.TIntProcedure; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; import java.io.*; @@ -52,12 +53,15 @@ public final class VfsAwareMapIndexStorage extends MapIndexStorage ourInvalidatedSessionIds = ContainerUtil.createConcurrentIntObjectMap(); + @TestOnly public VfsAwareMapIndexStorage(@NotNull File storageFile, @NotNull KeyDescriptor keyDescriptor, @NotNull DataExternalizer valueExternalizer, - final int cacheSize + final int cacheSize, + final boolean readOnly ) throws IOException { - this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false); + super(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, true, readOnly); + myBuildKeyHashToVirtualFileMapping = false; } public VfsAwareMapIndexStorage(@NotNull File storageFile, @@ -66,7 +70,7 @@ public final class VfsAwareMapIndexStorage extends MapIndexStorage implements IndexStorage myDataExternalizer; private final boolean myKeyIsUniqueForIndexedFile; + private final boolean myReadOnly; - public MapIndexStorage(@NotNull File storageFile, + protected MapIndexStorage(@NotNull File storageFile, @NotNull KeyDescriptor keyDescriptor, @NotNull DataExternalizer valueExternalizer, final int cacheSize, boolean keyIsUniqueForIndexedFile) throws IOException { - this(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, true); + this(storageFile, keyDescriptor, valueExternalizer, cacheSize, keyIsUniqueForIndexedFile, true, false); } protected MapIndexStorage(@NotNull File storageFile, @@ -56,12 +58,14 @@ public abstract class MapIndexStorage implements IndexStorage valueExternalizer, final int cacheSize, boolean keyIsUniqueForIndexedFile, - boolean initialize) throws IOException { + boolean initialize, + boolean readOnly) throws IOException { myBaseStorageFile = storageFile; myKeyDescriptor = keyDescriptor; myCacheSize = cacheSize; myDataExternalizer = valueExternalizer; myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile; + myReadOnly = readOnly; if (initialize) initMapAndCache(); } @@ -76,7 +80,7 @@ public abstract class MapIndexStorage implements IndexStorage(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile); + map = new ValueContainerMap(getStorageFile(), myKeyDescriptor, myDataExternalizer, myKeyIsUniqueForIndexedFile, myReadOnly); } finally { PersistentHashMapValueStorage.CreationTimeOptions.EXCEPTIONAL_IO_CANCELLATION.set(null); PersistentHashMapValueStorage.CreationTimeOptions.COMPACT_CHUNKS_WITH_VALUE_DESERIALIZATION.set(null); @@ -112,7 +116,7 @@ public abstract class MapIndexStorage implements IndexStorage valueContainer) { - if (valueContainer.isDirty()) { + if (!myReadOnly && valueContainer.isDirty()) { try { map.put(key, valueContainer); } @@ -201,6 +205,9 @@ public abstract class MapIndexStorage implements IndexStorage extends PersistentHashMap> { @NotNull private final DataExternalizer myValueExternalizer; private final boolean myKeyIsUniqueForIndexedFile; + private final boolean myReadOnly; ValueContainerMap(@NotNull final File file, @NotNull KeyDescriptor keyKeyDescriptor, @NotNull DataExternalizer valueExternalizer, - boolean keyIsUniqueForIndexedFile + boolean keyIsUniqueForIndexedFile, + boolean readOnly ) throws IOException { super(file, keyKeyDescriptor, new ValueContainerExternalizer(valueExternalizer)); myValueExternalizer = valueExternalizer; myKeyIsUniqueForIndexedFile = keyIsUniqueForIndexedFile; + myReadOnly = readOnly; } @NotNull @@ -68,6 +71,11 @@ class ValueContainerMap extends PersistentHashMap implements DataExternalizer> { @NotNull private final DataExternalizer myValueExternalizer;