diff --git a/java/java-tests/testSrc/com/intellij/java/index/IndexTest.groovy b/java/java-tests/testSrc/com/intellij/java/index/IndexTest.groovy index c392eed1ecde..c542d8766e7f 100644 --- a/java/java-tests/testSrc/com/intellij/java/index/IndexTest.groovy +++ b/java/java-tests/testSrc/com/intellij/java/index/IndexTest.groovy @@ -63,6 +63,7 @@ import com.intellij.util.indexing.* import com.intellij.util.indexing.impl.MapIndexStorage import com.intellij.util.indexing.impl.MapReduceIndex import com.intellij.util.io.* +import com.intellij.util.ref.GCUtil import com.siyeh.ig.JavaOverridingMethodUtil import groovy.transform.CompileStatic import org.jetbrains.annotations.NotNull @@ -515,6 +516,48 @@ class IndexTest extends JavaCodeInsightFixtureTestCase { assertTrue(stamp != ((StubIndexImpl)StubIndex.instance).getIndexModificationStamp(JavaStubIndexKeys.CLASS_SHORT_NAMES, project)) } + void "test internalErrorOfStubProcessingInvalidatesIndex"() throws IOException { + final VirtualFile vFile = myFixture.addClass("class Foo {}").getContainingFile().getVirtualFile() + + assertTrue(findClass("Foo") != null) + + runFindClassStubIndexQueryThatProducesInvalidResult("Foo") + + GCUtil.tryGcSoftlyReachableObjects() // invalidates cache in findClass + assertNull(findClass("Foo")) + + // check invalidation of transient indices state + def document = FileDocumentManager.instance.getDocument(vFile) + document.setText("class Foo2 {}") + PsiDocumentManager.getInstance(project).commitDocument(document) + + assertTrue(findClass("Foo2") != null) + + runFindClassStubIndexQueryThatProducesInvalidResult("Foo2") + GCUtil.tryGcSoftlyReachableObjects() // invalidates cache in findClass + assertNull(findClass("Foo2")) + } + + private void runFindClassStubIndexQueryThatProducesInvalidResult(String qName) { + try { + def foundFile = [null] + + StubIndex.instance. + processElements(JavaStubIndexKeys.CLASS_FQN, qName.hashCode(), project, GlobalSearchScope.allScope(project), PsiFile.class, new Processor() { + @Override + boolean process(PsiFile file) { + foundFile[0] = file + return false + } + }) + + fail("Unexpected") + } + catch (AssertionError ignored) { + // stub mismatch + } + } + void "test do not collect stub tree while holding stub elements"() throws IOException { final VirtualFile vFile = myFixture.addClass("class Foo {}").getContainingFile().getVirtualFile() diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 6b4436da446d..c09520a86943 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -44,6 +44,8 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static com.intellij.util.indexing.FileBasedIndex.getFileId; + @State(name = "FileBasedIndex", storages = @Storage(value = "stubIndex.xml", roamingType = RoamingType.DISABLED)) public class StubIndexImpl extends StubIndex implements PersistentStateComponent, ApplicationComponent { private static final AtomicReference ourForcedClean = new AtomicReference<>(null); @@ -345,6 +347,8 @@ public class StubIndexImpl extends StubIndex implements PersistentStateComponent myAccessValidator.stoppedProcessingActivityForIndex(stubUpdatingIndexId); stubUpdatingIndex.getReadLock().unlock(); FileBasedIndexImpl.enableUpToDateCheckForCurrentThread(); + + wipeProblematicFileIdsForParticularKeyAndStubIndex(indexKey, key, stubUpdatingIndex); } } catch (StorageException e) { @@ -363,6 +367,32 @@ public class StubIndexImpl extends StubIndex implements PersistentStateComponent return true; } + // Self repair for IDEA-181227, caused by (yet) unknown file event processing problem in indices + // FileBasedIndex.requestReindex doesn't handle the situation properly because update requires old data that was lost + private void wipeProblematicFileIdsForParticularKeyAndStubIndex(@NotNull StubIndexKey indexKey, + @NotNull Key key, + UpdatableIndex stubUpdatingIndex) { + Set filesWithProblems = myStubProcessingHelper.takeAccumulatedFilesWithIndexProblems(); + + if (filesWithProblems != null) { + stubUpdatingIndex.getWriteLock().lock(); + try { + Map artificialOldValues = new THashMap<>(); + artificialOldValues.put(key, new StubIdList()); + + for(VirtualFile file:filesWithProblems) { + updateIndex(indexKey, getFileId(file), artificialOldValues, Collections.emptyMap()); + ((FileBasedIndexImpl)FileBasedIndex.getInstance()).runUpdateForInMemoryIndices(() -> { + updateIndex(indexKey, getFileId(file), artificialOldValues, Collections.emptyMap()); + return true; + }); + } + } finally { + stubUpdatingIndex.getWriteLock().unlock(); + } + } + } + @Override public void forceRebuild(@NotNull Throwable e) { FileBasedIndex.getInstance().scheduleRebuild(StubUpdatingIndex.INDEX_ID, e); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java index c4213b096081..18afdb85d2a9 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubProcessingHelper.java @@ -4,12 +4,17 @@ import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.ModalityState; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.indexing.FileBasedIndex; +import gnu.trove.THashSet; +import org.jetbrains.annotations.Nullable; + +import java.util.Set; /** * Author: dmitrylomov */ public class StubProcessingHelper extends StubProcessingHelperBase { private final FileBasedIndex myFileBasedIndex; + private final ThreadLocal> myFilesHavingProblems = new ThreadLocal<>(); public StubProcessingHelper(FileBasedIndex fileBasedIndex) { myFileBasedIndex = fileBasedIndex; @@ -17,8 +22,18 @@ public class StubProcessingHelper extends StubProcessingHelperBase { @Override protected void onInternalError(final VirtualFile file) { + Set set = myFilesHavingProblems.get(); + if (set == null) myFilesHavingProblems.set(set = new THashSet<>()); + set.add(file); // requestReindex() may want to acquire write lock (for indices not requiring content loading) // thus, because here we are under read lock, need to use invoke later ApplicationManager.getApplication().invokeLater(() -> myFileBasedIndex.requestReindex(file), ModalityState.NON_MODAL); } + + @Nullable + Set takeAccumulatedFilesWithIndexProblems() { + Set filesWithProblems = myFilesHavingProblems.get(); + if (filesWithProblems != null) myFilesHavingProblems.set(null); + return filesWithProblems; + } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 656bb812eef9..66390cd13b33 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -1228,20 +1228,15 @@ public class FileBasedIndexImpl extends FileBasedIndex implements BaseComponent, document -> belongsToScope(myFileDocumentManager.getFile(document), restrictedFile, filter)); if (!documentsToProcessForProject.isEmpty()) { - final StorageGuard.StorageModeExitHandler guard = setDataBufferingEnabled(true); - try { - DocumentUpdateTask task = myUnsavedDataUpdateTasks.get(indexId); - assert task != null : "Task for unsaved data indexing was not initialized for index " + indexId; - - boolean processedAll = task.processAll(documentsToProcessForProject, project) && documentsToProcessForProject.size() == documents.size(); - - if (processedAll && !hasActiveTransactions()) { - ProgressManager.checkCanceled(); - myUpToDateIndicesForUnsavedOrTransactedDocuments.add(indexId); - } - } - finally { - guard.leave(); + DocumentUpdateTask task = myUnsavedDataUpdateTasks.get(indexId); + assert task != null : "Task for unsaved data indexing was not initialized for index " + indexId; + + if(runUpdate(true, () -> task.processAll(documentsToProcessForProject, project)) && + documentsToProcessForProject.size() == documents.size() && + !hasActiveTransactions() + ) { + ProgressManager.checkCanceled(); + myUpToDateIndicesForUnsavedOrTransactedDocuments.add(indexId); } } } @@ -1376,24 +1371,32 @@ public class FileBasedIndexImpl extends FileBasedIndex implements BaseComponent, private volatile boolean myPreviousDataBufferingState; private final Object myBufferingStateUpdateLock = new Object(); - @NotNull - private StorageGuard.StorageModeExitHandler setDataBufferingEnabled(final boolean enabled) { - StorageGuard.StorageModeExitHandler storageModeExitHandler = myStorageLock.enter(enabled); + public void runUpdateForInMemoryIndices(Computable update) { + runUpdate(true, update); + } + + private boolean runUpdate(boolean transientInMemoryIndices, Computable update) { + StorageGuard.StorageModeExitHandler storageModeExitHandler = myStorageLock.enter(transientInMemoryIndices); - if (myPreviousDataBufferingState != enabled) { + if (myPreviousDataBufferingState != transientInMemoryIndices) { synchronized (myBufferingStateUpdateLock) { - if (myPreviousDataBufferingState != enabled) { + if (myPreviousDataBufferingState != transientInMemoryIndices) { IndexConfiguration state = getState(); for (ID indexId : state.getIndexIDs()) { final MapReduceIndex index = (MapReduceIndex)state.getIndex(indexId); assert index != null; - ((MemoryIndexStorage)index.getStorage()).setBufferingEnabled(enabled); + ((MemoryIndexStorage)index.getStorage()).setBufferingEnabled(transientInMemoryIndices); } - myPreviousDataBufferingState = enabled; + myPreviousDataBufferingState = transientInMemoryIndices; } } } - return storageModeExitHandler; + + try { + return update.compute(); + } finally { + storageModeExitHandler.leave(); + } } private void cleanupMemoryStorage() { @@ -1693,7 +1696,7 @@ public class FileBasedIndexImpl extends FileBasedIndex implements BaseComponent, myWriteLock = lock.writeLock(); } private void scheduleUpdate(@NotNull final ID indexId, @NotNull Computable update, VirtualFile file, final int inputId, final boolean hasContent) { - if (updateWithBufferingEnabled(update)) { + if (runUpdate(false, update)) { myReadLock.lock(); try { UpdatableIndex index = getIndex(indexId); @@ -1710,16 +1713,6 @@ public class FileBasedIndexImpl extends FileBasedIndex implements BaseComponent, } } - private boolean updateWithBufferingEnabled(@NotNull final Computable update) { - final StorageGuard.StorageModeExitHandler lock = setDataBufferingEnabled(false); - try { - return update.compute(); - } - finally { - lock.leave(); - } - } - private boolean needsFileContentLoading(@NotNull ID indexId) { return !myNotRequiringContentIndices.contains(indexId); }