From 4516cf5994863c6cabad287c605333681083f911 Mon Sep 17 00:00:00 2001 From: Dmitry Batkovich Date: Tue, 3 Dec 2019 21:53:33 +0300 Subject: [PATCH] add access type to dumb mode index access GitOrigin-RevId: babc505bca1efad9671af2de9d47ac5dd19a912f --- .../DefaultClassNavigationContributor.java | 5 +- .../intellij/util/indexing/IndexTest.groovy | 2 +- .../util/indexing/DumbModeAccessType.java | 10 +++ .../util/indexing/FileBasedIndex.java | 10 ++- .../cache/impl/IndexCacheManagerImpl.java | 3 +- .../psi/impl/search/PsiSearchHelperImpl.java | 3 +- .../intellij/find/impl/FindInProjectTask.java | 3 +- .../com/intellij/psi/stubs/StubIndexImpl.java | 21 +++--- .../util/indexing/FileBasedIndexImpl.java | 71 +++++++++++++++---- 9 files changed, 100 insertions(+), 28 deletions(-) create mode 100644 platform/indexing-api/src/com/intellij/util/indexing/DumbModeAccessType.java diff --git a/java/java-impl/src/com/intellij/ide/util/gotoByName/DefaultClassNavigationContributor.java b/java/java-impl/src/com/intellij/ide/util/gotoByName/DefaultClassNavigationContributor.java index 68d0a04a3604..b728c796cf92 100644 --- a/java/java-impl/src/com/intellij/ide/util/gotoByName/DefaultClassNavigationContributor.java +++ b/java/java-impl/src/com/intellij/ide/util/gotoByName/DefaultClassNavigationContributor.java @@ -32,6 +32,7 @@ import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.search.PsiShortNamesCache; import com.intellij.psi.util.ClassUtil; import com.intellij.util.Processor; +import com.intellij.util.indexing.DumbModeAccessType; import com.intellij.util.indexing.FileBasedIndex; import com.intellij.util.indexing.FindSymbolParameters; import com.intellij.util.indexing.IdFilter; @@ -67,7 +68,7 @@ public class DefaultClassNavigationContributor implements ChooseByNameContributo Project project = scope.getProject(); FileBasedIndex.getInstance().ignoreDumbMode(() -> { PsiShortNamesCache.getInstance(project).processAllClassNames(processor, scope, filter); - }, project); + }, project, DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE); } @Override @@ -100,7 +101,7 @@ public class DefaultClassNavigationContributor implements ChooseByNameContributo return processor.process(aClass); } }, parameters.getSearchScope(), parameters.getIdFilter()); - }, parameters.getProject()); + }, parameters.getProject(), DumbModeAccessType.RELIABLE_DATA_ONLY); } @Nullable diff --git a/java/java-tests/testSrc/com/intellij/util/indexing/IndexTest.groovy b/java/java-tests/testSrc/com/intellij/util/indexing/IndexTest.groovy index 6111444e779a..5b4739cb2518 100644 --- a/java/java-tests/testSrc/com/intellij/util/indexing/IndexTest.groovy +++ b/java/java-tests/testSrc/com/intellij/util/indexing/IndexTest.groovy @@ -1266,7 +1266,7 @@ class IndexTest extends JavaCodeInsightFixtureTestCase { found = fileBasedIndex.getContainingFiles(IdIndex.NAME, wordHash, scope).contains(file) } if (inDumbMode) { - fileBasedIndex.ignoreDumbMode(runnable, project) + fileBasedIndex.ignoreDumbMode(runnable, project, DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE) } else { runnable.run() } diff --git a/platform/indexing-api/src/com/intellij/util/indexing/DumbModeAccessType.java b/platform/indexing-api/src/com/intellij/util/indexing/DumbModeAccessType.java new file mode 100644 index 000000000000..3082d31a7ee8 --- /dev/null +++ b/platform/indexing-api/src/com/intellij/util/indexing/DumbModeAccessType.java @@ -0,0 +1,10 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing; + +import org.jetbrains.annotations.ApiStatus; + +@ApiStatus.Experimental +public enum DumbModeAccessType { + RELIABLE_DATA_ONLY, + RAW_INDEX_DATA_ACCEPTABLE +} diff --git a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java index 7cdfa78b42d5..1e8bc558b83c 100644 --- a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java +++ b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java @@ -42,6 +42,12 @@ public abstract class FileBasedIndex { @Nullable public abstract VirtualFile getFileBeingCurrentlyIndexed(); + @ApiStatus.Internal + @ApiStatus.Experimental + public DumbModeAccessType getCurrentDumbModeAccessType() { + throw new UnsupportedOperationException(); + } + public abstract void registerIndexableSet(@NotNull IndexableFileSet set, @Nullable Project project); public abstract void removeIndexableSet(@NotNull IndexableFileSet set); @@ -132,7 +138,9 @@ public abstract class FileBasedIndex { @ApiStatus.Experimental @ApiStatus.Internal - public void ignoreDumbMode(@NotNull Runnable runnable, @NotNull Project project) { + public void ignoreDumbMode(@NotNull Runnable runnable, + @NotNull Project project, + @NotNull DumbModeAccessType dumbModeAccessType) { throw new UnsupportedOperationException(); } diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/IndexCacheManagerImpl.java b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/IndexCacheManagerImpl.java index 6331ba67a759..cd2e9e269533 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/IndexCacheManagerImpl.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/cache/impl/IndexCacheManagerImpl.java @@ -18,6 +18,7 @@ import com.intellij.psi.impl.cache.impl.id.IdIndexEntry; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; import com.intellij.util.Processors; +import com.intellij.util.indexing.DumbModeAccessType; import com.intellij.util.indexing.FileBasedIndex; import org.jetbrains.annotations.NotNull; @@ -58,7 +59,7 @@ public class IndexCacheManagerImpl implements CacheManager{ Processor processor = Processors.cancelableCollectProcessor(result); FileBasedIndex.getInstance().ignoreDumbMode(() -> { collectVirtualFilesWithWord(word, occurenceMask, scope, caseSensitively, processor); - }, myProject); + }, myProject, DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE); return result.isEmpty() ? VirtualFile.EMPTY_ARRAY : result.toArray(VirtualFile.EMPTY_ARRAY); } diff --git a/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java b/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java index ca2e9a15a213..f3f36ce2e1c7 100644 --- a/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java +++ b/platform/indexing-impl/src/com/intellij/psi/impl/search/PsiSearchHelperImpl.java @@ -39,6 +39,7 @@ import com.intellij.util.Processors; import com.intellij.util.SmartList; import com.intellij.util.codeInsight.CommentUtilCore; import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.DumbModeAccessType; import com.intellij.util.indexing.FileBasedIndex; import com.intellij.util.text.StringSearcher; import gnu.trove.THashMap; @@ -1061,7 +1062,7 @@ public class PsiSearchHelperImpl implements PsiSearchHelper { Boolean[] result = {null}; if (FileBasedIndex.isIndexAccessDuringDumbModeEnabled()) { ReadAction.nonBlocking(() -> { - FileBasedIndex.getInstance().ignoreDumbMode(() -> result[0] = query.compute(), project); + FileBasedIndex.getInstance().ignoreDumbMode(() -> result[0] = query.compute(), project, DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE); }).executeSynchronously(); } return result[0] != null ? result[0] : DumbService.getInstance(project).runReadActionInSmartMode(query); diff --git a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java index 563a6b858386..dc16fcfdedf5 100644 --- a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java +++ b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectTask.java @@ -45,6 +45,7 @@ import com.intellij.usages.impl.UsageViewManagerImpl; import com.intellij.util.Processor; import com.intellij.util.Processors; import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.DumbModeAccessType; import com.intellij.util.indexing.FileBasedIndex; import com.intellij.util.indexing.FileBasedIndexImpl; import com.intellij.util.ui.UIUtil; @@ -428,7 +429,7 @@ class FindInProjectTask { final List hits = new ArrayList<>(); FileBasedIndex.getInstance().ignoreDumbMode(() -> { FileBasedIndex.getInstance().getFilesWithKey(TrigramIndex.INDEX_ID, keys, Processors.cancelableCollectProcessor(hits), scope); - }, myProject); + }, myProject, DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE); for (VirtualFile hit : hits) { if (myFileMask.value(hit)) { diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 944aa09d89e8..f6f10bbf681f 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -20,7 +20,6 @@ import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.newvfs.ManagingFS; -import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry; import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; import com.intellij.psi.PsiElement; import com.intellij.psi.search.GlobalSearchScope; @@ -52,6 +51,7 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; +import java.util.function.IntPredicate; @State(name = "FileBasedIndex", storages = { @Storage(value = StoragePathMacros.CACHE_FILE), @@ -368,25 +368,28 @@ public final class StubIndexImpl extends StubIndexEx implements PersistentStateC @NotNull final Class requiredClass, @NotNull final Processor processor) { boolean dumb = DumbService.isDumb(project); + if (dumb) { + DumbModeAccessType accessType = FileBasedIndex.getInstance().getCurrentDumbModeAccessType(); + if (accessType == DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE) { + throw new AssertionError("raw index data access is not available for StubIndex"); + } + } + IdIterator ids = getContainingIds(indexKey, key, project, idFilter, scope); PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); + IntPredicate accessibleFileFilter = ((FileBasedIndexImpl)FileBasedIndex.getInstance()).getAccessibleFileIdFilter(project); // already ensured up-to-date in getContainingIds() method try { while (ids.hasNext()) { int id = ids.next(); ProgressManager.checkCanceled(); + if (!accessibleFileFilter.test(id)) { + continue; + } VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); if (file == null || (scope != null && !scope.contains(file))) { continue; } - if (dumb) { - if (!(file instanceof VirtualFileSystemEntry) || !((VirtualFileSystemEntry)file).isFileIndexed()) { - continue; - } - if (((FileBasedIndexImpl)FileBasedIndex.getInstance()).getChangedFilesCollector().containsFile(file)) { - continue; - } - } StubIdList list = myCachedStubIds.get(indexKey).getValue().computeIfAbsent(new CompositeKey(key, id), __ -> { return myStubProcessingHelper.retrieveStubIdList(indexKey, key, file); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 38a8f3202ad3..a885b6e3e117 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -99,6 +99,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.IntPredicate; import java.util.function.Predicate; import java.util.stream.Stream; @@ -110,7 +111,7 @@ import static com.intellij.serviceContainer.PlatformComponentManagerImplKt.handl public final class FileBasedIndexImpl extends FileBasedIndex { private static final ThreadLocal ourIndexedFile = new ThreadLocal<>(); private static final ThreadLocal ourFileToBeIndexed = new ThreadLocal<>(); - private static final ThreadLocal ourDumbModeIgnored = new ThreadLocal<>(); + private static final ThreadLocal ourDumbModeAccessType = new ThreadLocal<>(); static final Logger LOG = Logger.getInstance(FileBasedIndexImpl.class); private static final String CORRUPTION_MARKER_NAME = "corruption.marker"; private static final NotificationGroup NOTIFICATIONS = new NotificationGroup("Indexing", NotificationDisplayType.BALLOON, false); @@ -698,8 +699,13 @@ public final class FileBasedIndexImpl extends FileBasedIndex { public Map getFileData(@NotNull ID id, @NotNull VirtualFile virtualFile, @NotNull Project project) { if (!(virtualFile instanceof VirtualFileWithId)) return Collections.emptyMap(); int fileId = getFileId(virtualFile); - Map map = processExceptions(id, virtualFile, GlobalSearchScope.fileScope(project, virtualFile), index -> index.getIndexedFileData(fileId)); - return ContainerUtil.notNullize(map); + + // TODO revise behaviour later + if (getAccessibleFileIdFilter(project).test(fileId)) { + Map map = processExceptions(id, virtualFile, GlobalSearchScope.fileScope(project, virtualFile), index -> index.getIndexedFileData(fileId)); + return ContainerUtil.notNullize(map); + } + return Collections.emptyMap(); } private static final ThreadLocal myUpToDateCheckState = new ThreadLocal<>(); @@ -765,7 +771,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex { if (filter == GlobalSearchScope.EMPTY_SCOPE) { return; } - if (ourDumbModeIgnored.get() != Boolean.TRUE && ActionUtil.isDumbMode(project)) { + if (ourDumbModeAccessType.get() == null && ActionUtil.isDumbMode(project)) { handleDumbMode(project); } @@ -781,7 +787,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex { if (!RebuildStatus.isOk(indexId)) { throw new ServiceNotReadyException(); } - if (ourDumbModeIgnored.get() != Boolean.TRUE) { + if (ourDumbModeAccessType.get() == null) { forceUpdate(project, filter, restrictedFile); } if (!areUnsavedDocumentsIndexed(indexId)) { // todo: check scope ? @@ -903,11 +909,14 @@ public final class FileBasedIndexImpl extends FileBasedIndex { @NotNull K key, @NotNull Project project, @NotNull IdValueProcessor processor) { + IntPredicate accessibleFileFilter = getAccessibleFileIdFilter(project); + return processValueIterator(indexId, key, null, GlobalSearchScope.allScope(project), valueIt -> { while (valueIt.hasNext()) { V value = valueIt.next(); for (ValueContainer.IntIterator inputIdsIterator = valueIt.getInputIdsIterator(); inputIdsIterator.hasNext(); ) { - if (!processor.process(inputIdsIterator.next(), value)) { + int id = inputIdsIterator.next(); + if (accessibleFileFilter.test(id) && !processor.process(id, value)) { return false; } ProgressManager.checkCanceled(); @@ -956,6 +965,9 @@ public final class FileBasedIndexImpl extends FileBasedIndex { if (!(restrictToFile instanceof VirtualFileWithId)) return true; int restrictedFileId = getFileId(restrictToFile); + + if (!getAccessibleFileIdFilter(scope.getProject()).test(restrictedFileId)) return true; + return processValueIterator(indexId, dataKey, restrictToFile, scope, valueIt -> { while (valueIt.hasNext()) { V value = valueIt.next(); @@ -976,13 +988,14 @@ public final class FileBasedIndexImpl extends FileBasedIndex { @NotNull ValueProcessor processor) { PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); IdFilter filter = idFilter != null ? idFilter : projectIndexableFiles(scope.getProject()); + IntPredicate accessibleFileFilter = getAccessibleFileIdFilter(scope.getProject()); return processValueIterator(indexId, dataKey, null, scope, valueIt -> { while (valueIt.hasNext()) { final V value = valueIt.next(); for (final ValueContainer.IntIterator inputIdsIterator = valueIt.getInputIdsIterator(); inputIdsIterator.hasNext(); ) { final int id = inputIdsIterator.next(); - if (filter != null && !filter.containsFileId(id)) continue; + if (!accessibleFileFilter.test(id) || (filter != null && !filter.containsFileId(id))) continue; VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); if (file != null && scope.accept(file)) { if (!processor.process(file, value)) { @@ -1141,12 +1154,15 @@ public final class FileBasedIndexImpl extends FileBasedIndex { @NotNull final GlobalSearchScope filter, @Nullable final Condition valueChecker, @Nullable final ProjectIndexableFilesFilter projectFilesFilter) { + IntPredicate accessibleFileFilter = getAccessibleFileIdFilter(filter.getProject()); ThrowableConvertor, TIntHashSet, StorageException> convertor = index -> InvertedIndexUtil.collectInputIdsContainingAllKeys(index, dataKeys, __ -> { ProgressManager.checkCanceled(); return true; }, valueChecker, - projectFilesFilter == null ? null : projectFilesFilter::containsFileId); + projectFilesFilter == null ? accessibleFileFilter::test : id -> { + return projectFilesFilter.containsFileId(id) && accessibleFileFilter.test(id); + }); return processExceptions(indexId, null, filter, convertor); } @@ -1190,15 +1206,20 @@ public final class FileBasedIndexImpl extends FileBasedIndex { @ApiStatus.Internal @ApiStatus.Experimental @Override - public void ignoreDumbMode(@NotNull Runnable runnable, @NotNull Project project) { + public void ignoreDumbMode(@NotNull Runnable runnable, + @NotNull Project project, + @NotNull DumbModeAccessType dumbModeAccessType) { assert ApplicationManager.getApplication().isReadAccessAllowed(); if (DumbService.isDumb(project) && FileBasedIndex.isIndexAccessDuringDumbModeEnabled()) { - ourDumbModeIgnored.set(Boolean.TRUE); + if (ourDumbModeAccessType.get() != null) { + throw new AssertionError("reentrant dumb mode ignorance"); + } + ourDumbModeAccessType.set(dumbModeAccessType); try { runnable.run(); } finally { - ourDumbModeIgnored.set(null); + ourDumbModeAccessType.set(null); } } else { runnable.run(); @@ -1766,6 +1787,11 @@ public final class FileBasedIndexImpl extends FileBasedIndex { return file != null ? file : ourFileToBeIndexed.get(); } + @Override + public DumbModeAccessType getCurrentDumbModeAccessType() { + return ourDumbModeAccessType.get(); + } + private class VirtualFileUpdateTask extends UpdateTask { @Override void doProcess(VirtualFile item, Project project) { @@ -2025,7 +2051,11 @@ public final class FileBasedIndexImpl extends FileBasedIndex { } public boolean containsFile(VirtualFile file) { - return myFilesToUpdate.containsKey(getIdMaskingNonIdBasedFile(file)); + return containsFileId(getIdMaskingNonIdBasedFile(file)); + } + + public boolean containsFileId(int fileId) { + return myFilesToUpdate.containsKey(fileId); } Collection getAllFilesToUpdate() { @@ -2619,6 +2649,23 @@ public final class FileBasedIndexImpl extends FileBasedIndex { } } + @ApiStatus.Internal + @NotNull + public IntPredicate getAccessibleFileIdFilter(@Nullable Project project) { + boolean dumb = ActionUtil.isDumbMode(project); + if (!dumb) return f -> true; + + DumbModeAccessType dumbModeAccessType = getCurrentDumbModeAccessType(); + if (dumbModeAccessType == null) { + throw new IllegalStateException("index access is not allowed in dumb mode"); + } + + if (dumbModeAccessType == DumbModeAccessType.RAW_INDEX_DATA_ACCEPTABLE) return f -> true; + + assert dumbModeAccessType == DumbModeAccessType.RELIABLE_DATA_ONLY; + return fileId -> !getChangedFilesCollector().containsFileId(fileId); + } + @TestOnly public void waitForVfsEventsExecuted(long timeout, @NotNull TimeUnit unit) throws Exception { ApplicationManager.getApplication().assertIsDispatchThread();