IDEA-333226: simplify the code: there is no need to capture indexing dependencies fingerprint when task is queued

We only need the fingerprint when the task is started, and we don't care that it might have changed while the task was in queue.

Real problem is that obtaining a fingerprint might be a long operation (e.g., when running from sources, we analyze classpath - hundreds of thousands of files). We want to make sure that we know the thread which is used to perform calculations - it should be indexing/scanning thread.

GitOrigin-RevId: 91625bd43e1e5d4be23baf5034505c8770c810ef
This commit is contained in:
Andrei.Kuznetsov
2023-09-25 11:58:39 +02:00
committed by intellij-monorepo-bot
parent f74f9d41b5
commit 5cb26bc2a1
11 changed files with 31 additions and 62 deletions

View File

@@ -21,7 +21,6 @@ import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.VirtualFileFilter
import com.intellij.util.SystemProperties
import com.intellij.util.indexing.UnindexedFilesIndexer
import com.intellij.util.indexing.dependencies.ProjectIndexingDependenciesService
import com.intellij.util.indexing.roots.IndexableEntityProviderMethods.createIterators
import com.intellij.util.indexing.roots.IndexableFilesIterator
import com.intellij.util.indexing.roots.IndexableSetContributorFilesIterator.Companion.createProjectUnAwareIndexableSetContributors
@@ -70,8 +69,7 @@ internal class SdkPreIndexingService: Disposable {
val task = object : Task.Backgroundable(null, JavaUiBundle.message("project.wizard.sdk.preindexing.progress.title")) {
override fun run(indicator: ProgressIndicator) {
UnindexedFilesIndexer(defaultProject, providers, "SDK pre-indexing", LongSets.emptySet(),
defaultProject.service<ProjectIndexingDependenciesService>().getLatestIndexingRequestToken()).perform(indicator)
UnindexedFilesIndexer(defaultProject, providers, "SDK pre-indexing", LongSets.emptySet()).perform(indicator)
}
}

View File

@@ -5,7 +5,6 @@ import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.Processor;
import com.intellij.util.indexing.dependencies.IndexingRequestToken;
import com.intellij.util.indexing.dependencies.ProjectIndexingDependenciesService;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
@@ -23,8 +22,8 @@ public final class FileBasedIndexProjectHandler {
mightHaveManyChangedFilesInProject(project)) {
String indexingReason = "On refresh of files in " + project.getName();
IndexingRequestToken rescan = project.getService(ProjectIndexingDependenciesService.class).invalidateAllStamps();
new UnindexedFilesIndexer(project, indexingReason, rescan).queue(project);
project.getService(ProjectIndexingDependenciesService.class).invalidateAllStamps(); // TODO-ank: do we need to invalidate explicitly?
new UnindexedFilesIndexer(project, indexingReason).queue(project);
}
}

View File

@@ -10,7 +10,6 @@ import com.intellij.openapi.progress.util.PingProgress
import com.intellij.openapi.progress.util.ProgressIndicatorUtils
import com.intellij.openapi.project.Project
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.util.indexing.dependencies.IndexingRequestToken
import com.intellij.util.indexing.diagnostic.IndexDiagnosticDumper
import com.intellij.util.indexing.diagnostic.ProjectDumbIndexingHistoryImpl
import com.intellij.util.indexing.roots.IndexableFilesIterator
@@ -184,24 +183,24 @@ class PerProjectIndexingQueue(private val project: Project) {
estimatedFilesCount.value = newValue
}
fun flushNow(reason: String, indexingRequest: IndexingRequestToken) {
fun flushNow(reason: String) {
val (filesInQueue, totalFiles, scanningIds) = getAndResetQueuedFiles()
if (totalFiles > 0) {
// note that DumbModeWhileScanningTrigger will not finish dumb mode until scanning is finished
UnindexedFilesIndexer(project, filesInQueue, reason, scanningIds, indexingRequest).queue(project)
UnindexedFilesIndexer(project, filesInQueue, reason, scanningIds).queue(project)
}
else {
LOG.info("Finished for " + project.name + ". No files to index with loading content.")
}
}
fun flushNowSync(indexingReason: String?, indicator: ProgressIndicator, indexingRequest: IndexingRequestToken) {
fun flushNowSync(indexingReason: String?, indicator: ProgressIndicator) {
val (filesInQueue, totalFiles, scanningIds) = getAndResetQueuedFiles()
if (totalFiles > 0) {
val projectDumbIndexingHistory = ProjectDumbIndexingHistoryImpl(project)
try {
UnindexedFilesIndexer(project, filesInQueue, indexingReason ?: "Flushing queue of project ${project.name}",
scanningIds, indexingRequest).indexFiles(
scanningIds).indexFiles(
projectDumbIndexingHistory, indicator)
}
catch (e: Throwable) {

View File

@@ -13,7 +13,6 @@ import com.intellij.psi.stubs.StubTreeBuilder
import com.intellij.psi.stubs.StubUpdatingIndex
import com.intellij.util.application
import com.intellij.util.indexing.dependencies.AppIndexingDependenciesService
import com.intellij.util.indexing.dependencies.ProjectIndexingDependenciesService
import com.intellij.util.indexing.diagnostic.ProjectScanningHistory
import com.intellij.util.indexing.diagnostic.ScanningType
import com.intellij.util.indexing.roots.IndexableFilesIterator
@@ -44,11 +43,9 @@ class RescanIndexesAction : RecoveryAction {
if (predefinedIndexableFilesIterators.isEmpty()) return emptyList()
}
application.service<AppIndexingDependenciesService>().invalidateAllStamps()
val rescan = project.service<ProjectIndexingDependenciesService>().getLatestIndexingRequestToken()
object : UnindexedFilesScanner(project, false, false,
predefinedIndexableFilesIterators, null, "Rescanning indexes recovery action",
if(predefinedIndexableFilesIterators == null) ScanningType.FULL_FORCED else ScanningType.PARTIAL_FORCED,
rescan) {
if(predefinedIndexableFilesIterators == null) ScanningType.FULL_FORCED else ScanningType.PARTIAL_FORCED) {
private val stubIndex =
runCatching { (FileBasedIndex.getInstance() as FileBasedIndexImpl).getIndex(StubUpdatingIndex.INDEX_ID) }
.onFailure { logger<RescanIndexesAction>().error(it) }.getOrNull()

View File

@@ -19,6 +19,7 @@ import com.intellij.util.gist.GistManager;
import com.intellij.util.gist.GistManagerImpl;
import com.intellij.util.indexing.contentQueue.IndexUpdateRunner;
import com.intellij.util.indexing.dependencies.IndexingRequestToken;
import com.intellij.util.indexing.dependencies.ProjectIndexingDependenciesService;
import com.intellij.util.indexing.diagnostic.IndexDiagnosticDumper;
import com.intellij.util.indexing.diagnostic.ProjectDumbIndexingHistoryImpl;
import com.intellij.util.indexing.roots.IndexableFilesIterator;
@@ -44,12 +45,10 @@ public final class UnindexedFilesIndexer extends DumbModeTask {
private final @NotNull Map<@NotNull IndexableFilesIterator, @NotNull Collection<@NotNull VirtualFile>> providerToFiles;
private final @NonNls @NotNull String indexingReason;
private final @NotNull LongSet scanningIds;
private final @NotNull IndexingRequestToken indexingRequest;
UnindexedFilesIndexer(@NotNull Project project,
@NonNls @NotNull String indexingReason,
@NotNull IndexingRequestToken indexingRequest) {
this(project, Collections.emptyMap(), indexingReason, LongSets.emptySet(), indexingRequest);
@NonNls @NotNull String indexingReason) {
this(project, Collections.emptyMap(), indexingReason, LongSets.emptySet());
}
/**
@@ -60,14 +59,12 @@ public final class UnindexedFilesIndexer extends DumbModeTask {
public UnindexedFilesIndexer(@NotNull Project project,
@NotNull Map<@NotNull IndexableFilesIterator, @NotNull Collection<@NotNull VirtualFile>> providerToFiles,
@NonNls @NotNull String indexingReason,
@NotNull LongSet scanningIds,
@NotNull IndexingRequestToken indexingRequest) {
@NotNull LongSet scanningIds) {
myProject = project;
myIndex = (FileBasedIndexImpl)FileBasedIndex.getInstance();
this.providerToFiles = providerToFiles;
this.indexingReason = indexingReason;
this.scanningIds = scanningIds;
this.indexingRequest = indexingRequest;
}
void indexFiles(@NotNull ProjectDumbIndexingHistoryImpl projectDumbIndexingHistory,
@@ -96,6 +93,7 @@ public final class UnindexedFilesIndexer extends DumbModeTask {
private void doIndexFiles(@NotNull ProjectDumbIndexingHistoryImpl projectDumbIndexingHistory,
@NotNull ProgressIndicator progressIndicator) {
int numberOfIndexingThreads = UnindexedFilesUpdater.getNumberOfIndexingThreads();
IndexingRequestToken indexingRequest = myProject.getService(ProjectIndexingDependenciesService.class).getLatestIndexingRequestToken();
IndexUpdateRunner indexUpdateRunner = new IndexUpdateRunner(myIndex, indexingRequest, numberOfIndexingThreads);
List<IndexUpdateRunner.FileSet> fileSets = getExplicitlyRequestedFilesSets();
@@ -237,7 +235,7 @@ public final class UnindexedFilesIndexer extends DumbModeTask {
LongArraySet ids = new LongArraySet(scanningIds.size() + otherIndexingTask.scanningIds.size());
ids.addAll(scanningIds);
ids.addAll(otherIndexingTask.scanningIds);
return new UnindexedFilesIndexer(myProject, mergedFilesToIndex, mergedReason, ids, indexingRequest.mergeWith(otherIndexingTask.indexingRequest));
return new UnindexedFilesIndexer(myProject, mergedFilesToIndex, mergedReason, ids);
}
@NotNull

View File

@@ -77,7 +77,6 @@ public class UnindexedFilesScanner extends FilesScanningTaskBase {
protected final Project myProject;
private final boolean myStartSuspended;
private final boolean myOnProjectOpen;
private final IndexingRequestToken indexingRequest;
private final @NotNull @NonNls String myIndexingReason;
private final @NotNull ScanningType myScanningType;
private final PushedFilePropertiesUpdater myPusher;
@@ -92,8 +91,7 @@ public class UnindexedFilesScanner extends FilesScanningTaskBase {
@Nullable List<IndexableFilesIterator> predefinedIndexableFilesIterators,
@Nullable StatusMark mark,
@Nullable @NonNls String indexingReason,
@NotNull ScanningType scanningType,
@NotNull IndexingRequestToken indexingRequest) {
@NotNull ScanningType scanningType) {
super(project);
myProject = project;
myStartSuspended = startSuspended;
@@ -108,7 +106,6 @@ public class UnindexedFilesScanner extends FilesScanningTaskBase {
if (isFullIndexUpdate()) {
myProject.putUserData(CONTENT_SCANNED, null);
}
this.indexingRequest = indexingRequest;
}
@Override
@@ -146,8 +143,7 @@ public class UnindexedFilesScanner extends FilesScanningTaskBase {
mergeIterators(myPredefinedIndexableFilesIterators, oldTask.myPredefinedIndexableFilesIterators),
StatusMark.mergeStatus(myProvidedStatusMark, oldTask.myProvidedStatusMark),
reason,
ScanningType.Companion.merge(oldTask.myScanningType, oldTask.myScanningType),
indexingRequest.mergeWith(oldTask.indexingRequest)
ScanningType.Companion.merge(oldTask.myScanningType, oldTask.myScanningType)
);
}
@@ -326,13 +322,12 @@ public class UnindexedFilesScanner extends FilesScanningTaskBase {
@NotNull CheckCancelOnlyProgressIndicator indicator) {
if (shouldScanInSmartMode()) {
// Switch to dumb mode and index
myProject.getService(PerProjectIndexingQueue.class).flushNow(myIndexingReason, indexingRequest);
myProject.getService(PerProjectIndexingQueue.class).flushNow(myIndexingReason);
}
else {
// Already in dumb mode. Just invoke indexer
myProject.getService(PerProjectIndexingQueue.class).flushNowSync(indexingReason,
indicator.originalIndicatorOnlyToFlushIndexingQueueSynchronously(),
indexingRequest);
indicator.originalIndicatorOnlyToFlushIndexingQueueSynchronously());
}
}
@@ -404,6 +399,7 @@ public class UnindexedFilesScanner extends FilesScanningTaskBase {
if (providers.isEmpty()) {
return;
}
IndexingRequestToken indexingRequest = myProject.getService(ProjectIndexingDependenciesService.class).getLatestIndexingRequestToken();
List<IndexableFileScanner.ScanSession> sessions =
ContainerUtil.map(IndexableFileScanner.EP_NAME.getExtensionList(), scanner -> scanner.startSession(project));
@@ -560,20 +556,18 @@ public class UnindexedFilesScanner extends FilesScanningTaskBase {
@Nullable @NonNls String indexingReason) {
FileBasedIndex.getInstance().loadIndexes();
((UserDataHolderEx)project).putUserDataIfAbsent(FIRST_SCANNING_REQUESTED, FirstScanningState.REQUESTED);
IndexingRequestToken indexingRequest = project.getService(ProjectIndexingDependenciesService.class).getLatestIndexingRequestToken();
if (TestModeFlags.is(INDEX_PROJECT_WITH_MANY_UPDATERS_TEST_KEY)) {
LOG.assertTrue(ApplicationManager.getApplication().isUnitTestMode());
List<IndexableFilesIterator> iterators = collectProviders(project, (FileBasedIndexImpl)FileBasedIndex.getInstance()).getFirst();
for (IndexableFilesIterator iterator : iterators) {
new UnindexedFilesScanner(project, startSuspended, true, Collections.singletonList(iterator), null, indexingReason,
ScanningType.FULL_ON_PROJECT_OPEN, indexingRequest).queue(project);
ScanningType.FULL_ON_PROJECT_OPEN).queue(project);
}
project.putUserData(CONTENT_SCANNED, true);
}
else {
new UnindexedFilesScanner(project, startSuspended, true, null, null, indexingReason, ScanningType.FULL_ON_PROJECT_OPEN,
indexingRequest).
queue(project);
new UnindexedFilesScanner(project, startSuspended, true, null, null, indexingReason, ScanningType.FULL_ON_PROJECT_OPEN)
.queue(project);
}
}

View File

@@ -6,8 +6,6 @@ import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.registry.Registry;
import com.intellij.util.SystemProperties;
import com.intellij.util.indexing.dependencies.IndexingRequestToken;
import com.intellij.util.indexing.dependencies.ProjectIndexingDependenciesService;
import com.intellij.util.indexing.dependenciesCache.DependenciesIndexedStatusService;
import com.intellij.util.indexing.diagnostic.ScanningType;
import com.intellij.util.indexing.roots.IndexableFilesIterator;
@@ -78,9 +76,8 @@ public final class UnindexedFilesUpdater {
}
public void queue() {
IndexingRequestToken indexingRequest = myProject.getService(ProjectIndexingDependenciesService.class).getLatestIndexingRequestToken();
new UnindexedFilesScanner(myProject, myStartSuspended, myOnProjectOpen, myPredefinedIndexableFilesIterators, myMark, myIndexingReason,
myScanningType, indexingRequest)
myScanningType)
.queue(myProject);
}

View File

@@ -8,5 +8,4 @@ interface IndexingRequestToken {
* Monotonically increasing number representing IndexingStamp
*/
fun getFileIndexingStamp(file: VirtualFile): FileIndexingStamp
fun mergeWith(other: IndexingRequestToken): IndexingRequestToken
}

View File

@@ -19,7 +19,6 @@ import java.nio.file.Path
import java.util.concurrent.atomic.AtomicReference
import java.util.function.IntConsumer
import kotlin.io.path.deleteIfExists
import kotlin.math.max
/**
* Service that tracks FileIndexingStamp.
@@ -92,11 +91,6 @@ class ProjectIndexingDependenciesService @NonInjectable @VisibleForTesting const
return getFileIndexingStamp(fileStamp)
}
override fun mergeWith(other: IndexingRequestToken): IndexingRequestToken {
return IndexingRequestTokenImpl(max(requestId, (other as IndexingRequestTokenImpl).requestId),
appIndexingRequest.mergeWith(other.appIndexingRequest))
}
@VisibleForTesting
fun getFileIndexingStamp(fileStamp: Int): FileIndexingStamp {
// we assume that stamp and file.modificationStamp never decrease => their sum only grow up

View File

@@ -12,8 +12,6 @@ import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileFilter;
import com.intellij.openapi.vfs.newvfs.impl.FakeVirtualFile;
import com.intellij.testFramework.LightPlatformTestCase;
import com.intellij.util.indexing.dependencies.IndexingRequestToken;
import com.intellij.util.indexing.dependencies.ProjectIndexingDependenciesService;
import com.intellij.util.indexing.diagnostic.ScanningType;
import com.intellij.util.indexing.roots.IndexableFilesIterator;
import com.intellij.util.indexing.roots.kind.IndexableSetOrigin;
@@ -56,9 +54,8 @@ public class ScanningIndexingTasksMergeTest extends LightPlatformTestCase {
map2.put(iter2, f2);
map2.put(iterShared, fShared.subList(1, 3));
IndexingRequestToken indexingRequest = getProject().getService(ProjectIndexingDependenciesService.class).getLatestIndexingRequestToken();
task1 = new UnindexedFilesIndexer(getProject(), map1, "test task1", LongSet.of(), indexingRequest);
task2 = new UnindexedFilesIndexer(getProject(), map2, "test task2", LongSet.of(), indexingRequest);
task1 = new UnindexedFilesIndexer(getProject(), map1, "test task1", LongSet.of());
task2 = new UnindexedFilesIndexer(getProject(), map2, "test task2", LongSet.of());
}
public void testTryMergeIndexingTasks() {
@@ -118,9 +115,8 @@ public class ScanningIndexingTasksMergeTest extends LightPlatformTestCase {
for (String[] situation : situations) {
IndexingRequestToken indexingRequest = getProject().getService(ProjectIndexingDependenciesService.class).getLatestIndexingRequestToken();
UnindexedFilesIndexer t1 = new UnindexedFilesIndexer(getProject(), situation[0], indexingRequest);
UnindexedFilesIndexer t2 = new UnindexedFilesIndexer(getProject(), situation[1], indexingRequest);
UnindexedFilesIndexer t1 = new UnindexedFilesIndexer(getProject(), situation[0]);
UnindexedFilesIndexer t2 = new UnindexedFilesIndexer(getProject(), situation[1]);
UnindexedFilesIndexer merged = t1.tryMergeWith(t2);
assertEquals(situation[2], merged.getIndexingReason());
}
@@ -154,8 +150,7 @@ public class ScanningIndexingTasksMergeTest extends LightPlatformTestCase {
@NotNull
private UnindexedFilesScanner createScanningTask(IndexableFilesIterator iter, String reason, ScanningType type) {
List<IndexableFilesIterator> iterators = iter == null ? null : Collections.singletonList(iter);
IndexingRequestToken indexingRequest = getProject().getService(ProjectIndexingDependenciesService.class).getLatestIndexingRequestToken();
return new UnindexedFilesScanner(getProject(), false, false, iterators, null, reason, type, indexingRequest);
return new UnindexedFilesScanner(getProject(), false, false, iterators, null, reason, type);
}
private void assertMergedStateInvariants(UnindexedFilesIndexer mergedTask) {

View File

@@ -317,8 +317,8 @@ class UnindexedFilesScannerTest {
}
private fun indexFiles(provider: SingleRootIndexableFilesIterator, dirtyFiles: Collection<VirtualFile>) {
val indexingStamp = project.service<ProjectIndexingDependenciesService>().invalidateAllStamps()
val indexingTask = UnindexedFilesIndexer(project, mapOf(provider to dirtyFiles), "Test", LongSet.of(), indexingStamp)
project.service<ProjectIndexingDependenciesService>().invalidateAllStamps()
val indexingTask = UnindexedFilesIndexer(project, mapOf(provider to dirtyFiles), "Test", LongSet.of())
val indicator = EmptyProgressIndicator()
ProgressManager.getInstance().runProcess({ indexingTask.perform(indicator) }, indicator)
}
@@ -345,8 +345,7 @@ class UnindexedFilesScannerTest {
private fun scanFiles(filesAndDirs: IndexableFilesIterator): Pair<ProjectScanningHistory, Map<IndexableFilesIterator, Collection<VirtualFile>>> {
val scanningHistoryRef = Ref<ProjectScanningHistory>()
val scanningTask = object : UnindexedFilesScanner(project, false, false, listOf(filesAndDirs), null, "Test", ScanningType.PARTIAL,
project.service<ProjectIndexingDependenciesService>().getLatestIndexingRequestToken()) {
val scanningTask = object : UnindexedFilesScanner(project, false, false, listOf(filesAndDirs), null, "Test", ScanningType.PARTIAL) {
override fun performScanningAndIndexing(indicator: CheckCancelOnlyProgressIndicator,
progressReporter: IndexingProgressReporter): ProjectScanningHistory {
return super.performScanningAndIndexing(indicator, progressReporter).also(scanningHistoryRef::set)