From 6dcd00918494bd54cdc97e55cb19ff7fa097393c Mon Sep 17 00:00:00 2001 From: "Andrei.Kuznetsov" Date: Thu, 2 Jan 2025 00:00:52 +0100 Subject: [PATCH] IJPL-3020: invoke UnindexedFilesFinder from scanning directly, not via ScanningUtil GitOrigin-RevId: 9b1c621e92570b6b036b920f5b826b9ceb158ea9 --- .../intellij/util/indexing/ScanningUtil.java | 57 +------------------ .../util/indexing/UnindexedFileStatus.kt | 3 +- .../util/indexing/UnindexedFilesFinder.java | 29 ++++++++-- .../util/indexing/UnindexedFilesScanner.kt | 39 +++++++++---- .../indexing/diagnostic/ScanningStatistics.kt | 4 +- 5 files changed, 57 insertions(+), 75 deletions(-) diff --git a/platform/lang-impl/src/com/intellij/util/indexing/ScanningUtil.java b/platform/lang-impl/src/com/intellij/util/indexing/ScanningUtil.java index 0ea4e7c4e5a6..b6dc6cb38eb7 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/ScanningUtil.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/ScanningUtil.java @@ -2,17 +2,12 @@ package com.intellij.util.indexing; import com.intellij.openapi.application.ReadAction; -import com.intellij.openapi.progress.ProcessCanceledException; -import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.project.Project; -import com.intellij.openapi.roots.ContentIterator; import com.intellij.openapi.roots.impl.FilePropertyPusher; import com.intellij.openapi.roots.impl.PushedFilePropertiesUpdater; import com.intellij.openapi.roots.impl.PushedFilePropertiesUpdaterImpl; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.util.SmartList; -import com.intellij.util.indexing.IndexingProgressReporter.CheckPauseOnlyProgressIndicator; -import com.intellij.util.indexing.diagnostic.ScanningStatistics; import com.intellij.util.indexing.roots.IndexableFilesIterator; import com.intellij.util.indexing.roots.kind.IndexableSetOrigin; import com.intellij.util.indexing.roots.kind.ModuleContentOrigin; @@ -23,30 +18,18 @@ import java.util.List; import static com.intellij.openapi.roots.impl.PushedFilePropertiesUpdaterImpl.getImmediateValuesEx; import static com.intellij.openapi.roots.impl.PushedFilePropertiesUpdaterImpl.getModuleImmediateValues; -final class ScanningUtil implements ContentIterator { - private final Project project; - private final PerProjectIndexingQueue.PerProviderSink perProviderSink; - private final CheckPauseOnlyProgressIndicator indicator; +final class ScanningUtil { private final List> pushers; private final List> pusherExs; private final Object[] moduleValues; - private final UnindexedFilesFinder unindexedFileFinder; - private final ScanningStatistics scanningStatistics; private final PushedFilePropertiesUpdater pushedFilePropertiesUpdater; private final boolean mayBeUsed; - ScanningUtil(Project project, @NotNull CheckPauseOnlyProgressIndicator indicator, IndexableFilesIterator provider, - UnindexedFilesFinder unindexedFileFinder, ScanningStatistics scanningStatistics, - PerProjectIndexingQueue.PerProviderSink perProviderSink) { - this.project = project; - this.indicator = indicator; - this.unindexedFileFinder = unindexedFileFinder; - this.scanningStatistics = scanningStatistics; + ScanningUtil(Project project, IndexableFilesIterator provider) { pushedFilePropertiesUpdater = PushedFilePropertiesUpdater.getInstance(project); // We always need to properly dispose perProviderSink. Make this fact explicit to clients by requiring clients to provide an instance - this.perProviderSink = perProviderSink; IndexableSetOrigin origin = provider.getOrigin(); if (origin instanceof ModuleContentOrigin && !((ModuleContentOrigin)origin).getModule().isDisposed()) { @@ -86,46 +69,12 @@ final class ScanningUtil implements ContentIterator { return mayBeUsed; } - @Override - public boolean processFile(@NotNull VirtualFile fileOrDir) { - indicator.freezeIfPaused(); // give a chance to suspend indexing - ProgressManager.checkCanceled(); - - try { - processFileRethrowExceptions(fileOrDir); - } - catch (ProcessCanceledException pce) { - throw pce; - } - catch (Exception e) { - UnindexedFilesScanner.LOG.error("Error while scanning " + fileOrDir.getPresentableUrl() + "\n" + - "To reindex this file IDE has to be restarted", e); - } - return true; - } - - private void processFileRethrowExceptions(@NotNull VirtualFile fileOrDir) { + public void applyPushers(@NotNull VirtualFile fileOrDir) { if (pushers != null && pushedFilePropertiesUpdater instanceof PushedFilePropertiesUpdaterImpl) { ((PushedFilePropertiesUpdaterImpl)pushedFilePropertiesUpdater).applyPushersToFile(fileOrDir, pushers, moduleValues); } else if (pusherExs != null && pushedFilePropertiesUpdater instanceof PushedFilePropertiesUpdaterImpl) { ((PushedFilePropertiesUpdaterImpl)pushedFilePropertiesUpdater).applyPushersToFile(fileOrDir, pusherExs, moduleValues); } - - UnindexedFileStatus status; - long statusTime = System.nanoTime(); - try { - status = - UnindexedFilesScanner.ourTestMode == UnindexedFilesScanner.TestMode.PUSHING ? null : unindexedFileFinder.getFileStatus(fileOrDir); - } - finally { - statusTime = System.nanoTime() - statusTime; - } - if (status != null) { - if (status.getShouldIndex() && UnindexedFilesScanner.ourTestMode == null) { - perProviderSink.addFile(fileOrDir); - } - scanningStatistics.addStatus(fileOrDir, status, statusTime, project); - } } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFileStatus.kt b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFileStatus.kt index 63a55aebe0bd..d8e5029703fa 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFileStatus.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFileStatus.kt @@ -9,7 +9,8 @@ data class UnindexedFileStatus( val indexesWereProvidedByInfrastructureExtension: Boolean, val timeProcessingUpToDateFiles: Long, val timeUpdatingContentLessIndexes: Long, - val timeIndexingWithoutContentViaInfrastructureExtension: Long + val timeIndexingWithoutContentViaInfrastructureExtension: Long, + val timeTotal: Long ) { val wasFullyIndexedByInfrastructureExtension: Boolean get() = !shouldIndex && indexesWereProvidedByInfrastructureExtension } \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java index 68f2aee38d5f..1000bc326795 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java @@ -49,6 +49,7 @@ final class UnindexedFilesFinder { private static final class UnindexedFileStatusBuilder { boolean shouldIndex = false; boolean indexesWereProvidedByInfrastructureExtension = false; + long timeTotalEvaluation = 0; long timeProcessingUpToDateFiles = 0; long timeUpdatingContentLessIndexes = 0; long timeIndexingWithoutContentViaInfrastructureExtension = 0; @@ -88,7 +89,8 @@ final class UnindexedFilesFinder { indexesWereProvidedByInfrastructureExtension, timeProcessingUpToDateFiles, timeUpdatingContentLessIndexes, - timeIndexingWithoutContentViaInfrastructureExtension); + timeIndexingWithoutContentViaInfrastructureExtension, + timeTotalEvaluation); } void explain(IndexedFileImpl indexedFile, IndexingReasonExplanationLogger logger) { @@ -159,7 +161,22 @@ final class UnindexedFilesFinder { this.indexingRequest = indexingRequest; } - public @Nullable("null if the file is not subject for indexing (a directory, invalid, etc.)") UnindexedFileStatus getFileStatus(@NotNull VirtualFile file) { + @Nullable("null if the file is not subject for indexing (a directory, invalid, etc.)") + public UnindexedFileStatus getFileStatus(@NotNull VirtualFile file) { + long statusTime = System.nanoTime(); + UnindexedFileStatusBuilder status = null; + try { + status = evaluateFileStatus(file); + } + finally { + if (status != null) { + status.timeTotalEvaluation = System.nanoTime() - statusTime; + } + } + return status == null ? null : status.build(); + } + + private UnindexedFileStatusBuilder evaluateFileStatus(@NotNull VirtualFile file) { ProgressManager.checkCanceled(); // give a chance to suspend indexing if (!file.isValid() || !(file instanceof VirtualFileWithId)) { return null; @@ -171,7 +188,7 @@ final class UnindexedFilesFinder { if (TRUST_INDEXING_FLAG) { if (IndexingFlag.isFileIndexed(file, indexingStamp)) { myFilterHandler.addFileId(myProject, FileBasedIndex.getFileId(file)); - return new UnindexedFileStatusBuilder(applicationMode).build(); + return new UnindexedFileStatusBuilder(applicationMode); } } @@ -209,13 +226,13 @@ final class UnindexedFilesFinder { } if (!wasInvalidated) { IndexingStamp.flushCache(inputId); - return fileStatusBuilder.build(); + return fileStatusBuilder; } } FileTypeManagerEx ex = FileTypeManagerEx.getInstanceEx(); if (!(ex instanceof FileTypeManagerImpl)) { - return fileStatusBuilder.build(); + return fileStatusBuilder; } Ref finalization = new Ref<>(); ((FileTypeManagerImpl)ex).freezeFileTypeTemporarilyWithProvidedValueIn(file, fileType, () -> { @@ -295,7 +312,7 @@ final class UnindexedFilesFinder { finalization.get().run(); fileStatusBuilder.explain(indexedFile, explanationLogger); - return fileStatusBuilder.build(); + return fileStatusBuilder; }); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesScanner.kt b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesScanner.kt index df6060da58e7..702794b8a3a7 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesScanner.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesScanner.kt @@ -384,13 +384,15 @@ class UnindexedFilesScanner ( } } - internal class ScanningSession(private val project: Project, - private val scanningHistory: ProjectScanningHistoryImpl, - private val forceReindexingTrigger: BiPredicate?, - private val filterHandler: FilesFilterScanningHandler, - private val indicator: CheckPauseOnlyProgressIndicator, - private val progressReporter: IndexingProgressReporter, - private val scanningRequest: ScanningRequestToken) { + internal class ScanningSession( + private val project: Project, + private val scanningHistory: ProjectScanningHistoryImpl, + private val forceReindexingTrigger: BiPredicate?, + private val filterHandler: FilesFilterScanningHandler, + private val indicator: CheckPauseOnlyProgressIndicator, + private val progressReporter: IndexingProgressReporter, + private val scanningRequest: ScanningRequestToken, + ) { fun collectIndexableFilesConcurrently(providers: List) { if (providers.isEmpty()) { @@ -502,10 +504,11 @@ class UnindexedFilesScanner ( scanningStatistics.startFileChecking() try { readAction { - val finder = UnindexedFilesFinder(project, sharedExplanationLogger, forceReindexingTrigger, + val finder = + if (ourTestMode == TestMode.PUSHING) null + else UnindexedFilesFinder(project, sharedExplanationLogger, forceReindexingTrigger, scanningRequest, filterHandler) - val scanningUtil = ScanningUtil(project, indicator, provider, finder, - scanningStatistics, perProviderSink) + val scanningUtil = ScanningUtil(project, provider) if (!scanningUtil.mayBeUsed()) { LOG.warn("Iterator based on $provider can't be used.") return@readAction @@ -513,13 +516,25 @@ class UnindexedFilesScanner ( while (files.isNotEmpty()) { val file = files.removeFirst() try { - if (file.isValid) - scanningUtil.processFile(file) + if (file.isValid) { + scanningUtil.applyPushers(file) + val status = finder?.getFileStatus(file) + if (status != null) { + if (status.shouldIndex && ourTestMode == null) { + perProviderSink.addFile(file) + } + scanningStatistics.addStatus(file, status, project) + } + } } catch (e: ProcessCanceledException) { files.addFirst(file) throw e } + catch (e: Exception) { + LOG.error("Error while scanning ${file.presentableUrl}\n" + + "To reindex this file IDE has to be restarted", e); + } } } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/ScanningStatistics.kt b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/ScanningStatistics.kt index 5dad1025f47b..c87a7464c55b 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/ScanningStatistics.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/ScanningStatistics.kt @@ -61,13 +61,13 @@ class ScanningStatistics(val fileSetName: String) { } } - fun addStatus(fileOrDir: VirtualFile, unindexedFileStatus: UnindexedFileStatus, statusTime: Long, project: Project) { + fun addStatus(fileOrDir: VirtualFile, unindexedFileStatus: UnindexedFileStatus, project: Project) { if (fileOrDir.isDirectory) return numberOfScannedFiles++ if (unindexedFileStatus.shouldIndex) { numberOfFilesForIndexing++ } - this.statusTime += statusTime + this.statusTime += unindexedFileStatus.timeTotal timeProcessingUpToDateFiles += unindexedFileStatus.timeProcessingUpToDateFiles timeUpdatingContentLessIndexes += unindexedFileStatus.timeUpdatingContentLessIndexes