From 8f0bdd4e89275d3e4471b1f5c5cabd973a4f183c Mon Sep 17 00:00:00 2001 From: Dmitro Batko Date: Tue, 7 Apr 2020 09:30:27 +0300 Subject: [PATCH] indexes: do not send to up-date all the files on the first indexation GitOrigin-RevId: 092efe8ace44e22e6a23d29e2eb5aac68d5b7eb1 --- .../psi/search/FileTypeMapReduceIndex.java | 19 +++++++------ .../util/indexing/EmptyUpdatableIndex.kt | 4 +-- .../util/indexing/FileBasedIndexImpl.java | 27 ++++++++++--------- .../util/indexing/FileIndexingState.java | 15 +++++++++++ .../intellij/util/indexing/IndexingStamp.java | 8 +++--- .../util/indexing/UnindexedFilesFinder.java | 11 ++++---- .../util/indexing/UpdatableIndex.java | 3 ++- .../util/indexing/VfsAwareMapReduceIndex.java | 15 ++++++----- 8 files changed, 61 insertions(+), 41 deletions(-) create mode 100644 platform/lang-impl/src/com/intellij/util/indexing/FileIndexingState.java diff --git a/platform/lang-impl/src/com/intellij/psi/search/FileTypeMapReduceIndex.java b/platform/lang-impl/src/com/intellij/psi/search/FileTypeMapReduceIndex.java index 3984346bdab1..87e19bea1b04 100644 --- a/platform/lang-impl/src/com/intellij/psi/search/FileTypeMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/search/FileTypeMapReduceIndex.java @@ -3,16 +3,17 @@ package com.intellij.psi.search; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.fileTypes.FileType; -import com.intellij.openapi.util.Comparing; import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.indexing.*; +import com.intellij.util.indexing.FileBasedIndexExtension; +import com.intellij.util.indexing.FileIndexingState; +import com.intellij.util.indexing.IndexedFile; +import com.intellij.util.indexing.VfsAwareMapReduceIndex; import com.intellij.util.indexing.impl.IndexStorage; import com.intellij.util.indexing.impl.MapInputDataDiffBuilder; import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.Collection; -import java.util.Map; class FileTypeMapReduceIndex extends VfsAwareMapReduceIndex { private static final Logger LOG = Logger.getInstance(FileTypeIndexImpl.class); @@ -22,16 +23,18 @@ class FileTypeMapReduceIndex extends VfsAwareMapReduceIndex { } @Override - public boolean isIndexedStateForFile(int fileId, @NotNull IndexedFile file) { - boolean isIndexed = super.isIndexedStateForFile(fileId, file); - if (!isIndexed) return false; + public @NotNull FileIndexingState getIndexingStateForFile(int fileId, @NotNull IndexedFile file) { + @NotNull FileIndexingState isIndexed = super.getIndexingStateForFile(fileId, file); + if (isIndexed != FileIndexingState.UP_TO_DATE) return isIndexed; try { Collection inputData = ((MapInputDataDiffBuilder) getKeysDiffBuilder(fileId)).getKeys(); FileType indexedFileType = ContainerUtil.getFirstItem(inputData); - return FileTypeKeyDescriptor.INSTANCE.isEqual(indexedFileType, file.getFileType()); + return FileTypeKeyDescriptor.INSTANCE.isEqual(indexedFileType, file.getFileType()) + ? FileIndexingState.UP_TO_DATE + : FileIndexingState.OUT_DATED; } catch (IOException e) { LOG.error(e); - return false; + return FileIndexingState.OUT_DATED; } } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/EmptyUpdatableIndex.kt b/platform/lang-impl/src/com/intellij/util/indexing/EmptyUpdatableIndex.kt index 3e36f720d1b8..4d2c4abdd3cf 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/EmptyUpdatableIndex.kt +++ b/platform/lang-impl/src/com/intellij/util/indexing/EmptyUpdatableIndex.kt @@ -6,8 +6,6 @@ import com.intellij.psi.search.GlobalSearchScope import com.intellij.util.Processor import com.intellij.util.indexing.impl.AbstractUpdateData import com.intellij.util.indexing.snapshot.EmptyValueContainer -import java.lang.UnsupportedOperationException -import java.util.concurrent.locks.Lock import java.util.concurrent.locks.ReadWriteLock import java.util.concurrent.locks.ReentrantReadWriteLock @@ -28,7 +26,7 @@ class EmptyUpdatableIndex : UpdatableIndex override fun setIndexedStateForFile(fileId: Int, file: IndexedFile) = Unit override fun resetIndexedStateForFile(fileId: Int) {} - override fun isIndexedStateForFile(fileId: Int, file: IndexedFile): Boolean = false + override fun getIndexingStateForFile(fileId: Int, file: IndexedFile): FileIndexingState = FileIndexingState.NOT_INDEXED override fun getModificationStamp(): Long = 0 diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index cc77059d6299..8445f310e9e9 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -1199,7 +1199,7 @@ public final class FileBasedIndexImpl extends FileBasedIndexEx { try { ProgressManager.checkCanceled(); final ID indexId = affectedIndexCandidates.get(i); - if (shouldIndexFile(fc, indexId) == FileIndexingState.SHOULD_INDEX) { + if (getInputFilter(indexId).acceptInput(file) && getIndexingState(fc, indexId).updateRequired()) { ProgressManager.checkCanceled(); if (!updateSingleIndex(indexId, file, inputId, fc)) { setIndexedStatus.set(Boolean.FALSE); @@ -1219,7 +1219,7 @@ public final class FileBasedIndexImpl extends FileBasedIndexEx { for(ID indexId : currentIndexedStates) { ProgressManager.checkCanceled(); - if (!getIndex(indexId).isIndexedStateForFile(inputId, fc)) { + if (getIndex(indexId).getIndexingStateForFile(inputId, fc).updateRequired()) { ProgressManager.checkCanceled(); if (!updateSingleIndex(indexId, file, inputId, null)) { setIndexedStatus.set(Boolean.FALSE); @@ -1535,13 +1535,19 @@ public final class FileBasedIndexImpl extends FileBasedIndexEx { } FileIndexingState shouldIndexFile(@NotNull IndexedFile file, @NotNull ID indexId) { - VirtualFile virtualFile = file.getFile(); - if (!getInputFilter(indexId).acceptInput(virtualFile)) { - return FileIndexingState.SHOULD_NOT_INDEX; + if (!getInputFilter(indexId).acceptInput(file.getFile())) { + return getIndexingState(file, indexId) == FileIndexingState.NOT_INDEXED + ? FileIndexingState.UP_TO_DATE + : FileIndexingState.OUT_DATED; } - return (isMock(virtualFile) || !getIndex(indexId).isIndexedStateForFile(((NewVirtualFile) virtualFile).getId(), file)) - ? FileIndexingState.SHOULD_INDEX - : FileIndexingState.UP_TO_DATE; + return getIndexingState(file, indexId); + } + + @NotNull + private FileIndexingState getIndexingState(@NotNull IndexedFile file, @NotNull ID indexId) { + VirtualFile virtualFile = file.getFile(); + if (isMock(virtualFile)) return FileIndexingState.NOT_INDEXED; + return getIndex(indexId).getIndexingStateForFile(((NewVirtualFile)virtualFile).getId(), file); } static boolean isMock(final VirtualFile file) { @@ -1743,9 +1749,4 @@ public final class FileBasedIndexImpl extends FileBasedIndexEx { } return version; } - - @ApiStatus.Internal - enum FileIndexingState { - SHOULD_INDEX, SHOULD_NOT_INDEX, UP_TO_DATE - } } \ No newline at end of file diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileIndexingState.java b/platform/lang-impl/src/com/intellij/util/indexing/FileIndexingState.java new file mode 100644 index 000000000000..1977f26360a2 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileIndexingState.java @@ -0,0 +1,15 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing; + +import org.jetbrains.annotations.ApiStatus; + +@ApiStatus.Experimental +public enum FileIndexingState { + NOT_INDEXED, + OUT_DATED, + UP_TO_DATE; + + public boolean updateRequired() { + return this != UP_TO_DATE; + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java index 7b8910376ec2..709089243dce 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java @@ -195,9 +195,11 @@ public class IndexingStamp { return version; } - public static boolean isFileIndexedStateCurrent(int fileId, ID indexName) { + public static FileIndexingState isFileIndexedStateCurrent(int fileId, ID indexName) { try { - return getIndexStamp(fileId, indexName) == getIndexCreationStamp(indexName); + long stamp = getIndexStamp(fileId, indexName); + if (stamp == 0) return FileIndexingState.NOT_INDEXED; + return stamp == getIndexCreationStamp(indexName) ? FileIndexingState.UP_TO_DATE : FileIndexingState.OUT_DATED; } catch (RuntimeException e) { final Throwable cause = e.getCause(); @@ -206,7 +208,7 @@ public class IndexingStamp { } } - return false; + return FileIndexingState.OUT_DATED; } public static void setFileIndexedStateCurrent(int fileId, ID id) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java index c3608a7f9fc6..fd9cf3e16478 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesFinder.java @@ -59,8 +59,7 @@ class UnindexedFilesFinder implements VirtualFileFilter { boolean isDirectory = file.isDirectory(); int inputId = Math.abs(FileBasedIndexImpl.getIdMaskingNonIdBasedFile(file)); if (!isDirectory && !myFileBasedIndex.isTooLarge(file)) { - - if (!myFileTypeIndex.isIndexedStateForFile(inputId, fileContent)) { + if (myFileTypeIndex.getIndexingStateForFile(inputId, fileContent) == FileIndexingState.OUT_DATED) { myFileBasedIndex.dropNontrivialIndexedStates(inputId); shouldIndexFile.set(true); } else { @@ -70,10 +69,10 @@ class UnindexedFilesFinder implements VirtualFileFilter { final ID indexId = affectedIndexCandidates.get(i); try { if (myFileBasedIndex.needsFileContentLoading(indexId)) { - FileBasedIndexImpl.FileIndexingState fileIndexingState = myFileBasedIndex.shouldIndexFile(fileContent, indexId); - if (fileIndexingState == FileBasedIndexImpl.FileIndexingState.UP_TO_DATE) { + FileIndexingState fileIndexingState = myFileBasedIndex.shouldIndexFile(fileContent, indexId); + if (fileIndexingState == FileIndexingState.UP_TO_DATE) { myStateProcessors.forEach(p -> p.processUpToDateFile(file, inputId, indexId)); - } else if (fileIndexingState == FileBasedIndexImpl.FileIndexingState.SHOULD_INDEX) { + } else if (fileIndexingState.updateRequired()) { if (myDoTraceForFilesToBeIndexed) { LOG.trace("Scheduling indexing of " + file + " by request of index " + indexId); } @@ -97,7 +96,7 @@ class UnindexedFilesFinder implements VirtualFileFilter { } for (ID indexId : myFileBasedIndex.getContentLessIndexes(isDirectory)) { - if (myFileBasedIndex.shouldIndexFile(fileContent, indexId) == FileBasedIndexImpl.FileIndexingState.SHOULD_INDEX) { + if (myFileBasedIndex.shouldIndexFile(fileContent, indexId).updateRequired()) { myFileBasedIndex.updateSingleIndex(indexId, file, inputId, new IndexedFileWrapper(fileContent)); } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java index 9fcf1a4a00de..afe15ab9a7ec 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java @@ -54,7 +54,8 @@ public interface UpdatableIndex extends InvertedIndex extends MapReduceIndex)myIndexId)) { - return false; + public @NotNull FileIndexingState getIndexingStateForFile(int fileId, @NotNull IndexedFile file) { + FileIndexingState baseState = IndexingStamp.isFileIndexedStateCurrent(fileId, (ID)myIndexId); + if (baseState != FileIndexingState.UP_TO_DATE) { + return baseState; } - if (mySubIndexerRetriever == null) return true; + if (mySubIndexerRetriever == null) return FileIndexingState.UP_TO_DATE; if (!(file instanceof FileContent)) { if (((CompositeDataIndexer)myIndexer).requiresContentForSubIndexerEvaluation(file)) { - return isIndexConfigurationUpToDate(fileId, file); + return isIndexConfigurationUpToDate(fileId, file) ? FileIndexingState.UP_TO_DATE : FileIndexingState.OUT_DATED; } } try { - return mySubIndexerRetriever.isIndexed(fileId, file); + return mySubIndexerRetriever.isIndexed(fileId, file) ? FileIndexingState.UP_TO_DATE : FileIndexingState.OUT_DATED; } catch (IOException e) { LOG.error(e); - return false; + return FileIndexingState.OUT_DATED; } }