From 825d617f01864672dff0f77aabc3920e8a3a7fed Mon Sep 17 00:00:00 2001 From: Dmitry Batkovich Date: Tue, 17 Dec 2019 12:05:00 +0300 Subject: [PATCH] implement indexing with CompositeDataIndexer GitOrigin-RevId: 14fc95f73f309d28843595716cc0c0e29b18eb76 --- .../com/intellij/ide/caches/FileContent.java | 9 +++ .../util/indexing/IndexedFileWrapper.java | 71 +++++++++++++++++++ .../intellij/psi/stubs/StubUpdatingIndex.java | 10 +-- .../util/indexing/FileBasedIndexImpl.java | 57 ++++++--------- .../util/indexing/UpdatableIndex.java | 5 +- .../util/indexing/VfsAwareMapReduceIndex.java | 65 +++++++++++++++-- .../indexing/hash/MergedInvertedIndex.java | 5 +- .../intellij/util/indexing/InvertedIndex.java | 5 ++ .../util/indexing/impl/MapReduceIndex.java | 2 +- 9 files changed, 178 insertions(+), 51 deletions(-) create mode 100644 platform/indexing-impl/src/com/intellij/util/indexing/IndexedFileWrapper.java diff --git a/platform/core-api/src/com/intellij/ide/caches/FileContent.java b/platform/core-api/src/com/intellij/ide/caches/FileContent.java index bc95f138a9ac..b122ce5783f1 100644 --- a/platform/core-api/src/com/intellij/ide/caches/FileContent.java +++ b/platform/core-api/src/com/intellij/ide/caches/FileContent.java @@ -22,6 +22,15 @@ public class FileContent extends UserDataHolderBase { myVirtualFile = virtualFile; } + @NotNull + public byte[] getBytesOrEmptyArray() { + try { + return getBytes(); + } catch (IOException e) { + return ArrayUtilRt.EMPTY_BYTE_ARRAY; + } + } + @NotNull public byte[] getBytes() throws IOException { if (myCachedBytes == null) { diff --git a/platform/indexing-impl/src/com/intellij/util/indexing/IndexedFileWrapper.java b/platform/indexing-impl/src/com/intellij/util/indexing/IndexedFileWrapper.java new file mode 100644 index 000000000000..503725a1f0fe --- /dev/null +++ b/platform/indexing-impl/src/com/intellij/util/indexing/IndexedFileWrapper.java @@ -0,0 +1,71 @@ +// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.util.indexing; + +import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.psi.PsiFile; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +class IndexedFileWrapper implements FileContent { + @NotNull + private final IndexedFile myFile; + + IndexedFileWrapper(@NotNull IndexedFile file) { + myFile = file; + } + + @NotNull + @Override + public byte[] getContent() { + throw new UnsupportedOperationException(); + } + + @NotNull + @Override + public CharSequence getContentAsText() { + throw new UnsupportedOperationException(); + } + + @NotNull + @Override + public PsiFile getPsiFile() { + throw new UnsupportedOperationException(); + } + + @NotNull + @Override + public FileType getFileType() { + return myFile.getFileType(); + } + + @NotNull + @Override + public VirtualFile getFile() { + return myFile.getFile(); + } + + @NotNull + @Override + public String getFileName() { + return myFile.getFileName(); + } + + @Override + public Project getProject() { + return myFile.getProject(); + } + + @Nullable + @Override + public T getUserData(@NotNull Key key) { + return myFile.getUserData(key); + } + + @Override + public void putUserData(@NotNull Key key, @Nullable T value) { + myFile.putUserData(key, value); + } +} diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index fb194e6ac44f..53c54be54ecc 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -66,7 +66,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension setIndexedStatus = Ref.create(Boolean.TRUE); getFileTypeManager().freezeFileTypeTemporarilyIn(file, () -> { - final Project finalProject = project == null ? ProjectUtil.guessProjectForFile(file) : project; - PsiFile psiFile = null; - FileContentImpl fc = null; + + FileContentImpl fc = new FileContentImpl(file, content.getBytesOrEmptyArray()); + PsiFile psiFile = content.getUserData(IndexingDataKeys.PSI_FILE); + initFileContent(fc, project == null ? ProjectUtil.guessProjectForFile(file) : project, psiFile); + + if (FileBasedIndex.ourSnapshotMappingsEnabled) { + IndexedHashesSupport.initIndexedHash(fc); + } + int inputId = Math.abs(getFileId(file)); Set> currentIndexedStates = new THashSet<>(IndexingStamp.getNontrivialFileIndexedStates(inputId)); @@ -1652,25 +1658,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex { //noinspection ForLoopReplaceableByForEach for (int i = 0, size = affectedIndexCandidates.size(); i < size; ++i) { final ID indexId = affectedIndexCandidates.get(i); - if (shouldIndexFile(file, indexId)) { - if (fc == null) { - byte[] currentBytes; - try { - currentBytes = content.getBytes(); - } - catch (IOException e) { - currentBytes = ArrayUtilRt.EMPTY_BYTE_ARRAY; - } - fc = new FileContentImpl(file, currentBytes); - - psiFile = content.getUserData(IndexingDataKeys.PSI_FILE); - initFileContent(fc, finalProject, psiFile); - - if (FileBasedIndex.ourSnapshotMappingsEnabled) { - IndexedHashesSupport.initIndexedHash(fc); - } - } - + if (shouldIndexFile(fc, indexId)) { try { ProgressManager.checkCanceled(); if (!updateSingleIndex(indexId, file, inputId, fc)) { @@ -1690,7 +1678,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex { } for(ID indexId:currentIndexedStates) { - if(!getIndex(indexId).isIndexedStateForFile(inputId, file)) { + if(!getIndex(indexId).isIndexedStateForFile(inputId, fc)) { ProgressManager.checkCanceled(); if (!updateSingleIndex(indexId, file, inputId, null)) { setIndexedStatus.set(Boolean.FALSE); @@ -1748,7 +1736,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex { final Computable update = index.update(inputId, currentFC); updateCalculated = true; - runIndexUpdate(indexId, update, file, inputId); + runIndexUpdate(indexId, update, currentFC, inputId); } catch (RuntimeException exception) { Throwable causeToRebuildIndex = getCauseToRebuildIndex(exception); @@ -1824,7 +1812,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex { private void runIndexUpdate(@NotNull ID indexId, @NotNull Computable update, - @Nullable VirtualFile file, + @Nullable IndexedFile file, int inputId) { if (runUpdate(false, update)) { ConcurrencyUtil.withLock(myReadLock, () -> { @@ -2261,6 +2249,8 @@ public final class FileBasedIndexImpl extends FileBasedIndex { return true; } getFileTypeManager().freezeFileTypeTemporarilyIn(file, () -> { + IndexedFile fileContent = new IndexedFileImpl(file); + boolean isUptoDate = true; boolean isDirectory = file.isDirectory(); if (!isDirectory && !isTooLarge(file)) { @@ -2269,7 +2259,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex { for (int i = 0, size = affectedIndexCandidates.size(); i < size; ++i) { final ID indexId = affectedIndexCandidates.get(i); try { - if (needsFileContentLoading(indexId) && shouldIndexFile(file, indexId)) { + if (needsFileContentLoading(indexId) && shouldIndexFile(fileContent, indexId)) { if (myDoTraceForFilesToBeIndexed) { LOG.trace("Scheduling indexing of " + file + " by request of index " + indexId); } @@ -2292,14 +2282,10 @@ public final class FileBasedIndexImpl extends FileBasedIndex { } } } - FileContent fileContent = null; int inputId = Math.abs(getIdMaskingNonIdBasedFile(file)); for (ID indexId : isDirectory ? myRegisteredIndexes.getIndicesForDirectories() : myRegisteredIndexes.getNotRequiringContentIndices()) { - if (shouldIndexFile(file, indexId)) { - if (fileContent == null) { - fileContent = new FileContentImpl(file); - } - updateSingleIndex(indexId, file, inputId, fileContent); + if (shouldIndexFile(fileContent, indexId)) { + updateSingleIndex(indexId, file, inputId, new IndexedFileWrapper(fileContent)); } } IndexingStamp.flushCache(inputId); @@ -2315,9 +2301,10 @@ public final class FileBasedIndexImpl extends FileBasedIndex { } } - private boolean shouldIndexFile(@NotNull VirtualFile file, @NotNull ID indexId) { - return getInputFilter(indexId).acceptInput(file) && - (isMock(file) || !getIndex(indexId).isIndexedStateForFile(((NewVirtualFile)file).getId(), file)); + private boolean shouldIndexFile(@NotNull IndexedFile file, @NotNull ID indexId) { + VirtualFile virtualFile = file.getFile(); + return getInputFilter(indexId).acceptInput(virtualFile) && + (isMock(virtualFile) || !getIndex(indexId).isIndexedStateForFile(((NewVirtualFile) virtualFile).getId(), file)); } static boolean isMock(final VirtualFile file) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java index b376367845c2..58e0835b5d2e 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java @@ -16,7 +16,6 @@ package com.intellij.util.indexing; -import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.Processor; import com.intellij.util.indexing.impl.AbstractUpdateData; @@ -48,10 +47,10 @@ public interface UpdatableIndex extends InvertedIndex getIndexedFileData(int fileId) throws StorageException; - void setIndexedStateForFile(int fileId, @NotNull VirtualFile file); + void setIndexedStateForFile(int fileId, @NotNull IndexedFile file); void resetIndexedStateForFile(int fileId); - boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file); + boolean isIndexedStateForFile(int fileId, @NotNull IndexedFile file); long getModificationStamp(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java index 59ac85ed5a54..e4b7a551e699 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java @@ -8,13 +8,13 @@ import com.intellij.openapi.application.impl.ApplicationInfoImpl; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.util.io.ByteArraySequence; -import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.search.GlobalSearchScope; import com.intellij.util.ConcurrencyUtil; import com.intellij.util.Processor; import com.intellij.util.containers.ContainerUtil; import com.intellij.util.indexing.impl.*; import com.intellij.util.indexing.impl.forward.*; +import com.intellij.util.indexing.impl.perFileVersion.PersistentSubIndexerRetriever; import com.intellij.util.indexing.snapshot.*; import gnu.trove.THashSet; import gnu.trove.TIntObjectHashMap; @@ -48,6 +48,8 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex> myInMemoryKeysAndValues = new TIntObjectHashMap<>(); + @SuppressWarnings("rawtypes") + private final PersistentSubIndexerRetriever mySubIndexerRetriever; private final SnapshotInputMappingIndex mySnapshotInputMappings; private final boolean myUpdateMappings; private final boolean mySingleEntryIndex; @@ -90,6 +92,19 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex)backendStorage).init(snapshotInputMappings, ((IntForwardIndex)forwardIndexMap)); } } + if (myIndexer instanceof CompositeDataIndexer && InvertedIndex.ARE_COMPOSITE_INDEXERS_ENABLED) { + try { + //noinspection unchecked,rawtypes,ConstantConditions + mySubIndexerRetriever = new PersistentSubIndexerRetriever((ID)myIndexId, + extension.getVersion(), + (CompositeDataIndexer) myIndexer); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } else { + mySubIndexerRetriever = null; + } mySnapshotInputMappings = IndexImporterMappingIndex.wrap(snapshotInputMappings, extension); myUpdateMappings = mySnapshotInputMappings instanceof UpdatableSnapshotInputMappingIndex; mySingleEntryIndex = extension instanceof SingleEntryFileBasedIndexExtension; @@ -167,8 +182,16 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex)myIndexId); + if (mySubIndexerRetriever != null) { + try { + mySubIndexerRetriever.setIndexedState(fileId, file); + } + catch (IOException e) { + LOG.error(e); + } + } } @Override @@ -177,8 +200,23 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex)myIndexId); + public boolean isIndexedStateForFile(int fileId, @NotNull IndexedFile file) { + if (!IndexingStamp.isFileIndexedStateCurrent(fileId, (ID)myIndexId)) { + return false; + } + if (mySubIndexerRetriever == null) return true; + if (!(file instanceof FileContent)) { + if (((CompositeDataIndexer)myIndexer).requiresContentForSubIndexerEvaluation(file)) { + return false; + } + } + try { + return mySubIndexerRetriever.isIndexed(fileId, file); + } + catch (IOException e) { + LOG.error(e); + return false; + } } @Override @@ -343,6 +381,14 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex extends MapReduceIndex)mySnapshotInputMappings).flush(); } + if (mySubIndexerRetriever != null) { + mySubIndexerRetriever.flush(); + } } @Override @@ -365,6 +414,14 @@ public class VfsAwareMapReduceIndex extends MapReduceIndex implements UpdatableIndex implements UpdatableIndex { + @ApiStatus.Internal + boolean ARE_COMPOSITE_INDEXERS_ENABLED = SystemProperties.getBooleanProperty("com.intellij.composite.indexers", false); + @NotNull ValueContainer getData(@NotNull Key key) throws StorageException; diff --git a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java index 7ba599eb4ce4..c492f166b961 100644 --- a/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java +++ b/platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java @@ -49,7 +49,7 @@ public abstract class MapReduceIndex implements InvertedIndex< protected final DataExternalizer myValueExternalizer; protected final IndexExtension myExtension; protected final AtomicLong myModificationStamp = new AtomicLong(); - private final DataIndexer myIndexer; + protected final DataIndexer myIndexer; private final ForwardIndex myForwardIndex; private final ForwardIndexAccessor myForwardIndexAccessor;