MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

implement indexing with CompositeDataIndexer

Browse Source 
GitOrigin-RevId: 14fc95f73f309d28843595716cc0c0e29b18eb76
This commit is contained in:
Dmitry Batkovich
2019-12-17 14:06:19 +00:00
committed by intellij-monorepo-bot
parent d3798ea0ef
commit 825d617f01
9 changed files with 178 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/core-api/src/com/intellij/ide/caches/FileContent.java
+9
View File
@@ -22,6 +22,15 @@ public class FileContent extends UserDataHolderBase {
myVirtualFile = virtualFile;
}
@NotNull
public byte[] getBytesOrEmptyArray() {
try {
return getBytes();
} catch (IOException e) {
return ArrayUtilRt.EMPTY_BYTE_ARRAY;
}
}
@NotNull
public byte[] getBytes() throws IOException {
if (myCachedBytes == null) {
platform/indexing-impl/src/com/intellij/util/indexing/IndexedFileWrapper.java
+71
View File
@@ -0,0 +1,71 @@
// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util.indexing;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiFile;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
class IndexedFileWrapper implements FileContent {
@NotNull
private final IndexedFile myFile;
IndexedFileWrapper(@NotNull IndexedFile file) {
myFile = file;
}
@NotNull
@Override
public byte[] getContent() {
throw new UnsupportedOperationException();
}
@NotNull
@Override
public CharSequence getContentAsText() {
throw new UnsupportedOperationException();
}
@NotNull
@Override
public PsiFile getPsiFile() {
throw new UnsupportedOperationException();
}
@NotNull
@Override
public FileType getFileType() {
return myFile.getFileType();
}
@NotNull
@Override
public VirtualFile getFile() {
return myFile.getFile();
}
@NotNull
@Override
public String getFileName() {
return myFile.getFileName();
}
@Override
public Project getProject() {
return myFile.getProject();
}
@Nullable
@Override
public <T> T getUserData(@NotNull Key<T> key) {
return myFile.getUserData(key);
}
@Override
public <T> void putUserData(@NotNull Key<T> key, @Nullable T value) {
myFile.putUserData(key, value);
}
}
platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java
+5 -5
View File
@@ -66,7 +66,7 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
FileBasedIndex fileBasedIndex = FileBasedIndex.getInstance();
if (file instanceof NewVirtualFile &&
fileBasedIndex instanceof FileBasedIndexImpl &&
((FileBasedIndexImpl)fileBasedIndex).getIndex(INDEX_ID).isIndexedStateForFile(((NewVirtualFile)file).getId(), file)) {
((FileBasedIndexImpl)fileBasedIndex).getIndex(INDEX_ID).isIndexedStateForFile(((NewVirtualFile)file).getId(), new IndexedFileImpl(file, fileType))) {
return true;
}
}
@@ -354,10 +354,10 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
}
@Override
public void setIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
public void setIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
super.setIndexedStateForFile(fileId, file);
try {
myStubVersionMap.persistIndexedState(fileId, file);
myStubVersionMap.persistIndexedState(fileId, file.getFile());
} catch (IOException e) {
LOG.error(e);
}
@@ -365,12 +365,12 @@ public class StubUpdatingIndex extends SingleEntryFileBasedIndexExtension<Serial
}
@Override
public boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
public boolean isIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
boolean indexedStateForFile = super.isIndexedStateForFile(fileId, file);
if (!indexedStateForFile) return false;
try {
return myStubVersionMap.isIndexed(fileId, file);
return myStubVersionMap.isIndexed(fileId, file.getFile());
}
catch (IOException e) {
LOG.error(e);
platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java
+22 -35
View File
@@ -1642,9 +1642,15 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
final VirtualFile file = content.getVirtualFile();
Ref<Boolean> setIndexedStatus = Ref.create(Boolean.TRUE);
getFileTypeManager().freezeFileTypeTemporarilyIn(file, () -> {
final Project finalProject = project == null ? ProjectUtil.guessProjectForFile(file) : project;
PsiFile psiFile = null;
FileContentImpl fc = null;
FileContentImpl fc = new FileContentImpl(file, content.getBytesOrEmptyArray());
PsiFile psiFile = content.getUserData(IndexingDataKeys.PSI_FILE);
initFileContent(fc, project == null ? ProjectUtil.guessProjectForFile(file) : project, psiFile);
if (FileBasedIndex.ourSnapshotMappingsEnabled) {
IndexedHashesSupport.initIndexedHash(fc);
}
int inputId = Math.abs(getFileId(file));
Set<ID<?, ?>> currentIndexedStates = new THashSet<>(IndexingStamp.getNontrivialFileIndexedStates(inputId));
@@ -1652,25 +1658,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
//noinspection ForLoopReplaceableByForEach
for (int i = 0, size = affectedIndexCandidates.size(); i < size; ++i) {
final ID<?, ?> indexId = affectedIndexCandidates.get(i);
if (shouldIndexFile(file, indexId)) {
if (fc == null) {
byte[] currentBytes;
try {
currentBytes = content.getBytes();
}
catch (IOException e) {
currentBytes = ArrayUtilRt.EMPTY_BYTE_ARRAY;
}
fc = new FileContentImpl(file, currentBytes);
psiFile = content.getUserData(IndexingDataKeys.PSI_FILE);
initFileContent(fc, finalProject, psiFile);
if (FileBasedIndex.ourSnapshotMappingsEnabled) {
IndexedHashesSupport.initIndexedHash(fc);
}
}
if (shouldIndexFile(fc, indexId)) {
try {
ProgressManager.checkCanceled();
if (!updateSingleIndex(indexId, file, inputId, fc)) {
@@ -1690,7 +1678,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
}
for(ID<?, ?> indexId:currentIndexedStates) {
if(!getIndex(indexId).isIndexedStateForFile(inputId, file)) {
if(!getIndex(indexId).isIndexedStateForFile(inputId, fc)) {
ProgressManager.checkCanceled();
if (!updateSingleIndex(indexId, file, inputId, null)) {
setIndexedStatus.set(Boolean.FALSE);
@@ -1748,7 +1736,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
final Computable<Boolean> update = index.update(inputId, currentFC);
updateCalculated = true;
runIndexUpdate(indexId, update, file, inputId);
runIndexUpdate(indexId, update, currentFC, inputId);
}
catch (RuntimeException exception) {
Throwable causeToRebuildIndex = getCauseToRebuildIndex(exception);
@@ -1824,7 +1812,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
private void runIndexUpdate(@NotNull ID<?, ?> indexId,
@NotNull Computable<Boolean> update,
@Nullable VirtualFile file,
@Nullable IndexedFile file,
int inputId) {
if (runUpdate(false, update)) {
ConcurrencyUtil.withLock(myReadLock, () -> {
@@ -2261,6 +2249,8 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
return true;
}
getFileTypeManager().freezeFileTypeTemporarilyIn(file, () -> {
IndexedFile fileContent = new IndexedFileImpl(file);
boolean isUptoDate = true;
boolean isDirectory = file.isDirectory();
if (!isDirectory && !isTooLarge(file)) {
@@ -2269,7 +2259,7 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
for (int i = 0, size = affectedIndexCandidates.size(); i < size; ++i) {
final ID<?, ?> indexId = affectedIndexCandidates.get(i);
try {
if (needsFileContentLoading(indexId) && shouldIndexFile(file, indexId)) {
if (needsFileContentLoading(indexId) && shouldIndexFile(fileContent, indexId)) {
if (myDoTraceForFilesToBeIndexed) {
LOG.trace("Scheduling indexing of " + file + " by request of index " + indexId);
}
@@ -2292,14 +2282,10 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
}
}
}
FileContent fileContent = null;
int inputId = Math.abs(getIdMaskingNonIdBasedFile(file));
for (ID<?, ?> indexId : isDirectory ? myRegisteredIndexes.getIndicesForDirectories() : myRegisteredIndexes.getNotRequiringContentIndices()) {
if (shouldIndexFile(file, indexId)) {
if (fileContent == null) {
fileContent = new FileContentImpl(file);
}
updateSingleIndex(indexId, file, inputId, fileContent);
if (shouldIndexFile(fileContent, indexId)) {
updateSingleIndex(indexId, file, inputId, new IndexedFileWrapper(fileContent));
}
}
IndexingStamp.flushCache(inputId);
@@ -2315,9 +2301,10 @@ public final class FileBasedIndexImpl extends FileBasedIndex {
}
}
private boolean shouldIndexFile(@NotNull VirtualFile file, @NotNull ID<?, ?> indexId) {
return getInputFilter(indexId).acceptInput(file) &&
(isMock(file) || !getIndex(indexId).isIndexedStateForFile(((NewVirtualFile)file).getId(), file));
private boolean shouldIndexFile(@NotNull IndexedFile file, @NotNull ID<?, ?> indexId) {
VirtualFile virtualFile = file.getFile();
return getInputFilter(indexId).acceptInput(virtualFile) &&
(isMock(virtualFile) || !getIndex(indexId).isIndexedStateForFile(((NewVirtualFile) virtualFile).getId(), file));
}
static boolean isMock(final VirtualFile file) {
platform/lang-impl/src/com/intellij/util/indexing/UpdatableIndex.java
+2 -3
View File
@@ -16,7 +16,6 @@
package com.intellij.util.indexing;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import com.intellij.util.indexing.impl.AbstractUpdateData;
@@ -48,10 +47,10 @@ public interface UpdatableIndex<Key, Value, Input> extends InvertedIndex<Key,Val
@NotNull
Map<Key, Value> getIndexedFileData(int fileId) throws StorageException;
void setIndexedStateForFile(int fileId, @NotNull VirtualFile file);
void setIndexedStateForFile(int fileId, @NotNull IndexedFile file);
void resetIndexedStateForFile(int fileId);
boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file);
boolean isIndexedStateForFile(int fileId, @NotNull IndexedFile file);
long getModificationStamp();
platform/lang-impl/src/com/intellij/util/indexing/VfsAwareMapReduceIndex.java
+61 -4
View File
@@ -8,13 +8,13 @@ import com.intellij.openapi.application.impl.ApplicationInfoImpl;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.util.io.ByteArraySequence;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.ConcurrencyUtil;
import com.intellij.util.Processor;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.indexing.impl.*;
import com.intellij.util.indexing.impl.forward.*;
import com.intellij.util.indexing.impl.perFileVersion.PersistentSubIndexerRetriever;
import com.intellij.util.indexing.snapshot.*;
import gnu.trove.THashSet;
import gnu.trove.TIntObjectHashMap;
@@ -48,6 +48,8 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
private final AtomicBoolean myInMemoryMode = new AtomicBoolean();
private final TIntObjectHashMap<Map<Key, Value>> myInMemoryKeysAndValues = new TIntObjectHashMap<>();
@SuppressWarnings("rawtypes")
private final PersistentSubIndexerRetriever mySubIndexerRetriever;
private final SnapshotInputMappingIndex<Key, Value, Input> mySnapshotInputMappings;
private final boolean myUpdateMappings;
private final boolean mySingleEntryIndex;
@@ -90,6 +92,19 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
((SnapshotSingleValueIndexStorage<Key, Value, Input>)backendStorage).init(snapshotInputMappings, ((IntForwardIndex)forwardIndexMap));
}
}
if (myIndexer instanceof CompositeDataIndexer && InvertedIndex.ARE_COMPOSITE_INDEXERS_ENABLED) {
try {
//noinspection unchecked,rawtypes,ConstantConditions
mySubIndexerRetriever = new PersistentSubIndexerRetriever((ID)myIndexId,
extension.getVersion(),
(CompositeDataIndexer) myIndexer);
}
catch (IOException e) {
throw new RuntimeException(e);
}
} else {
mySubIndexerRetriever = null;
}
mySnapshotInputMappings = IndexImporterMappingIndex.wrap(snapshotInputMappings, extension);
myUpdateMappings = mySnapshotInputMappings instanceof UpdatableSnapshotInputMappingIndex;
mySingleEntryIndex = extension instanceof SingleEntryFileBasedIndexExtension;
@@ -167,8 +182,16 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
}
@Override
public void setIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
public void setIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
IndexingStamp.setFileIndexedStateCurrent(fileId, (ID<?, ?>)myIndexId);
if (mySubIndexerRetriever != null) {
try {
mySubIndexerRetriever.setIndexedState(fileId, file);
}
catch (IOException e) {
LOG.error(e);
}
}
}
@Override
@@ -177,8 +200,23 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
}
@Override
public boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
return IndexingStamp.isFileIndexedStateCurrent(fileId, (ID<?, ?>)myIndexId);
public boolean isIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
if (!IndexingStamp.isFileIndexedStateCurrent(fileId, (ID<?, ?>)myIndexId)) {
return false;
}
if (mySubIndexerRetriever == null) return true;
if (!(file instanceof FileContent)) {
if (((CompositeDataIndexer)myIndexer).requiresContentForSubIndexerEvaluation(file)) {
return false;
}
}
try {
return mySubIndexerRetriever.isIndexed(fileId, file);
}
catch (IOException e) {
LOG.error(e);
return false;
}
}
@Override
@@ -343,6 +381,14 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
LOG.error(e);
}
}
if (mySubIndexerRetriever != null) {
try {
mySubIndexerRetriever.clear();
}
catch (IOException e) {
LOG.error(e);
}
}
}
@Override
@@ -351,6 +397,9 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
if (mySnapshotInputMappings != null && myUpdateMappings) {
((UpdatableSnapshotInputMappingIndex<Key, Value, Input>)mySnapshotInputMappings).flush();
}
if (mySubIndexerRetriever != null) {
mySubIndexerRetriever.flush();
}
}
@Override
@@ -365,6 +414,14 @@ public class VfsAwareMapReduceIndex<Key, Value, Input> extends MapReduceIndex<Ke
LOG.error(e);
}
}
if (mySubIndexerRetriever != null) {
try {
mySubIndexerRetriever.close();
}
catch (IOException e) {
LOG.error(e);
}
}
}
@Nullable
platform/lang-impl/src/com/intellij/util/indexing/hash/MergedInvertedIndex.java
+2 -3
View File
@@ -2,7 +2,6 @@
package com.intellij.util.indexing.hash;
import com.intellij.openapi.util.Computable;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.util.Processor;
import com.intellij.util.indexing.*;
@@ -129,7 +128,7 @@ public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Valu
}
@Override
public void setIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
public void setIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
myBaseIndex.setIndexedStateForFile(fileId, file);
}
@@ -139,7 +138,7 @@ public class MergedInvertedIndex<Key, Value> implements UpdatableIndex<Key, Valu
}
@Override
public boolean isIndexedStateForFile(int fileId, @NotNull VirtualFile file) {
public boolean isIndexedStateForFile(int fileId, @NotNull IndexedFile file) {
return myBaseIndex.isIndexedStateForFile(fileId, file);
}
platform/util/src/com/intellij/util/indexing/InvertedIndex.java
+5
View File
@@ -17,6 +17,8 @@
package com.intellij.util.indexing;
import com.intellij.openapi.util.Computable;
import com.intellij.util.SystemProperties;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -24,6 +26,9 @@ import org.jetbrains.annotations.Nullable;
* @author Eugene Zhuravlev
*/
public interface InvertedIndex<Key, Value, Input> {
@ApiStatus.Internal
boolean ARE_COMPOSITE_INDEXERS_ENABLED = SystemProperties.getBooleanProperty("com.intellij.composite.indexers", false);
@NotNull
ValueContainer<Value> getData(@NotNull Key key) throws StorageException;
platform/util/src/com/intellij/util/indexing/impl/MapReduceIndex.java
+1 -1
View File
@@ -49,7 +49,7 @@ public abstract class MapReduceIndex<Key,Value, Input> implements InvertedIndex<
protected final DataExternalizer<Value> myValueExternalizer;
protected final IndexExtension<Key, Value, Input> myExtension;
protected final AtomicLong myModificationStamp = new AtomicLong();
private final DataIndexer<Key, Value, Input> myIndexer;
protected final DataIndexer<Key, Value, Input> myIndexer;
private final ForwardIndex myForwardIndex;
private final ForwardIndexAccessor<Key, Value> myForwardIndexAccessor;