mirror of
https://gitflic.ru/project/openide/openide.git
synced 2026-09-27 10:03:11 +07:00
introduced INDEX_DATA_OUTDATED saved indexing state that is set on index invalidation upon events.
It allows us to clean index data properly upon file deletion and also avoid index clearing for changed files upon shutdown (IDEA-118211)
This commit is contained in:
Maxim.Mossienko
@@ -33,6 +33,7 @@ import java.io.*;
|
||||
public class ID<K, V> {
|
||||
private static final TIntObjectHashMap<ID> ourRegistry = new TIntObjectHashMap<ID>();
|
||||
private static final TObjectIntHashMap<String> ourNameToIdRegistry = new TObjectIntHashMap<String>();
|
||||
public static final int MAX_NUMBER_OF_INDICES = Short.MAX_VALUE;
|
||||
|
||||
private final String myName;
|
||||
private final short myUniqueId;
|
||||
@@ -80,7 +81,7 @@ public class ID<K, V> {
|
||||
}
|
||||
|
||||
int n = ourNameToIdRegistry.size() + 1;
|
||||
assert n <= Short.MAX_VALUE : "Number of indices exceeded";
|
||||
assert n <= MAX_NUMBER_OF_INDICES : "Number of indices exceeded";
|
||||
|
||||
ourNameToIdRegistry.put(name, n);
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
|
||||
final boolean versionFileExisted = versionFile.exists();
|
||||
final File indexRootDir = IndexInfrastructure.getIndexRootDir(indexKey);
|
||||
boolean needRebuild = false;
|
||||
if (forceClean || IndexInfrastructure.versionDiffers(versionFile, version)) {
|
||||
if (forceClean || IndexingStamp.versionDiffers(versionFile, version)) {
|
||||
final String[] children = indexRootDir.list();
|
||||
// rebuild only if there exists what to rebuild
|
||||
needRebuild = !forceClean && (versionFileExisted || children != null && children.length > 0);
|
||||
@@ -135,7 +135,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
|
||||
LOG.info("Version has changed for stub index " + extension.getKey() + ". The index will be rebuilt.");
|
||||
}
|
||||
FileUtil.delete(indexRootDir);
|
||||
IndexInfrastructure.rewriteVersion(versionFile, version);
|
||||
IndexingStamp.rewriteVersion(versionFile, version);
|
||||
}
|
||||
|
||||
for (int attempt = 0; attempt < 2; attempt++) {
|
||||
@@ -157,7 +157,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe
|
||||
LOG.info(e);
|
||||
needRebuild = true;
|
||||
FileUtil.delete(indexRootDir);
|
||||
IndexInfrastructure.rewriteVersion(versionFile, version);
|
||||
IndexingStamp.rewriteVersion(versionFile, version);
|
||||
}
|
||||
}
|
||||
return needRebuild;
|
||||
|
||||
@@ -86,7 +86,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi
|
||||
if (((IStubFileElementType)elementType).shouldBuildStubFor(file)) {
|
||||
return true;
|
||||
}
|
||||
if (IndexingStamp.isFileIndexed(file, INDEX_ID, IndexInfrastructure.getIndexCreationStamp(INDEX_ID))) {
|
||||
if (IndexingStamp.isFileIndexedStateCurrent(file, INDEX_ID)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,7 +126,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
private final MessageBusConnection myConnection;
|
||||
private final FileDocumentManager myFileDocumentManager;
|
||||
private final FileTypeManager myFileTypeManager;
|
||||
private final ConcurrentHashSet<ID<?, ?>> myUpToDateIndices = new ConcurrentHashSet<ID<?, ?>>();
|
||||
private final ConcurrentHashSet<ID<?, ?>> myUpToDateIndicesForUnsavedOrTransactedDocuments = new ConcurrentHashSet<ID<?, ?>>();
|
||||
private volatile SmartFMap<Document, PsiFile> myTransactionMap = SmartFMap.emptyMap();
|
||||
|
||||
@Nullable private final String myConfigPath;
|
||||
@@ -158,7 +158,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
public void transactionStarted(final Document doc, final PsiFile file) {
|
||||
if (file != null) {
|
||||
myTransactionMap = myTransactionMap.plus(doc, file);
|
||||
myUpToDateIndices.clear();
|
||||
myUpToDateIndicesForUnsavedOrTransactedDocuments.clear();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -244,7 +244,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
ApplicationManager.getApplication().addApplicationListener(new ApplicationAdapter() {
|
||||
@Override
|
||||
public void writeActionStarted(Object action) {
|
||||
myUpToDateIndices.clear();
|
||||
myUpToDateIndicesForUnsavedOrTransactedDocuments.clear();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -378,13 +378,13 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
final File versionFile = IndexInfrastructure.getVersionFile(name);
|
||||
final boolean versionFileExisted = versionFile.exists();
|
||||
boolean versionChanged = false;
|
||||
if (isCurrentVersionCorrupted || IndexInfrastructure.versionDiffers(versionFile, version)) {
|
||||
if (isCurrentVersionCorrupted || IndexingStamp.versionDiffers(versionFile, version)) {
|
||||
if (!isCurrentVersionCorrupted && versionFileExisted) {
|
||||
versionChanged = true;
|
||||
LOG.info("Version has changed for index " + name + ". The index will be rebuilt.");
|
||||
}
|
||||
FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
|
||||
IndexInfrastructure.rewriteVersion(versionFile, version);
|
||||
IndexingStamp.rewriteVersion(versionFile, version);
|
||||
}
|
||||
|
||||
initIndexStorage(extension, version, versionFile);
|
||||
@@ -411,8 +411,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
final UpdatableIndex<K, V, FileContent> index = createIndex(name, extension, memStorage);
|
||||
final InputFilter inputFilter = extension.getInputFilter();
|
||||
|
||||
assert inputFilter != null : "Index extension " + name + " must provide non-null input filter";
|
||||
|
||||
myIndices.put(name, new Pair<UpdatableIndex<?, ?, FileContent>, InputFilter>(index, new IndexableFilesFilter(inputFilter)));
|
||||
if (inputFilter instanceof FileTypeSpecificInputFilter) {
|
||||
((FileTypeSpecificInputFilter)inputFilter).registerFileTypesUsedForIndexing(new Consumer<FileType>() {
|
||||
@@ -455,7 +453,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
}
|
||||
|
||||
FileUtil.delete(IndexInfrastructure.getIndexRootDir(name));
|
||||
IndexInfrastructure.rewriteVersion(versionFile, version);
|
||||
IndexingStamp.rewriteVersion(versionFile, version);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -660,7 +658,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
finally {
|
||||
LOG.info("START INDEX SHUTDOWN");
|
||||
try {
|
||||
myChangedFilesCollector.forceUpdate(null, null, null, true);
|
||||
myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
|
||||
IndexingStamp.flushCaches();
|
||||
|
||||
for (ID<?, ?> indexId : myIndices.keySet()) {
|
||||
@@ -813,7 +811,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
if (isUpToDateCheckEnabled()) {
|
||||
try {
|
||||
checkRebuild(indexId, false);
|
||||
myChangedFilesCollector.forceUpdate(project, filter, restrictedFile, false);
|
||||
myChangedFilesCollector.forceUpdate(project, filter, restrictedFile);
|
||||
indexUnsavedDocuments(indexId, project, filter, restrictedFile);
|
||||
}
|
||||
catch (StorageException e) {
|
||||
@@ -1395,7 +1393,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
|
||||
private void advanceIndexVersion(ID<?, ?> indexId) {
|
||||
try {
|
||||
IndexInfrastructure.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId));
|
||||
IndexingStamp.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId));
|
||||
}
|
||||
catch (IOException e) {
|
||||
LOG.error(e);
|
||||
@@ -1416,7 +1414,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
@Nullable Project project,
|
||||
GlobalSearchScope filter,
|
||||
VirtualFile restrictedFile) throws StorageException {
|
||||
if (myUpToDateIndices.contains(indexId)) {
|
||||
if (myUpToDateIndicesForUnsavedOrTransactedDocuments.contains(indexId)) {
|
||||
return; // no need to index unsaved docs
|
||||
}
|
||||
|
||||
@@ -1458,7 +1456,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
ProgressManager.checkCanceled();
|
||||
// assume all tasks were finished or cancelled in the same time
|
||||
// safe to set the flag here, because it will be cleared under the WriteAction
|
||||
myUpToDateIndices.add(indexId);
|
||||
myUpToDateIndicesForUnsavedOrTransactedDocuments.add(indexId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1700,7 +1698,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
|
||||
void processRefreshedFile(@NotNull Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) {
|
||||
myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
|
||||
myChangedFilesCollector.processFileImpl(project, fileContent, false); // ProcessCanceledException will cause re-adding the file to processing list
|
||||
myChangedFilesCollector.processFileImpl(project, fileContent); // ProcessCanceledException will cause re-adding the file to processing list
|
||||
}
|
||||
|
||||
public void indexFileContent(@Nullable Project project, @NotNull com.intellij.ide.caches.FileContent content) {
|
||||
@@ -1837,11 +1835,10 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
public void run() {
|
||||
if (file.isValid()) {
|
||||
if (hasContent) {
|
||||
IndexingStamp.update(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId));
|
||||
IndexingStamp.setFileIndexedStateCurrent(file, indexId);
|
||||
}
|
||||
else {
|
||||
// mark the file as unindexed
|
||||
IndexingStamp.update(file, indexId, IndexInfrastructure.INVALID_STAMP);
|
||||
IndexingStamp.setFileIndexedStateUnindexed(file, indexId);
|
||||
}
|
||||
if (myNotRequiringContentIndices.contains(indexId)) IndexingStamp.flushCache(file);
|
||||
}
|
||||
@@ -2028,8 +2025,8 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
for (int i = 0, size = candidates.size(); i < size; ++i) {
|
||||
final ID<?, ?> indexId = candidates.get(i);
|
||||
if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) {
|
||||
if (IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId))) {
|
||||
IndexingStamp.update(file, indexId, IndexInfrastructure.INVALID_STAMP2);
|
||||
if (IndexingStamp.isFileIndexedStateCurrent(file, indexId)) {
|
||||
IndexingStamp.setFileIndexedStateOutdated(file, indexId);
|
||||
resetStamp = true;
|
||||
}
|
||||
scheduleForUpdate = true;
|
||||
@@ -2091,52 +2088,50 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
cleanProcessedFlag(file);
|
||||
IndexingStamp.flushCache(file);
|
||||
|
||||
Collection<ID<?, ?>> existingIndexedIds = IndexingStamp.getIndexedIds(file);
|
||||
List<ID<?, ?>> nontrivialFileIndexedStates = IndexingStamp.getNontrivialFileIndexedStates(file);
|
||||
|
||||
for(ID<?, ?> indexId:existingIndexedIds) {
|
||||
for (ID<?, ?> indexId : nontrivialFileIndexedStates) {
|
||||
if (myNotRequiringContentIndices.contains(indexId)) {
|
||||
try {
|
||||
updateSingleIndex(indexId, file, null);
|
||||
} catch (StorageException e) {
|
||||
}
|
||||
catch (StorageException e) {
|
||||
LOG.info(e);
|
||||
requestRebuild(indexId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Collection<ID<?, ?>> indexedIdsToUpdate = ContainerUtil.intersection(existingIndexedIds, myRequiringContentIndices);
|
||||
Collection<ID<?, ?>> fileIndexedStatesToUpdate = ContainerUtil.intersection(nontrivialFileIndexedStates, myRequiringContentIndices);
|
||||
|
||||
if (markForReindex) {
|
||||
// only mark the file as unindexed, reindex will be done lazily
|
||||
if (!indexedIdsToUpdate.isEmpty()) {
|
||||
// only mark the file as outdated, reindex will be done lazily
|
||||
if (!fileIndexedStatesToUpdate.isEmpty()) {
|
||||
final List<ID<?, ?>> finalNontrivialFileIndexedStates = nontrivialFileIndexedStates;
|
||||
ApplicationManager.getApplication().runReadAction(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
IndexingStamp.removeAllIndexedState(file);
|
||||
//noinspection ForLoopReplaceableByForEach
|
||||
for (int i = 0, size = finalNontrivialFileIndexedStates.size(); i < size; ++i) {
|
||||
final ID<?, ?> indexId = finalNontrivialFileIndexedStates.get(i);
|
||||
if (needsFileContentLoading(indexId) && IndexingStamp.isFileIndexedStateCurrent(file, indexId)) {
|
||||
IndexingStamp.setFileIndexedStateOutdated(file, indexId);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
// the file is for sure not a dir and it was previously indexed by at least one index
|
||||
if (!isTooLarge(file)) scheduleForUpdate(file);
|
||||
}
|
||||
}
|
||||
else {
|
||||
boolean removed = myFilesToUpdate.remove(file);
|
||||
if (removed) {
|
||||
// file was scheduled for update, it might mean we have data in indices which would have been lazily updated
|
||||
List<ID<?, ?>> affectedContentIndices = calculateAffectedContentIndices(file);
|
||||
affectedContentIndices.addAll(indexedIdsToUpdate);
|
||||
indexedIdsToUpdate = affectedContentIndices;
|
||||
}
|
||||
|
||||
if (!indexedIdsToUpdate.isEmpty()) {
|
||||
final Collection<ID<?, ?>> finalIndexedIdsToUpdate = indexedIdsToUpdate;
|
||||
myFutureInvalidations.offer(new InvalidationTask(file) {
|
||||
@Override
|
||||
public void run() {
|
||||
removeFileDataFromIndices(finalIndexedIdsToUpdate, file);
|
||||
}
|
||||
});
|
||||
}
|
||||
else if (!fileIndexedStatesToUpdate.isEmpty()) { // file was removed, its data should be (lazily) wiped for every index
|
||||
final Collection<ID<?, ?>> finalFileIndexedStatesToUpdate = fileIndexedStatesToUpdate;
|
||||
myFutureInvalidations.offer(new InvalidationTask(file) {
|
||||
@Override
|
||||
public void run() {
|
||||
removeFileDataFromIndices(finalFileIndexedStatesToUpdate, file);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
IndexingStamp.flushCache(file);
|
||||
@@ -2256,12 +2251,12 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
myUpdateSemaphoreRef.compareAndSet(semaphore, null);
|
||||
}
|
||||
|
||||
private void forceUpdate(@Nullable Project project, @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedTo, boolean onlyRemoveOutdatedData) {
|
||||
private void forceUpdate(@Nullable Project project, @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedTo) {
|
||||
myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
|
||||
ProjectIndexableFilesFilter indexableFilesFilter = projectIndexableFiles(project);
|
||||
|
||||
UpdateSemaphore updateSemaphore;
|
||||
do{
|
||||
do {
|
||||
updateSemaphore = obtainForceUpdateSemaphore();
|
||||
try {
|
||||
for (VirtualFile file : getAllFilesToUpdate()) {
|
||||
@@ -2274,7 +2269,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
try {
|
||||
updateSemaphore.down();
|
||||
// process only files that can affect result
|
||||
processFileImpl(project, new com.intellij.ide.caches.FileContent(file), onlyRemoveOutdatedData);
|
||||
processFileImpl(project, new com.intellij.ide.caches.FileContent(file));
|
||||
}
|
||||
catch (ProcessCanceledException e) {
|
||||
updateSemaphore.reportUpdateCanceled();
|
||||
@@ -2296,28 +2291,23 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding.
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
finally {
|
||||
releaseForceUpdateSemaphore(updateSemaphore);
|
||||
}
|
||||
// if some other thread was unable to complete indexing because of PCE,
|
||||
// we should try again and ensure the file is indexed before proceeding further
|
||||
// if some other thread was unable to complete indexing because of PCE,
|
||||
// we should try again and ensure the file is indexed before proceeding further
|
||||
}
|
||||
while (updateSemaphore.isUpdateCanceled());
|
||||
}
|
||||
|
||||
private void processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent, boolean onlyRemoveOutdatedData) {
|
||||
private void processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) {
|
||||
final VirtualFile file = fileContent.getVirtualFile();
|
||||
final boolean reallyRemoved = myFilesToUpdate.remove(file);
|
||||
if (reallyRemoved && file.isValid()) {
|
||||
try {
|
||||
if (onlyRemoveOutdatedData || isTooLarge(file)) {
|
||||
// on shutdown there is no need to re-index the file, just remove outdated data from indices
|
||||
removeFileDataFromIndices(calculateAffectedContentIndices(file), file);
|
||||
if (onlyRemoveOutdatedData && file instanceof VirtualFileSystemEntry) {
|
||||
((VirtualFileSystemEntry)file).setFileIndexed(false); // we should be able index this file via UnindexedFileFinder later
|
||||
}
|
||||
if (isTooLarge(file)) {
|
||||
removeFileDataFromIndices(ContainerUtil.intersection(IndexingStamp.getNontrivialFileIndexedStates(file), myRequiringContentIndices), file);
|
||||
}
|
||||
else {
|
||||
doIndexFileContent(project, fileContent);
|
||||
@@ -2359,16 +2349,6 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
}
|
||||
}
|
||||
|
||||
private List<ID<?, ?>> calculateAffectedContentIndices(VirtualFile file) {
|
||||
final List<ID<?, ?>> affected = new ArrayList<ID<?, ?>>();
|
||||
for (final ID<?, ?> indexId : getAffectedIndexCandidates(file)) { // non requiring content indices should be flushed
|
||||
if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) {
|
||||
affected.add(indexId);
|
||||
}
|
||||
}
|
||||
return affected;
|
||||
}
|
||||
|
||||
private class UnindexedFilesFinder implements CollectingContentIterator {
|
||||
private final List<VirtualFile> myFiles = new ArrayList<VirtualFile>();
|
||||
@Nullable
|
||||
@@ -2463,7 +2443,7 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
}
|
||||
|
||||
private static boolean isFileIndexed(VirtualFile file, @NotNull ID<?, ?> indexId) {
|
||||
return IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId));
|
||||
return IndexingStamp.isFileIndexedStateCurrent(file, indexId);
|
||||
}
|
||||
|
||||
private boolean isUnderConfigOrSystem(@NotNull VirtualFile file) {
|
||||
@@ -2512,14 +2492,14 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
Document document = instance.getDocument(virtualFile);
|
||||
if (document != null && instance.isDocumentUnsaved(document)) {
|
||||
for(ID<?,?> psiBackedIndex:myPsiDependentIndices) {
|
||||
myUpToDateIndices.remove(psiBackedIndex);
|
||||
myUpToDateIndicesForUnsavedOrTransactedDocuments.remove(psiBackedIndex);
|
||||
}
|
||||
} else { // change in persistent file
|
||||
if (virtualFile instanceof VirtualFileWithId) {
|
||||
boolean wasIndexed = false;
|
||||
for (ID<?, ?> psiBackedIndex : myPsiDependentIndices) {
|
||||
if (isFileIndexed(virtualFile, psiBackedIndex)) {
|
||||
IndexingStamp.update(virtualFile, psiBackedIndex, IndexInfrastructure.INVALID_STAMP2);
|
||||
IndexingStamp.setFileIndexedStateOutdated(virtualFile, psiBackedIndex);
|
||||
wasIndexed = true;
|
||||
}
|
||||
}
|
||||
@@ -2538,8 +2518,8 @@ public class FileBasedIndexImpl extends FileBasedIndex {
|
||||
|
||||
@Override
|
||||
public void removeIndexableSet(@NotNull IndexableFileSet set) {
|
||||
if (!myIndexableSetToProjectMap.containsKey(set)) return; // avoid second unfiltered forceUpdate
|
||||
myChangedFilesCollector.forceUpdate(myIndexableSetToProjectMap.get(set), null, null, true);
|
||||
if (!myIndexableSetToProjectMap.containsKey(set)) return;
|
||||
myChangedFilesCollector.ensureAllInvalidateTasksCompleted();
|
||||
IndexingStamp.flushCaches();
|
||||
myIndexableSets.remove(set);
|
||||
myIndexableSetToProjectMap.remove(set);
|
||||
|
||||
@@ -21,27 +21,20 @@ package com.intellij.util.indexing;
|
||||
|
||||
import com.intellij.openapi.application.ApplicationManager;
|
||||
import com.intellij.openapi.application.PathManager;
|
||||
import com.intellij.openapi.util.io.FileUtil;
|
||||
import com.intellij.openapi.util.io.FileUtilRt;
|
||||
import com.intellij.openapi.vfs.VirtualFile;
|
||||
import com.intellij.openapi.vfs.ex.dummy.DummyFileSystem;
|
||||
import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
|
||||
import com.intellij.psi.stubs.StubIndexKey;
|
||||
import com.intellij.psi.stubs.StubUpdatingIndex;
|
||||
import com.intellij.util.containers.ConcurrentHashMap;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.File;
|
||||
import java.util.Locale;
|
||||
|
||||
@SuppressWarnings({"HardCodedStringLiteral"})
|
||||
public class IndexInfrastructure {
|
||||
private static final int VERSION = 9;
|
||||
private static final ConcurrentHashMap<ID<?, ?>, Long> ourIndexIdToCreationStamp = new ConcurrentHashMap<ID<?, ?>, Long>();
|
||||
private static final boolean ourUnitTestMode = ApplicationManager.getApplication().isUnitTestMode();
|
||||
public static final long INVALID_STAMP = -1L;
|
||||
public static final long INVALID_STAMP2 = -2L;
|
||||
private static final String STUB_VERSIONS = ".versions";
|
||||
|
||||
private IndexInfrastructure() {
|
||||
@@ -79,68 +72,6 @@ public class IndexInfrastructure {
|
||||
return indexDir;
|
||||
}
|
||||
|
||||
private static volatile long ourLastStamp; // ensure any file index stamp increases
|
||||
|
||||
public static synchronized void rewriteVersion(@NotNull final File file, final int version) throws IOException {
|
||||
final long prevLastModifiedValue = file.lastModified();
|
||||
if (file.exists()) {
|
||||
FileUtil.delete(file);
|
||||
}
|
||||
file.getParentFile().mkdirs();
|
||||
final DataOutputStream os = FileUtilRt.doIOOperation(new FileUtilRt.RepeatableIOOperation<DataOutputStream, FileNotFoundException>() {
|
||||
@Nullable
|
||||
@Override
|
||||
public DataOutputStream execute(boolean lastAttempt) throws FileNotFoundException {
|
||||
try {
|
||||
return new DataOutputStream(new FileOutputStream(file));
|
||||
} catch (FileNotFoundException ex) {
|
||||
if (lastAttempt) throw ex;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
});
|
||||
assert os != null;
|
||||
try {
|
||||
os.writeInt(version);
|
||||
os.writeInt(VERSION);
|
||||
}
|
||||
finally {
|
||||
ourIndexIdToCreationStamp.clear();
|
||||
os.close();
|
||||
long max = Math.max(System.currentTimeMillis(), Math.max(prevLastModifiedValue, ourLastStamp) + 2000);
|
||||
ourLastStamp = max;
|
||||
file.setLastModified(max);
|
||||
}
|
||||
}
|
||||
|
||||
public static long getIndexCreationStamp(@NotNull ID<?, ?> indexName) {
|
||||
Long version = ourIndexIdToCreationStamp.get(indexName);
|
||||
if (version != null) return version.longValue();
|
||||
|
||||
long stamp = getVersionFile(indexName).lastModified();
|
||||
ourIndexIdToCreationStamp.putIfAbsent(indexName, stamp);
|
||||
|
||||
return stamp;
|
||||
}
|
||||
|
||||
public static boolean versionDiffers(@NotNull File versionFile, final int currentIndexVersion) {
|
||||
try {
|
||||
ourLastStamp = Math.max(ourLastStamp, versionFile.lastModified());
|
||||
final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile)));
|
||||
try {
|
||||
final int savedIndexVersion = in.readInt();
|
||||
final int commonVersion = in.readInt();
|
||||
return savedIndexVersion != currentIndexVersion || commonVersion != VERSION;
|
||||
}
|
||||
finally {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static VirtualFile findFileById(@NotNull PersistentFS fs, final int id) {
|
||||
if (ourUnitTestMode) {
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
|
||||
package com.intellij.util.indexing;
|
||||
|
||||
import com.intellij.openapi.util.io.FileUtil;
|
||||
import com.intellij.openapi.util.io.FileUtilRt;
|
||||
import com.intellij.openapi.vfs.InvalidVirtualFileAccessException;
|
||||
import com.intellij.openapi.vfs.VirtualFile;
|
||||
import com.intellij.openapi.vfs.newvfs.FileAttribute;
|
||||
@@ -29,11 +31,8 @@ import gnu.trove.TObjectProcedure;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
@@ -43,36 +42,154 @@ import java.util.concurrent.ConcurrentMap;
|
||||
/**
|
||||
* @author Eugene Zhuravlev
|
||||
* Date: Dec 25, 2007
|
||||
*
|
||||
* A file has three indexed states (per particular index): indexed (with particular index_stamp), outdated and (trivial) unindexed
|
||||
* if index version is advanced or we rebuild it then index_stamp is advanced, we rebuild everything
|
||||
* if we get remove file event -> we should remove all indexed state from indices data for it (if state is nontrivial)
|
||||
* and set its indexed state to unindexed
|
||||
* if we get other event we set indexed state to outdated
|
||||
*
|
||||
* Index stamp is file timestamp of the index directory, we assume that index stamps are monotonically increasing
|
||||
*/
|
||||
public class IndexingStamp {
|
||||
private IndexingStamp() {
|
||||
private static final long UNINDEXED_STAMP = -1L; // we don't store trivial "absent" state
|
||||
private static final long INDEX_DATA_OUTDATED_STAMP = -2L;
|
||||
|
||||
private static final int VERSION = 9;
|
||||
private static final ConcurrentHashMap<ID<?, ?>, Long> ourIndexIdToCreationStamp = new ConcurrentHashMap<ID<?, ?>, Long>();
|
||||
private static volatile long ourLastStamp; // ensure any file index stamp increases
|
||||
|
||||
private IndexingStamp() {}
|
||||
|
||||
public static synchronized void rewriteVersion(@NotNull final File file, final int version) throws IOException {
|
||||
final long prevLastModifiedValue = file.lastModified();
|
||||
if (file.exists()) {
|
||||
FileUtil.delete(file);
|
||||
}
|
||||
file.getParentFile().mkdirs();
|
||||
final DataOutputStream os = FileUtilRt.doIOOperation(new FileUtilRt.RepeatableIOOperation<DataOutputStream, FileNotFoundException>() {
|
||||
@Nullable
|
||||
@Override
|
||||
public DataOutputStream execute(boolean lastAttempt) throws FileNotFoundException {
|
||||
try {
|
||||
return new DataOutputStream(new FileOutputStream(file));
|
||||
}
|
||||
catch (FileNotFoundException ex) {
|
||||
if (lastAttempt) throw ex;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
});
|
||||
assert os != null;
|
||||
try {
|
||||
os.writeInt(version);
|
||||
os.writeInt(VERSION);
|
||||
}
|
||||
finally {
|
||||
ourIndexIdToCreationStamp.clear();
|
||||
os.close();
|
||||
long max = Math.max(System.currentTimeMillis(), Math.max(prevLastModifiedValue, ourLastStamp) + 2000);
|
||||
ourLastStamp = max;
|
||||
file.setLastModified(max);
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean versionDiffers(@NotNull File versionFile, final int currentIndexVersion) {
|
||||
try {
|
||||
ourLastStamp = Math.max(ourLastStamp, versionFile.lastModified());
|
||||
final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile)));
|
||||
try {
|
||||
final int savedIndexVersion = in.readInt();
|
||||
final int commonVersion = in.readInt();
|
||||
return savedIndexVersion != currentIndexVersion || commonVersion != VERSION;
|
||||
}
|
||||
finally {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static long getIndexCreationStamp(@NotNull ID<?, ?> indexName) {
|
||||
Long version = ourIndexIdToCreationStamp.get(indexName);
|
||||
if (version != null) return version.longValue();
|
||||
|
||||
long stamp = IndexInfrastructure.getVersionFile(indexName).lastModified();
|
||||
ourIndexIdToCreationStamp.putIfAbsent(indexName, stamp);
|
||||
|
||||
return stamp;
|
||||
}
|
||||
|
||||
public static boolean isFileIndexedStateCurrent(VirtualFile file, ID<?, ?> indexName) {
|
||||
try {
|
||||
return getIndexStamp(file, indexName) == getIndexCreationStamp(indexName);
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
final Throwable cause = e.getCause();
|
||||
if (!(cause instanceof IOException)) {
|
||||
throw e; // in case of IO exceptions consider file unindexed
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static void setFileIndexedStateCurrent(VirtualFile file, ID<?, ?> id) {
|
||||
update(file, id, getIndexCreationStamp(id));
|
||||
}
|
||||
|
||||
public static void setFileIndexedStateUnindexed(VirtualFile file, ID<?, ?> id) {
|
||||
update(file, id, UNINDEXED_STAMP);
|
||||
}
|
||||
|
||||
public static void setFileIndexedStateOutdated(VirtualFile file, ID<?, ?> id) {
|
||||
update(file, id, INDEX_DATA_OUTDATED_STAMP);
|
||||
}
|
||||
|
||||
/**
|
||||
* The class is meant to be accessed from synchronized block only
|
||||
*/
|
||||
private static class Timestamps {
|
||||
private static final FileAttribute PERSISTENCE = new FileAttribute("__index_stamps__", 1, false);
|
||||
private static final FileAttribute PERSISTENCE = new FileAttribute("__index_stamps__", 2, false);
|
||||
private TObjectLongHashMap<ID<?, ?>> myIndexStamps;
|
||||
private boolean myIsDirty = false;
|
||||
|
||||
private Timestamps() {
|
||||
myIsDirty = true;
|
||||
}
|
||||
|
||||
private Timestamps(@Nullable DataInputStream stream) throws IOException {
|
||||
if (stream != null) {
|
||||
try {
|
||||
|
||||
int[] outdatedIndices = null;
|
||||
long dominatingIndexStamp = DataInputOutputUtil.readTIME(stream);
|
||||
long diff = dominatingIndexStamp - DataInputOutputUtil.timeBase;
|
||||
if (diff != 0 && diff < ID.MAX_NUMBER_OF_INDICES) {
|
||||
int numberOfOutdatedIndices = (int)diff;
|
||||
outdatedIndices = new int[numberOfOutdatedIndices];
|
||||
while(numberOfOutdatedIndices > 0) {
|
||||
outdatedIndices[--numberOfOutdatedIndices] = DataInputOutputUtil.readINT(stream);
|
||||
}
|
||||
dominatingIndexStamp = DataInputOutputUtil.readTIME(stream);
|
||||
}
|
||||
|
||||
while(stream.available() > 0) {
|
||||
ID<?, ?> id = ID.findById(DataInputOutputUtil.readINT(stream));
|
||||
if (id != null) {
|
||||
long stamp = IndexInfrastructure.getIndexCreationStamp(id);
|
||||
long stamp = getIndexCreationStamp(id);
|
||||
if (myIndexStamps == null) myIndexStamps = new TObjectLongHashMap<ID<?, ?>>(5, 0.98f);
|
||||
if (stamp <= dominatingIndexStamp) myIndexStamps.put(id, stamp);
|
||||
}
|
||||
}
|
||||
|
||||
if (outdatedIndices != null) {
|
||||
for(int outdatedIndexId:outdatedIndices) {
|
||||
ID<?, ?> id = ID.findById(outdatedIndexId);
|
||||
if (id != null) {
|
||||
long stamp = INDEX_DATA_OUTDATED_STAMP;
|
||||
if (myIndexStamps == null) myIndexStamps = new TObjectLongHashMap<ID<?, ?>>(5, 0.98f);
|
||||
if (stamp <= dominatingIndexStamp) myIndexStamps.put(id, stamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
stream.close();
|
||||
@@ -80,23 +197,53 @@ public class IndexingStamp {
|
||||
}
|
||||
}
|
||||
|
||||
// Indexed stamp compact format:
|
||||
// (DataInputOutputUtil.timeBase + numberOfOutdatedIndices outdated_index_id+)? (dominating_index_stamp) index_id*
|
||||
// Note, that FSRecords.REASONABLY_SMALL attribute storage allocation policy will give an attribute 32 bytes to each file
|
||||
// Compact format allows 22 indexed states in this state
|
||||
private void writeToStream(final DataOutputStream stream) throws IOException {
|
||||
if (myIndexStamps != null && !myIndexStamps.isEmpty()) {
|
||||
final long[] dominatingIndexStamp = new long[1];
|
||||
final long[] data = new long[2];
|
||||
final int dominatingStampIndex = 0;
|
||||
final int numberOfOutdatedIndex = 1;
|
||||
myIndexStamps.forEachEntry(
|
||||
new TObjectLongProcedure<ID<?, ?>>() {
|
||||
@Override
|
||||
public boolean execute(ID<?, ?> a, long b) {
|
||||
dominatingIndexStamp[0] = Math.max(dominatingIndexStamp[0], b);
|
||||
if (b == INDEX_DATA_OUTDATED_STAMP) {
|
||||
++data[numberOfOutdatedIndex];
|
||||
b = getIndexCreationStamp(a);
|
||||
}
|
||||
data[dominatingStampIndex] = Math.max(data[dominatingStampIndex], b);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
);
|
||||
DataInputOutputUtil.writeTIME(stream, dominatingIndexStamp[0]);
|
||||
if (data[numberOfOutdatedIndex] > 0) {
|
||||
assert data[numberOfOutdatedIndex] < ID.MAX_NUMBER_OF_INDICES;
|
||||
DataInputOutputUtil.writeTIME(stream, DataInputOutputUtil.timeBase + data[numberOfOutdatedIndex]);
|
||||
myIndexStamps.forEachEntry(new TObjectLongProcedure<ID<?, ?>>() {
|
||||
@Override
|
||||
public boolean execute(final ID<?, ?> id, final long timestamp) {
|
||||
try {
|
||||
if (timestamp == INDEX_DATA_OUTDATED_STAMP) {
|
||||
DataInputOutputUtil.writeINT(stream, id.getUniqueId());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
DataInputOutputUtil.writeTIME(stream, data[dominatingStampIndex]);
|
||||
myIndexStamps.forEachEntry(new TObjectLongProcedure<ID<?, ?>>() {
|
||||
@Override
|
||||
public boolean execute(final ID<?, ?> id, final long timestamp) {
|
||||
try {
|
||||
if (timestamp == INDEX_DATA_OUTDATED_STAMP) return true;
|
||||
DataInputOutputUtil.writeINT(stream, id.getUniqueId());
|
||||
return true;
|
||||
}
|
||||
@@ -117,7 +264,7 @@ public class IndexingStamp {
|
||||
|
||||
private void set(ID<?, ?> id, long tmst) {
|
||||
try {
|
||||
if (tmst < 0) {
|
||||
if (tmst == UNINDEXED_STAMP) {
|
||||
if (myIndexStamps == null) return;
|
||||
myIndexStamps.remove(id);
|
||||
return;
|
||||
@@ -139,20 +286,6 @@ public class IndexingStamp {
|
||||
private static final ConcurrentMap<VirtualFile, Timestamps> myTimestampsCache = new ConcurrentHashMap<VirtualFile, Timestamps>();
|
||||
private static final BlockingQueue<VirtualFile> ourFinishedFiles = new ArrayBlockingQueue<VirtualFile>(100);
|
||||
|
||||
public static boolean isFileIndexed(VirtualFile file, ID<?, ?> indexName, final long indexCreationStamp) {
|
||||
try {
|
||||
return getIndexStamp(file, indexName) == indexCreationStamp;
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
final Throwable cause = e.getCause();
|
||||
if (!(cause instanceof IOException)) {
|
||||
throw e; // in case of IO exceptions consider file unindexed
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static long getIndexStamp(@NotNull VirtualFile file, ID<?, ?> indexName) {
|
||||
synchronized (getStripedLock(file)) {
|
||||
Timestamps stamp = createOrGetTimeStamp(file);
|
||||
@@ -190,16 +323,8 @@ public class IndexingStamp {
|
||||
}
|
||||
}
|
||||
|
||||
public static void removeAllIndexedState(@NotNull VirtualFile file) {
|
||||
synchronized (getStripedLock(file)) {
|
||||
if (file instanceof NewVirtualFile && file.isValid()) {
|
||||
myTimestampsCache.put(file, new Timestamps());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public static Collection<ID<?,?>> getIndexedIds(@NotNull VirtualFile file) {
|
||||
public static List<ID<?,?>> getNontrivialFileIndexedStates(@NotNull VirtualFile file) {
|
||||
synchronized (getStripedLock(file)) {
|
||||
try {
|
||||
Timestamps stamp = createOrGetTimeStamp(file);
|
||||
|
||||
Reference in New Issue
Block a user