diff --git a/platform/core-api/src/com/intellij/util/indexing/ID.java b/platform/core-api/src/com/intellij/util/indexing/ID.java index 0a3535a7f00f..77ba1ee983e3 100644 --- a/platform/core-api/src/com/intellij/util/indexing/ID.java +++ b/platform/core-api/src/com/intellij/util/indexing/ID.java @@ -33,6 +33,7 @@ import java.io.*; public class ID { private static final TIntObjectHashMap ourRegistry = new TIntObjectHashMap(); private static final TObjectIntHashMap ourNameToIdRegistry = new TObjectIntHashMap(); + public static final int MAX_NUMBER_OF_INDICES = Short.MAX_VALUE; private final String myName; private final short myUniqueId; @@ -80,7 +81,7 @@ public class ID { } int n = ourNameToIdRegistry.size() + 1; - assert n <= Short.MAX_VALUE : "Number of indices exceeded"; + assert n <= MAX_NUMBER_OF_INDICES : "Number of indices exceeded"; ourNameToIdRegistry.put(name, n); diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 6682da34ed4c..61a521e729eb 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -127,7 +127,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe final boolean versionFileExisted = versionFile.exists(); final File indexRootDir = IndexInfrastructure.getIndexRootDir(indexKey); boolean needRebuild = false; - if (forceClean || IndexInfrastructure.versionDiffers(versionFile, version)) { + if (forceClean || IndexingStamp.versionDiffers(versionFile, version)) { final String[] children = indexRootDir.list(); // rebuild only if there exists what to rebuild needRebuild = !forceClean && (versionFileExisted || children != null && children.length > 0); @@ -135,7 +135,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe LOG.info("Version has changed for stub index " + extension.getKey() + ". The index will be rebuilt."); } FileUtil.delete(indexRootDir); - IndexInfrastructure.rewriteVersion(versionFile, version); + IndexingStamp.rewriteVersion(versionFile, version); } for (int attempt = 0; attempt < 2; attempt++) { @@ -157,7 +157,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe LOG.info(e); needRebuild = true; FileUtil.delete(indexRootDir); - IndexInfrastructure.rewriteVersion(versionFile, version); + IndexingStamp.rewriteVersion(versionFile, version); } } return needRebuild; diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 4cb07a623f03..5a735a85be70 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -86,7 +86,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi if (((IStubFileElementType)elementType).shouldBuildStubFor(file)) { return true; } - if (IndexingStamp.isFileIndexed(file, INDEX_ID, IndexInfrastructure.getIndexCreationStamp(INDEX_ID))) { + if (IndexingStamp.isFileIndexedStateCurrent(file, INDEX_ID)) { return true; } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 94f6bd2807c1..4c73d04cdc1f 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -126,7 +126,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { private final MessageBusConnection myConnection; private final FileDocumentManager myFileDocumentManager; private final FileTypeManager myFileTypeManager; - private final ConcurrentHashSet> myUpToDateIndices = new ConcurrentHashSet>(); + private final ConcurrentHashSet> myUpToDateIndicesForUnsavedOrTransactedDocuments = new ConcurrentHashSet>(); private volatile SmartFMap myTransactionMap = SmartFMap.emptyMap(); @Nullable private final String myConfigPath; @@ -158,7 +158,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { public void transactionStarted(final Document doc, final PsiFile file) { if (file != null) { myTransactionMap = myTransactionMap.plus(doc, file); - myUpToDateIndices.clear(); + myUpToDateIndicesForUnsavedOrTransactedDocuments.clear(); } } @@ -244,7 +244,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { ApplicationManager.getApplication().addApplicationListener(new ApplicationAdapter() { @Override public void writeActionStarted(Object action) { - myUpToDateIndices.clear(); + myUpToDateIndicesForUnsavedOrTransactedDocuments.clear(); } }); @@ -378,13 +378,13 @@ public class FileBasedIndexImpl extends FileBasedIndex { final File versionFile = IndexInfrastructure.getVersionFile(name); final boolean versionFileExisted = versionFile.exists(); boolean versionChanged = false; - if (isCurrentVersionCorrupted || IndexInfrastructure.versionDiffers(versionFile, version)) { + if (isCurrentVersionCorrupted || IndexingStamp.versionDiffers(versionFile, version)) { if (!isCurrentVersionCorrupted && versionFileExisted) { versionChanged = true; LOG.info("Version has changed for index " + name + ". The index will be rebuilt."); } FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); - IndexInfrastructure.rewriteVersion(versionFile, version); + IndexingStamp.rewriteVersion(versionFile, version); } initIndexStorage(extension, version, versionFile); @@ -411,8 +411,6 @@ public class FileBasedIndexImpl extends FileBasedIndex { final UpdatableIndex index = createIndex(name, extension, memStorage); final InputFilter inputFilter = extension.getInputFilter(); - assert inputFilter != null : "Index extension " + name + " must provide non-null input filter"; - myIndices.put(name, new Pair, InputFilter>(index, new IndexableFilesFilter(inputFilter))); if (inputFilter instanceof FileTypeSpecificInputFilter) { ((FileTypeSpecificInputFilter)inputFilter).registerFileTypesUsedForIndexing(new Consumer() { @@ -455,7 +453,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { } FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); - IndexInfrastructure.rewriteVersion(versionFile, version); + IndexingStamp.rewriteVersion(versionFile, version); } } } @@ -660,7 +658,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { finally { LOG.info("START INDEX SHUTDOWN"); try { - myChangedFilesCollector.forceUpdate(null, null, null, true); + myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); IndexingStamp.flushCaches(); for (ID indexId : myIndices.keySet()) { @@ -813,7 +811,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { if (isUpToDateCheckEnabled()) { try { checkRebuild(indexId, false); - myChangedFilesCollector.forceUpdate(project, filter, restrictedFile, false); + myChangedFilesCollector.forceUpdate(project, filter, restrictedFile); indexUnsavedDocuments(indexId, project, filter, restrictedFile); } catch (StorageException e) { @@ -1395,7 +1393,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { private void advanceIndexVersion(ID indexId) { try { - IndexInfrastructure.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId)); + IndexingStamp.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId)); } catch (IOException e) { LOG.error(e); @@ -1416,7 +1414,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { @Nullable Project project, GlobalSearchScope filter, VirtualFile restrictedFile) throws StorageException { - if (myUpToDateIndices.contains(indexId)) { + if (myUpToDateIndicesForUnsavedOrTransactedDocuments.contains(indexId)) { return; // no need to index unsaved docs } @@ -1458,7 +1456,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { ProgressManager.checkCanceled(); // assume all tasks were finished or cancelled in the same time // safe to set the flag here, because it will be cleared under the WriteAction - myUpToDateIndices.add(indexId); + myUpToDateIndicesForUnsavedOrTransactedDocuments.add(indexId); } } } @@ -1700,7 +1698,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { void processRefreshedFile(@NotNull Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) { myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); - myChangedFilesCollector.processFileImpl(project, fileContent, false); // ProcessCanceledException will cause re-adding the file to processing list + myChangedFilesCollector.processFileImpl(project, fileContent); // ProcessCanceledException will cause re-adding the file to processing list } public void indexFileContent(@Nullable Project project, @NotNull com.intellij.ide.caches.FileContent content) { @@ -1837,11 +1835,10 @@ public class FileBasedIndexImpl extends FileBasedIndex { public void run() { if (file.isValid()) { if (hasContent) { - IndexingStamp.update(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)); + IndexingStamp.setFileIndexedStateCurrent(file, indexId); } else { - // mark the file as unindexed - IndexingStamp.update(file, indexId, IndexInfrastructure.INVALID_STAMP); + IndexingStamp.setFileIndexedStateUnindexed(file, indexId); } if (myNotRequiringContentIndices.contains(indexId)) IndexingStamp.flushCache(file); } @@ -2028,8 +2025,8 @@ public class FileBasedIndexImpl extends FileBasedIndex { for (int i = 0, size = candidates.size(); i < size; ++i) { final ID indexId = candidates.get(i); if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) { - if (IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId))) { - IndexingStamp.update(file, indexId, IndexInfrastructure.INVALID_STAMP2); + if (IndexingStamp.isFileIndexedStateCurrent(file, indexId)) { + IndexingStamp.setFileIndexedStateOutdated(file, indexId); resetStamp = true; } scheduleForUpdate = true; @@ -2091,52 +2088,50 @@ public class FileBasedIndexImpl extends FileBasedIndex { cleanProcessedFlag(file); IndexingStamp.flushCache(file); - Collection> existingIndexedIds = IndexingStamp.getIndexedIds(file); + List> nontrivialFileIndexedStates = IndexingStamp.getNontrivialFileIndexedStates(file); - for(ID indexId:existingIndexedIds) { + for (ID indexId : nontrivialFileIndexedStates) { if (myNotRequiringContentIndices.contains(indexId)) { try { updateSingleIndex(indexId, file, null); - } catch (StorageException e) { + } + catch (StorageException e) { LOG.info(e); requestRebuild(indexId); } } } - Collection> indexedIdsToUpdate = ContainerUtil.intersection(existingIndexedIds, myRequiringContentIndices); + Collection> fileIndexedStatesToUpdate = ContainerUtil.intersection(nontrivialFileIndexedStates, myRequiringContentIndices); if (markForReindex) { - // only mark the file as unindexed, reindex will be done lazily - if (!indexedIdsToUpdate.isEmpty()) { + // only mark the file as outdated, reindex will be done lazily + if (!fileIndexedStatesToUpdate.isEmpty()) { + final List> finalNontrivialFileIndexedStates = nontrivialFileIndexedStates; ApplicationManager.getApplication().runReadAction(new Runnable() { @Override public void run() { - IndexingStamp.removeAllIndexedState(file); + //noinspection ForLoopReplaceableByForEach + for (int i = 0, size = finalNontrivialFileIndexedStates.size(); i < size; ++i) { + final ID indexId = finalNontrivialFileIndexedStates.get(i); + if (needsFileContentLoading(indexId) && IndexingStamp.isFileIndexedStateCurrent(file, indexId)) { + IndexingStamp.setFileIndexedStateOutdated(file, indexId); + } + } } }); // the file is for sure not a dir and it was previously indexed by at least one index if (!isTooLarge(file)) scheduleForUpdate(file); } } - else { - boolean removed = myFilesToUpdate.remove(file); - if (removed) { - // file was scheduled for update, it might mean we have data in indices which would have been lazily updated - List> affectedContentIndices = calculateAffectedContentIndices(file); - affectedContentIndices.addAll(indexedIdsToUpdate); - indexedIdsToUpdate = affectedContentIndices; - } - - if (!indexedIdsToUpdate.isEmpty()) { - final Collection> finalIndexedIdsToUpdate = indexedIdsToUpdate; - myFutureInvalidations.offer(new InvalidationTask(file) { - @Override - public void run() { - removeFileDataFromIndices(finalIndexedIdsToUpdate, file); - } - }); - } + else if (!fileIndexedStatesToUpdate.isEmpty()) { // file was removed, its data should be (lazily) wiped for every index + final Collection> finalFileIndexedStatesToUpdate = fileIndexedStatesToUpdate; + myFutureInvalidations.offer(new InvalidationTask(file) { + @Override + public void run() { + removeFileDataFromIndices(finalFileIndexedStatesToUpdate, file); + } + }); } IndexingStamp.flushCache(file); @@ -2256,12 +2251,12 @@ public class FileBasedIndexImpl extends FileBasedIndex { myUpdateSemaphoreRef.compareAndSet(semaphore, null); } - private void forceUpdate(@Nullable Project project, @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedTo, boolean onlyRemoveOutdatedData) { + private void forceUpdate(@Nullable Project project, @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedTo) { myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); ProjectIndexableFilesFilter indexableFilesFilter = projectIndexableFiles(project); UpdateSemaphore updateSemaphore; - do{ + do { updateSemaphore = obtainForceUpdateSemaphore(); try { for (VirtualFile file : getAllFilesToUpdate()) { @@ -2274,7 +2269,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { try { updateSemaphore.down(); // process only files that can affect result - processFileImpl(project, new com.intellij.ide.caches.FileContent(file), onlyRemoveOutdatedData); + processFileImpl(project, new com.intellij.ide.caches.FileContent(file)); } catch (ProcessCanceledException e) { updateSemaphore.reportUpdateCanceled(); @@ -2296,28 +2291,23 @@ public class FileBasedIndexImpl extends FileBasedIndex { break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding. } } - } finally { releaseForceUpdateSemaphore(updateSemaphore); } - // if some other thread was unable to complete indexing because of PCE, - // we should try again and ensure the file is indexed before proceeding further + // if some other thread was unable to complete indexing because of PCE, + // we should try again and ensure the file is indexed before proceeding further } while (updateSemaphore.isUpdateCanceled()); } - private void processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent, boolean onlyRemoveOutdatedData) { + private void processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) { final VirtualFile file = fileContent.getVirtualFile(); final boolean reallyRemoved = myFilesToUpdate.remove(file); if (reallyRemoved && file.isValid()) { try { - if (onlyRemoveOutdatedData || isTooLarge(file)) { - // on shutdown there is no need to re-index the file, just remove outdated data from indices - removeFileDataFromIndices(calculateAffectedContentIndices(file), file); - if (onlyRemoveOutdatedData && file instanceof VirtualFileSystemEntry) { - ((VirtualFileSystemEntry)file).setFileIndexed(false); // we should be able index this file via UnindexedFileFinder later - } + if (isTooLarge(file)) { + removeFileDataFromIndices(ContainerUtil.intersection(IndexingStamp.getNontrivialFileIndexedStates(file), myRequiringContentIndices), file); } else { doIndexFileContent(project, fileContent); @@ -2359,16 +2349,6 @@ public class FileBasedIndexImpl extends FileBasedIndex { } } - private List> calculateAffectedContentIndices(VirtualFile file) { - final List> affected = new ArrayList>(); - for (final ID indexId : getAffectedIndexCandidates(file)) { // non requiring content indices should be flushed - if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) { - affected.add(indexId); - } - } - return affected; - } - private class UnindexedFilesFinder implements CollectingContentIterator { private final List myFiles = new ArrayList(); @Nullable @@ -2463,7 +2443,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { } private static boolean isFileIndexed(VirtualFile file, @NotNull ID indexId) { - return IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)); + return IndexingStamp.isFileIndexedStateCurrent(file, indexId); } private boolean isUnderConfigOrSystem(@NotNull VirtualFile file) { @@ -2512,14 +2492,14 @@ public class FileBasedIndexImpl extends FileBasedIndex { Document document = instance.getDocument(virtualFile); if (document != null && instance.isDocumentUnsaved(document)) { for(ID psiBackedIndex:myPsiDependentIndices) { - myUpToDateIndices.remove(psiBackedIndex); + myUpToDateIndicesForUnsavedOrTransactedDocuments.remove(psiBackedIndex); } } else { // change in persistent file if (virtualFile instanceof VirtualFileWithId) { boolean wasIndexed = false; for (ID psiBackedIndex : myPsiDependentIndices) { if (isFileIndexed(virtualFile, psiBackedIndex)) { - IndexingStamp.update(virtualFile, psiBackedIndex, IndexInfrastructure.INVALID_STAMP2); + IndexingStamp.setFileIndexedStateOutdated(virtualFile, psiBackedIndex); wasIndexed = true; } } @@ -2538,8 +2518,8 @@ public class FileBasedIndexImpl extends FileBasedIndex { @Override public void removeIndexableSet(@NotNull IndexableFileSet set) { - if (!myIndexableSetToProjectMap.containsKey(set)) return; // avoid second unfiltered forceUpdate - myChangedFilesCollector.forceUpdate(myIndexableSetToProjectMap.get(set), null, null, true); + if (!myIndexableSetToProjectMap.containsKey(set)) return; + myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); IndexingStamp.flushCaches(); myIndexableSets.remove(set); myIndexableSetToProjectMap.remove(set); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java index 226542e49366..f795c3594dba 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexInfrastructure.java @@ -21,27 +21,20 @@ package com.intellij.util.indexing; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.application.PathManager; -import com.intellij.openapi.util.io.FileUtil; -import com.intellij.openapi.util.io.FileUtilRt; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.ex.dummy.DummyFileSystem; import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; import com.intellij.psi.stubs.StubIndexKey; import com.intellij.psi.stubs.StubUpdatingIndex; -import com.intellij.util.containers.ConcurrentHashMap; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.*; +import java.io.File; import java.util.Locale; @SuppressWarnings({"HardCodedStringLiteral"}) public class IndexInfrastructure { - private static final int VERSION = 9; - private static final ConcurrentHashMap, Long> ourIndexIdToCreationStamp = new ConcurrentHashMap, Long>(); private static final boolean ourUnitTestMode = ApplicationManager.getApplication().isUnitTestMode(); - public static final long INVALID_STAMP = -1L; - public static final long INVALID_STAMP2 = -2L; private static final String STUB_VERSIONS = ".versions"; private IndexInfrastructure() { @@ -79,68 +72,6 @@ public class IndexInfrastructure { return indexDir; } - private static volatile long ourLastStamp; // ensure any file index stamp increases - - public static synchronized void rewriteVersion(@NotNull final File file, final int version) throws IOException { - final long prevLastModifiedValue = file.lastModified(); - if (file.exists()) { - FileUtil.delete(file); - } - file.getParentFile().mkdirs(); - final DataOutputStream os = FileUtilRt.doIOOperation(new FileUtilRt.RepeatableIOOperation() { - @Nullable - @Override - public DataOutputStream execute(boolean lastAttempt) throws FileNotFoundException { - try { - return new DataOutputStream(new FileOutputStream(file)); - } catch (FileNotFoundException ex) { - if (lastAttempt) throw ex; - return null; - } - } - }); - assert os != null; - try { - os.writeInt(version); - os.writeInt(VERSION); - } - finally { - ourIndexIdToCreationStamp.clear(); - os.close(); - long max = Math.max(System.currentTimeMillis(), Math.max(prevLastModifiedValue, ourLastStamp) + 2000); - ourLastStamp = max; - file.setLastModified(max); - } - } - - public static long getIndexCreationStamp(@NotNull ID indexName) { - Long version = ourIndexIdToCreationStamp.get(indexName); - if (version != null) return version.longValue(); - - long stamp = getVersionFile(indexName).lastModified(); - ourIndexIdToCreationStamp.putIfAbsent(indexName, stamp); - - return stamp; - } - - public static boolean versionDiffers(@NotNull File versionFile, final int currentIndexVersion) { - try { - ourLastStamp = Math.max(ourLastStamp, versionFile.lastModified()); - final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile))); - try { - final int savedIndexVersion = in.readInt(); - final int commonVersion = in.readInt(); - return savedIndexVersion != currentIndexVersion || commonVersion != VERSION; - } - finally { - in.close(); - } - } - catch (IOException e) { - return true; - } - } - @Nullable public static VirtualFile findFileById(@NotNull PersistentFS fs, final int id) { if (ourUnitTestMode) { diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java index 86ab5750c685..ff9c740cbf16 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexingStamp.java @@ -16,6 +16,8 @@ package com.intellij.util.indexing; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.io.FileUtilRt; import com.intellij.openapi.vfs.InvalidVirtualFileAccessException; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.newvfs.FileAttribute; @@ -29,11 +31,8 @@ import gnu.trove.TObjectProcedure; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; +import java.io.*; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; @@ -43,36 +42,154 @@ import java.util.concurrent.ConcurrentMap; /** * @author Eugene Zhuravlev * Date: Dec 25, 2007 + * + * A file has three indexed states (per particular index): indexed (with particular index_stamp), outdated and (trivial) unindexed + * if index version is advanced or we rebuild it then index_stamp is advanced, we rebuild everything + * if we get remove file event -> we should remove all indexed state from indices data for it (if state is nontrivial) + * and set its indexed state to unindexed + * if we get other event we set indexed state to outdated + * + * Index stamp is file timestamp of the index directory, we assume that index stamps are monotonically increasing */ public class IndexingStamp { - private IndexingStamp() { + private static final long UNINDEXED_STAMP = -1L; // we don't store trivial "absent" state + private static final long INDEX_DATA_OUTDATED_STAMP = -2L; + + private static final int VERSION = 9; + private static final ConcurrentHashMap, Long> ourIndexIdToCreationStamp = new ConcurrentHashMap, Long>(); + private static volatile long ourLastStamp; // ensure any file index stamp increases + + private IndexingStamp() {} + + public static synchronized void rewriteVersion(@NotNull final File file, final int version) throws IOException { + final long prevLastModifiedValue = file.lastModified(); + if (file.exists()) { + FileUtil.delete(file); + } + file.getParentFile().mkdirs(); + final DataOutputStream os = FileUtilRt.doIOOperation(new FileUtilRt.RepeatableIOOperation() { + @Nullable + @Override + public DataOutputStream execute(boolean lastAttempt) throws FileNotFoundException { + try { + return new DataOutputStream(new FileOutputStream(file)); + } + catch (FileNotFoundException ex) { + if (lastAttempt) throw ex; + return null; + } + } + }); + assert os != null; + try { + os.writeInt(version); + os.writeInt(VERSION); + } + finally { + ourIndexIdToCreationStamp.clear(); + os.close(); + long max = Math.max(System.currentTimeMillis(), Math.max(prevLastModifiedValue, ourLastStamp) + 2000); + ourLastStamp = max; + file.setLastModified(max); + } + } + + public static boolean versionDiffers(@NotNull File versionFile, final int currentIndexVersion) { + try { + ourLastStamp = Math.max(ourLastStamp, versionFile.lastModified()); + final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile))); + try { + final int savedIndexVersion = in.readInt(); + final int commonVersion = in.readInt(); + return savedIndexVersion != currentIndexVersion || commonVersion != VERSION; + } + finally { + in.close(); + } + } + catch (IOException e) { + return true; + } + } + + private static long getIndexCreationStamp(@NotNull ID indexName) { + Long version = ourIndexIdToCreationStamp.get(indexName); + if (version != null) return version.longValue(); + + long stamp = IndexInfrastructure.getVersionFile(indexName).lastModified(); + ourIndexIdToCreationStamp.putIfAbsent(indexName, stamp); + + return stamp; + } + + public static boolean isFileIndexedStateCurrent(VirtualFile file, ID indexName) { + try { + return getIndexStamp(file, indexName) == getIndexCreationStamp(indexName); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (!(cause instanceof IOException)) { + throw e; // in case of IO exceptions consider file unindexed + } + } + + return false; + } + + public static void setFileIndexedStateCurrent(VirtualFile file, ID id) { + update(file, id, getIndexCreationStamp(id)); + } + + public static void setFileIndexedStateUnindexed(VirtualFile file, ID id) { + update(file, id, UNINDEXED_STAMP); + } + + public static void setFileIndexedStateOutdated(VirtualFile file, ID id) { + update(file, id, INDEX_DATA_OUTDATED_STAMP); } /** * The class is meant to be accessed from synchronized block only */ private static class Timestamps { - private static final FileAttribute PERSISTENCE = new FileAttribute("__index_stamps__", 1, false); + private static final FileAttribute PERSISTENCE = new FileAttribute("__index_stamps__", 2, false); private TObjectLongHashMap> myIndexStamps; private boolean myIsDirty = false; - private Timestamps() { - myIsDirty = true; - } - private Timestamps(@Nullable DataInputStream stream) throws IOException { if (stream != null) { try { - + int[] outdatedIndices = null; long dominatingIndexStamp = DataInputOutputUtil.readTIME(stream); + long diff = dominatingIndexStamp - DataInputOutputUtil.timeBase; + if (diff != 0 && diff < ID.MAX_NUMBER_OF_INDICES) { + int numberOfOutdatedIndices = (int)diff; + outdatedIndices = new int[numberOfOutdatedIndices]; + while(numberOfOutdatedIndices > 0) { + outdatedIndices[--numberOfOutdatedIndices] = DataInputOutputUtil.readINT(stream); + } + dominatingIndexStamp = DataInputOutputUtil.readTIME(stream); + } + while(stream.available() > 0) { ID id = ID.findById(DataInputOutputUtil.readINT(stream)); if (id != null) { - long stamp = IndexInfrastructure.getIndexCreationStamp(id); + long stamp = getIndexCreationStamp(id); if (myIndexStamps == null) myIndexStamps = new TObjectLongHashMap>(5, 0.98f); if (stamp <= dominatingIndexStamp) myIndexStamps.put(id, stamp); } } + + if (outdatedIndices != null) { + for(int outdatedIndexId:outdatedIndices) { + ID id = ID.findById(outdatedIndexId); + if (id != null) { + long stamp = INDEX_DATA_OUTDATED_STAMP; + if (myIndexStamps == null) myIndexStamps = new TObjectLongHashMap>(5, 0.98f); + if (stamp <= dominatingIndexStamp) myIndexStamps.put(id, stamp); + } + } + } } finally { stream.close(); @@ -80,23 +197,53 @@ public class IndexingStamp { } } + // Indexed stamp compact format: + // (DataInputOutputUtil.timeBase + numberOfOutdatedIndices outdated_index_id+)? (dominating_index_stamp) index_id* + // Note, that FSRecords.REASONABLY_SMALL attribute storage allocation policy will give an attribute 32 bytes to each file + // Compact format allows 22 indexed states in this state private void writeToStream(final DataOutputStream stream) throws IOException { if (myIndexStamps != null && !myIndexStamps.isEmpty()) { - final long[] dominatingIndexStamp = new long[1]; + final long[] data = new long[2]; + final int dominatingStampIndex = 0; + final int numberOfOutdatedIndex = 1; myIndexStamps.forEachEntry( new TObjectLongProcedure>() { @Override public boolean execute(ID a, long b) { - dominatingIndexStamp[0] = Math.max(dominatingIndexStamp[0], b); + if (b == INDEX_DATA_OUTDATED_STAMP) { + ++data[numberOfOutdatedIndex]; + b = getIndexCreationStamp(a); + } + data[dominatingStampIndex] = Math.max(data[dominatingStampIndex], b); + return true; } } ); - DataInputOutputUtil.writeTIME(stream, dominatingIndexStamp[0]); + if (data[numberOfOutdatedIndex] > 0) { + assert data[numberOfOutdatedIndex] < ID.MAX_NUMBER_OF_INDICES; + DataInputOutputUtil.writeTIME(stream, DataInputOutputUtil.timeBase + data[numberOfOutdatedIndex]); + myIndexStamps.forEachEntry(new TObjectLongProcedure>() { + @Override + public boolean execute(final ID id, final long timestamp) { + try { + if (timestamp == INDEX_DATA_OUTDATED_STAMP) { + DataInputOutputUtil.writeINT(stream, id.getUniqueId()); + } + return true; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + DataInputOutputUtil.writeTIME(stream, data[dominatingStampIndex]); myIndexStamps.forEachEntry(new TObjectLongProcedure>() { @Override public boolean execute(final ID id, final long timestamp) { try { + if (timestamp == INDEX_DATA_OUTDATED_STAMP) return true; DataInputOutputUtil.writeINT(stream, id.getUniqueId()); return true; } @@ -117,7 +264,7 @@ public class IndexingStamp { private void set(ID id, long tmst) { try { - if (tmst < 0) { + if (tmst == UNINDEXED_STAMP) { if (myIndexStamps == null) return; myIndexStamps.remove(id); return; @@ -139,20 +286,6 @@ public class IndexingStamp { private static final ConcurrentMap myTimestampsCache = new ConcurrentHashMap(); private static final BlockingQueue ourFinishedFiles = new ArrayBlockingQueue(100); - public static boolean isFileIndexed(VirtualFile file, ID indexName, final long indexCreationStamp) { - try { - return getIndexStamp(file, indexName) == indexCreationStamp; - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (!(cause instanceof IOException)) { - throw e; // in case of IO exceptions consider file unindexed - } - } - - return false; - } - public static long getIndexStamp(@NotNull VirtualFile file, ID indexName) { synchronized (getStripedLock(file)) { Timestamps stamp = createOrGetTimeStamp(file); @@ -190,16 +323,8 @@ public class IndexingStamp { } } - public static void removeAllIndexedState(@NotNull VirtualFile file) { - synchronized (getStripedLock(file)) { - if (file instanceof NewVirtualFile && file.isValid()) { - myTimestampsCache.put(file, new Timestamps()); - } - } - } - @NotNull - public static Collection> getIndexedIds(@NotNull VirtualFile file) { + public static List> getNontrivialFileIndexedStates(@NotNull VirtualFile file) { synchronized (getStripedLock(file)) { try { Timestamps stamp = createOrGetTimeStamp(file);