From 67bf2bb4c75f7b8a30d3ea87fa151d30c44df4c5 Mon Sep 17 00:00:00 2001 From: Dmitry Lomov Date: Tue, 8 May 2012 18:26:13 +0200 Subject: [PATCH] FileBasedIndex --- .../psi/impl/cache/impl/IdCacheTest.java | 6 +- .../intellij/psi/search/UpdateCacheTest.java | 5 +- .../intellij/find/impl/FindInProjectUtil.java | 3 +- .../impl/cache/impl/id/IdTableBuilding.java | 2 +- .../psi/impl/cache/impl/todo/TodoIndex.java | 2 +- .../com/intellij/psi/stubs/StubIndexImpl.java | 12 +- .../intellij/psi/stubs/StubUpdatingIndex.java | 2 +- .../util/indexing/FileBasedIndex.java | 2341 +---------------- .../util/indexing/FileBasedIndexImpl.java | 2327 ++++++++++++++++ .../FileBasedIndexProjectHandler.java | 4 +- .../util/indexing/UnindexedFilesUpdater.java | 6 +- .../src/componentSets/Lang.xml | 3 +- .../testFramework/LightPlatformTestCase.java | 3 +- 13 files changed, 2407 insertions(+), 2309 deletions(-) create mode 100644 platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java diff --git a/java/java-tests/testSrc/com/intellij/psi/impl/cache/impl/IdCacheTest.java b/java/java-tests/testSrc/com/intellij/psi/impl/cache/impl/IdCacheTest.java index 5ec392f58c23..eaf7ae618ffe 100644 --- a/java/java-tests/testSrc/com/intellij/psi/impl/cache/impl/IdCacheTest.java +++ b/java/java-tests/testSrc/com/intellij/psi/impl/cache/impl/IdCacheTest.java @@ -21,6 +21,7 @@ import com.intellij.psi.search.UsageSearchContext; import com.intellij.testFramework.PsiTestUtil; import com.intellij.util.ArrayUtil; import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.FileBasedIndexImpl; import java.io.File; import java.util.Arrays; @@ -35,9 +36,8 @@ public class IdCacheTest extends CodeInsightTestCase{ protected void setUp() throws Exception { super.setUp(); - final FileBasedIndex fileBasedIndex = FileBasedIndex.getInstance(); - fileBasedIndex.requestRebuild(IdIndex.NAME); - fileBasedIndex.requestRebuild(TodoIndex.NAME); + FileBasedIndex.getInstance().requestRebuild(IdIndex.NAME); + FileBasedIndex.getInstance().requestRebuild(TodoIndex.NAME); String root = JavaTestUtil.getJavaTestDataPath()+ "/psi/impl/cache/"; diff --git a/java/java-tests/testSrc/com/intellij/psi/search/UpdateCacheTest.java b/java/java-tests/testSrc/com/intellij/psi/search/UpdateCacheTest.java index cd2cf6890ac3..be90964eafae 100644 --- a/java/java-tests/testSrc/com/intellij/psi/search/UpdateCacheTest.java +++ b/java/java-tests/testSrc/com/intellij/psi/search/UpdateCacheTest.java @@ -44,6 +44,7 @@ import com.intellij.testFramework.PsiTestUtil; import com.intellij.util.ArrayUtil; import com.intellij.util.Processor; import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.FileBasedIndexImpl; import org.jetbrains.annotations.NonNls; import java.io.File; @@ -55,8 +56,8 @@ public class UpdateCacheTest extends PsiTestCase{ protected void setUp() throws Exception { super.setUp(); - FileBasedIndex.requestRebuild(IdIndex.NAME); - FileBasedIndex.requestRebuild(TodoIndex.NAME); + FileBasedIndex.getInstance().requestRebuild(IdIndex.NAME); + FileBasedIndex.getInstance().requestRebuild(TodoIndex.NAME); } @Override diff --git a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java index a9c2eb118461..e5601ebb1b87 100644 --- a/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java +++ b/platform/lang-impl/src/com/intellij/find/impl/FindInProjectUtil.java @@ -502,8 +502,7 @@ public class FindInProjectUtil { if (!keys.isEmpty()) { fast = true; List hits = new ArrayList(); - FileBasedIndex.getInstance() - .getFilesWithKey(TrigramIndex.INDEX_ID, keys, new CommonProcessors.CollectProcessor(hits), scope); + FileBasedIndex.getInstance().getFilesWithKey(TrigramIndex.INDEX_ID, keys, new CommonProcessors.CollectProcessor(hits), scope); for (VirtualFile hit : hits) { resultFiles.add(pm.findFile(hit)); diff --git a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java index 8e09281d2675..bb8dc7f3104f 100644 --- a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java +++ b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/id/IdTableBuilding.java @@ -322,7 +322,7 @@ public class IdTableBuilding { final OccurrenceConsumer occurrenceConsumer = new OccurrenceConsumer(null, true); EditorHighlighter highlighter; - final EditorHighlighter editorHighlighter = inputData.getUserData(FileBasedIndex.EDITOR_HIGHLIGHTER); + final EditorHighlighter editorHighlighter = inputData.getUserData(FileBasedIndexImpl.EDITOR_HIGHLIGHTER); if (editorHighlighter != null && checkCanUseCachedEditorHighlighter(chars, editorHighlighter)) { highlighter = editorHighlighter; } diff --git a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java index 594436346a7d..e4c91b7904da 100644 --- a/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/impl/cache/impl/todo/TodoIndex.java @@ -54,7 +54,7 @@ public class TodoIndex extends FileBasedIndexExtension messageBus.connect().subscribe(IndexPatternProvider.INDEX_PATTERNS_CHANGED, new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { - FileBasedIndex.requestRebuild(NAME); + FileBasedIndex.getInstance().requestRebuild(NAME); } }); } diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index d11e89e0f5a7..05a5e6d08e00 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -200,7 +200,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe @NotNull final Project project, @Nullable final GlobalSearchScope scope, @NotNull final Processor processor) { - final FileBasedIndex fileBasedIndex = FileBasedIndex.getInstance(); + final FileBasedIndexImpl fileBasedIndex = (FileBasedIndexImpl)FileBasedIndex.getInstance(); fileBasedIndex.ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, scope); final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); @@ -211,11 +211,11 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe try { try { // disable up-to-date check to avoid locks on attempt to acquire index write lock while holding at the same time the readLock for this index - FileBasedIndex.disableUpToDateCheckForCurrentThread(); + FileBasedIndexImpl.disableUpToDateCheckForCurrentThread(); index.getReadLock().lock(); final ValueContainer container = index.getData(key); - final FileBasedIndex.ProjectIndexableFilesFilter projectFilesFilter = fileBasedIndex.projectIndexableFiles(project); + final FileBasedIndexImpl.ProjectIndexableFilesFilter projectFilesFilter = fileBasedIndex.projectIndexableFiles(project); return container.forEach(new ValueContainer.ContainerAction() { @Override @@ -320,14 +320,14 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } finally { index.getReadLock().unlock(); - FileBasedIndex.enableUpToDateCheckForCurrentThread(); + FileBasedIndexImpl.enableUpToDateCheckForCurrentThread(); } } catch (StorageException e) { forceRebuild(e); } catch (RuntimeException e) { - final Throwable cause = FileBasedIndex.getCauseToRebuildIndex(e); + final Throwable cause = FileBasedIndexImpl.getCauseToRebuildIndex(e); if (cause != null) { forceRebuild(cause); } @@ -354,7 +354,7 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } private static void requestRebuild() { - FileBasedIndex.requestRebuild(StubUpdatingIndex.INDEX_ID); + FileBasedIndex.getInstance().requestRebuild(StubUpdatingIndex.INDEX_ID); } @Override diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java index 40fa9d8ac2cd..428f29dd556e 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubUpdatingIndex.java @@ -232,7 +232,7 @@ public class StubUpdatingIndex extends CustomImplementationFileBasedIndexExtensi } catch (StorageException e) { LOG.info(e); - FileBasedIndex.requestRebuild(INDEX_ID); + FileBasedIndex.getInstance().requestRebuild(INDEX_ID); } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java index 6e8bcf2a307b..c2f3ad423b19 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndex.java @@ -13,1676 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.intellij.util.indexing; -import com.intellij.AppTopics; -import com.intellij.history.LocalHistory; -import com.intellij.ide.caches.CacheUpdater; -import com.intellij.lang.ASTNode; -import com.intellij.notification.NotificationDisplayType; -import com.intellij.notification.NotificationGroup; -import com.intellij.notification.NotificationType; -import com.intellij.openapi.application.ApplicationAdapter; import com.intellij.openapi.application.ApplicationManager; -import com.intellij.openapi.application.PathManager; import com.intellij.openapi.components.ApplicationComponent; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.editor.Document; -import com.intellij.openapi.editor.highlighter.EditorHighlighter; -import com.intellij.openapi.editor.impl.EditorHighlighterCache; -import com.intellij.openapi.extensions.Extensions; -import com.intellij.openapi.fileEditor.FileDocumentManager; -import com.intellij.openapi.fileEditor.FileDocumentManagerAdapter; -import com.intellij.openapi.fileTypes.*; -import com.intellij.openapi.fileTypes.impl.FileTypeManagerImpl; -import com.intellij.openapi.module.Module; -import com.intellij.openapi.module.ModuleManager; -import com.intellij.openapi.progress.*; -import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator; -import com.intellij.openapi.project.*; -import com.intellij.openapi.roots.*; -import com.intellij.openapi.util.*; -import com.intellij.openapi.util.io.FileUtil; -import com.intellij.openapi.util.registry.Registry; -import com.intellij.openapi.vfs.*; -import com.intellij.openapi.vfs.ex.VirtualFileManagerEx; -import com.intellij.openapi.vfs.newvfs.BulkFileListener; -import com.intellij.openapi.vfs.newvfs.ManagingFS; -import com.intellij.openapi.vfs.newvfs.NewVirtualFile; -import com.intellij.openapi.vfs.newvfs.events.VFileEvent; -import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon; -import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; -import com.intellij.psi.*; -import com.intellij.psi.impl.PsiDocumentTransactionListener; -import com.intellij.psi.impl.source.PsiFileImpl; -import com.intellij.psi.search.EverythingGlobalScope; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.Condition; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileWithId; import com.intellij.psi.search.GlobalSearchScope; -import com.intellij.psi.stubs.SerializationManager; -import com.intellij.util.*; -import com.intellij.util.concurrency.Semaphore; -import com.intellij.util.containers.ConcurrentHashSet; -import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.io.*; -import com.intellij.util.io.DataOutputStream; -import com.intellij.util.io.storage.HeavyProcessLatch; -import com.intellij.util.messages.MessageBus; -import com.intellij.util.messages.MessageBusConnection; -import gnu.trove.*; +import com.intellij.util.Processor; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import javax.swing.*; -import java.io.*; -import java.lang.ref.SoftReference; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.Collection; +import java.util.List; +import java.util.Set; /** - * @author Eugene Zhuravlev - * Date: Dec 20, 2007 + * Author: dmitrylomov */ - -public class FileBasedIndex implements ApplicationComponent { - private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.FileBasedIndex"); - @NonNls - private static final String CORRUPTION_MARKER_NAME = "corruption.marker"; - private final Map, Pair, InputFilter>> myIndices = new THashMap, Pair, InputFilter>>(); - private final Map, Semaphore> myUnsavedDataIndexingSemaphores = new THashMap, Semaphore>(); - private final TObjectIntHashMap> myIndexIdToVersionMap = new TObjectIntHashMap>(); - private final Set> myNotRequiringContentIndices = new THashSet>(); - private final Set> myRequiringContentIndices = new THashSet>(); - private final Set myNoLimitCheckTypes = new THashSet(); - - private final PerIndexDocumentVersionMap myLastIndexedDocStamps = new PerIndexDocumentVersionMap(); - @NotNull private final ChangedFilesCollector myChangedFilesCollector; - - private final List myIndexableSets = ContainerUtil.createEmptyCOWList(); - private final Map myIndexableSetToProjectMap = new THashMap(); - - private static final int OK = 1; - private static final int REQUIRES_REBUILD = 2; - private static final int REBUILD_IN_PROGRESS = 3; - private static final Map, AtomicInteger> ourRebuildStatus = new THashMap, AtomicInteger>(); - - private final VirtualFileManagerEx myVfManager; - private final FileDocumentManager myFileDocumentManager; - private final FileTypeManager myFileTypeManager; - private final ConcurrentHashSet> myUpToDateIndices = new ConcurrentHashSet>(); - private final Map myTransactionMap = new THashMap(); - - private static final int ALREADY_PROCESSED = 0x04; - - @Nullable private final String myConfigPath; - @Nullable private final String mySystemPath; - private final boolean myIsUnitTestMode; - @Nullable private ScheduledFuture myFlushingFuture; - private volatile int myLocalModCount; - private volatile int myFilesModCount; - - public void requestReindex(@NotNull final VirtualFile file) { - myChangedFilesCollector.invalidateIndices(file, true); - } - - public void requestReindexExcluded(@NotNull final VirtualFile file) { - myChangedFilesCollector.invalidateIndices(file, false); - } - - public FileBasedIndex(final VirtualFileManagerEx vfManager, FileDocumentManager fdm, - FileTypeManager fileTypeManager, @NotNull MessageBus bus, SerializationManager sm /*need this parameter to ensure component dependency*/) throws IOException { - myVfManager = vfManager; - myFileDocumentManager = fdm; - myFileTypeManager = fileTypeManager; - myIsUnitTestMode = ApplicationManager.getApplication().isUnitTestMode(); - myConfigPath = calcConfigPath(PathManager.getConfigPath()); - mySystemPath = calcConfigPath(PathManager.getSystemPath()); - - final MessageBusConnection connection = bus.connect(); - connection.subscribe(PsiDocumentTransactionListener.TOPIC, new PsiDocumentTransactionListener() { - @Override - public void transactionStarted(final Document doc, final PsiFile file) { - if (file != null) { - synchronized (myTransactionMap) { - myTransactionMap.put(doc, file); - } - myUpToDateIndices.clear(); - } - } - - @Override - public void transactionCompleted(final Document doc, final PsiFile file) { - synchronized (myTransactionMap) { - myTransactionMap.remove(doc); - } - } - }); - - connection.subscribe(FileTypeManager.TOPIC, new FileTypeListener() { - @Nullable private Map> myTypeToExtensionMap; - @Override - public void beforeFileTypesChanged(final FileTypeEvent event) { - cleanupProcessedFlag(); - myTypeToExtensionMap = new THashMap>(); - for (FileType type : myFileTypeManager.getRegisteredFileTypes()) { - myTypeToExtensionMap.put(type, getExtensions(type)); - } - } - - @Override - public void fileTypesChanged(final FileTypeEvent event) { - final Map> oldExtensions = myTypeToExtensionMap; - myTypeToExtensionMap = null; - if (oldExtensions != null) { - final Map> newExtensions = new THashMap>(); - for (FileType type : myFileTypeManager.getRegisteredFileTypes()) { - newExtensions.put(type, getExtensions(type)); - } - // we are interested only in extension changes or removals. - // addition of an extension is handled separately by RootsChanged event - if (!newExtensions.keySet().containsAll(oldExtensions.keySet())) { - rebuildAllIndices(); - return; - } - for (Map.Entry> entry : oldExtensions.entrySet()) { - FileType fileType = entry.getKey(); - Set strings = entry.getValue(); - if (!newExtensions.get(fileType).containsAll(strings)) { - rebuildAllIndices(); - return; - } - } - } - } - - @NotNull - private Set getExtensions(@NotNull FileType type) { - final Set set = new THashSet(); - for (FileNameMatcher matcher : myFileTypeManager.getAssociations(type)) { - set.add(matcher.getPresentableString()); - } - return set; - } - - private void rebuildAllIndices() { - for (ID indexId : myIndices.keySet()) { - try { - clearIndex(indexId); - } - catch (StorageException e) { - LOG.info(e); - } - } - scheduleIndexRebuild(true); - } - }); - - connection.subscribe(VirtualFileManager.VFS_CHANGES, new BulkFileListener() { - @Override - public void before(@NotNull List events) { - for (VFileEvent event : events) { - final Object requestor = event.getRequestor(); - if (requestor instanceof FileDocumentManager || requestor instanceof PsiManager || requestor == LocalHistory.VFS_EVENT_REQUESTOR) { - cleanupMemoryStorage(); - break; - } - } - } - - @Override - public void after(@NotNull List events) { - } - }); - - connection.subscribe(AppTopics.FILE_DOCUMENT_SYNC, new FileDocumentManagerAdapter() { - @Override - public void fileContentReloaded(VirtualFile file, @NotNull Document document) { - cleanupMemoryStorage(); - } - - @Override - public void unsavedDocumentsDropped() { - cleanupMemoryStorage(); - } - }); - - ApplicationManager.getApplication().addApplicationListener(new ApplicationAdapter() { - @Override - public void writeActionStarted(Object action) { - myUpToDateIndices.clear(); - } - }); - - myChangedFilesCollector = new ChangedFilesCollector(); - - /* - final File workInProgressFile = getMarkerFile(); - if (workInProgressFile.exists()) { - // previous IDEA session was closed incorrectly, so drop all indices - FileUtil.delete(PathManager.getIndexRoot()); - } - */ - - try { - final FileBasedIndexExtension[] extensions = Extensions.getExtensions(FileBasedIndexExtension.EXTENSION_POINT_NAME); - for (FileBasedIndexExtension extension : extensions) { - ourRebuildStatus.put(extension.getName(), new AtomicInteger(OK)); - } - - final File corruptionMarker = new File(PathManager.getIndexRoot(), CORRUPTION_MARKER_NAME); - final boolean currentVersionCorrupted = corruptionMarker.exists(); - boolean versionChanged = false; - for (FileBasedIndexExtension extension : extensions) { - versionChanged |= registerIndexer(extension, currentVersionCorrupted); - } - FileUtil.delete(corruptionMarker); - - String rebuildNotification = null; - if (currentVersionCorrupted) { - rebuildNotification = "Index files on disk are corrupted. Indices will be rebuilt."; - } - else if (versionChanged) { - rebuildNotification = "Index file format has changed for some indices. These indices will be rebuilt."; - } - if (rebuildNotification != null - && !ApplicationManager.getApplication().isHeadlessEnvironment() - && Registry.is("ide.showIndexRebuildMessage")) { - new NotificationGroup("Indexing", NotificationDisplayType.BALLOON, false) - .createNotification("Index Rebuild", rebuildNotification, NotificationType.INFORMATION, null).notify(null); - } - - dropUnregisteredIndices(); - - // check if rebuild was requested for any index during registration - for (ID indexId : myIndices.keySet()) { - if (ourRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, OK)) { - try { - clearIndex(indexId); - } - catch (StorageException e) { - requestRebuild(indexId); - LOG.error(e); - } - } - } - - myVfManager.addVirtualFileListener(myChangedFilesCollector); - - registerIndexableSet(new AdditionalIndexableFileSet(), null); - } - finally { - ShutDownTracker.getInstance().registerShutdownTask(new Runnable() { - @Override - public void run() { - performShutdown(); - } - }); - //FileUtil.createIfDoesntExist(workInProgressFile); - saveRegisteredIndices(myIndices.keySet()); - myFlushingFuture = FlushingDaemon.everyFiveSeconds(new Runnable() { - int lastModCount = 0; - @Override - public void run() { - if (lastModCount == myLocalModCount) { - flushAllIndices(lastModCount); - } - lastModCount = myLocalModCount; - } - }); - - } - } - - @Override - public void initComponent() { - } - - @Nullable - private static String calcConfigPath(final String path) { - try { - final String _path = FileUtil.toSystemIndependentName(new File(path).getCanonicalPath()); - return _path.endsWith("/")? _path : _path + "/" ; - } - catch (IOException e) { - LOG.info(e); - return null; - } - } - - private static class FileBasedIndexHolder { - private static final FileBasedIndex ourInstance = ApplicationManager.getApplication().getComponent(FileBasedIndex.class); - } - +public abstract class FileBasedIndex implements ApplicationComponent { public static FileBasedIndex getInstance() { - return FileBasedIndexHolder.ourInstance; - } - - /** - * @return true if registered index requires full rebuild for some reason, e.g. is just created or corrupted - * - * @param extension - * @param isCurrentVersionCorrupted - */ - private boolean registerIndexer(@NotNull final FileBasedIndexExtension extension, final boolean isCurrentVersionCorrupted) throws IOException { - final ID name = extension.getName(); - final int version = extension.getVersion(); - final File versionFile = IndexInfrastructure.getVersionFile(name); - final boolean versionFileExisted = versionFile.exists(); - boolean versionChanged = false; - if (isCurrentVersionCorrupted || IndexInfrastructure.versionDiffers(versionFile, version)) { - if (!isCurrentVersionCorrupted && versionFileExisted) { - versionChanged = true; - LOG.info("Version has changed for index " + name + ". The index will be rebuilt."); - } - FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); - IndexInfrastructure.rewriteVersion(versionFile, version); - } - - for (int attempt = 0; attempt < 2; attempt++) { - try { - final MapIndexStorage storage = new MapIndexStorage(IndexInfrastructure.getStorageFile(name), extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getCacheSize()); - final MemoryIndexStorage memStorage = new MemoryIndexStorage(storage); - final UpdatableIndex index = createIndex(name, extension, memStorage); - final InputFilter inputFilter = extension.getInputFilter(); - - assert inputFilter != null : "Index extension " + name + " must provide non-null input filter"; - - myIndices.put(name, new Pair, InputFilter>(index, new IndexableFilesFilter(inputFilter))); - myUnsavedDataIndexingSemaphores.put(name, new Semaphore()); - myIndexIdToVersionMap.put(name, version); - if (!extension.dependsOnFileContent()) { - myNotRequiringContentIndices.add(name); - } - else { - myRequiringContentIndices.add(name); - } - myNoLimitCheckTypes.addAll(extension.getFileTypesWithSizeLimitNotApplicable()); - break; - } - catch (IOException e) { - LOG.info(e); - FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); - IndexInfrastructure.rewriteVersion(versionFile, version); - } - } - return versionChanged; - } - - private static void saveRegisteredIndices(@NotNull Collection> ids) { - final File file = getRegisteredIndicesFile(); - try { - FileUtil.createIfDoesntExist(file); - final DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file))); - try { - os.writeInt(ids.size()); - for (ID id : ids) { - IOUtil.writeString(id.toString(), os); - } - } - finally { - os.close(); - } - } - catch (IOException ignored) { - } - } - - @NotNull - private static Set readRegisteredIndexNames() { - final Set result = new THashSet(); - try { - final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(getRegisteredIndicesFile()))); - try { - final int size = in.readInt(); - for (int idx = 0; idx < size; idx++) { - result.add(IOUtil.readString(in)); - } - } - finally { - in.close(); - } - } - catch (IOException ignored) { - } - return result; - } - - @NotNull - private static File getRegisteredIndicesFile() { - return new File(PathManager.getIndexRoot(), "registered"); - } - - @NotNull - private UpdatableIndex createIndex(@NotNull final ID indexId, @NotNull final FileBasedIndexExtension extension, @NotNull final MemoryIndexStorage storage) throws IOException { - final MapReduceIndex index; - if (extension instanceof CustomImplementationFileBasedIndexExtension) { - final UpdatableIndex custom = ((CustomImplementationFileBasedIndexExtension)extension).createIndexImplementation(indexId, this, storage); - - assert custom != null : "Custom index implementation must not be null; index: " + indexId; - - if (!(custom instanceof MapReduceIndex)) { - return custom; - } - index = (MapReduceIndex)custom; - } - else { - index = new MapReduceIndex(indexId, extension.getIndexer(), storage); - } - - final KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); - index.setInputIdToDataKeysIndex(new Factory>>() { - @Override - public PersistentHashMap> create() { - try { - return createIdToDataKeysIndex(indexId, keyDescriptor, storage); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - - return index; - } - - @NotNull - private static PersistentHashMap> createIdToDataKeysIndex(@NotNull final ID indexId, - @NotNull final KeyDescriptor keyDescriptor, - @NotNull MemoryIndexStorage storage) throws IOException { - final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(indexId); - final Ref isBufferingMode = new Ref(false); - final TIntObjectHashMap> tempMap = new TIntObjectHashMap>(); - - final DataExternalizer> dataExternalizer = new DataExternalizer>() { - @Override - public void save(DataOutput out, @NotNull Collection value) throws IOException { - try { - DataInputOutputUtil.writeINT(out, value.size()); - for (K key : value) { - keyDescriptor.save(out, key); - } - } - catch (IllegalArgumentException e) { - throw new IOException("Error saving data for index " + indexId, e); - } - } - - @NotNull - @Override - public Collection read(DataInput in) throws IOException { - try { - final int size = DataInputOutputUtil.readINT(in); - final List list = new ArrayList(size); - for (int idx = 0; idx < size; idx++) { - list.add(keyDescriptor.read(in)); - } - return list; - } - catch (IllegalArgumentException e) { - throw new IOException("Error reading data for index " + indexId, e); - } - } - }; - - // Important! Update IdToDataKeysIndex depending on the sate of "buffering" flag from the MemoryStorage. - // If buffering is on, all changes should be done in memory (similar to the way it is done in memory storage). - // Otherwise data in IdToDataKeysIndex will not be in sync with the 'main' data in the index on disk and index updates will be based on the - // wrong sets of keys for the given file. This will lead to unpredictable results in main index because it will not be - // cleared properly before updating (removed data will still be present on disk). See IDEA-52223 for illustration of possible effects. - - final PersistentHashMap> map = new PersistentHashMap>( - indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, dataExternalizer - ) { - - @Override - protected Collection doGet(Integer integer) throws IOException { - if (isBufferingMode.get()) { - final Collection collection = tempMap.get(integer); - if (collection != null) { - return collection; - } - } - return super.doGet(integer); - } - - @Override - protected void doPut(Integer integer, @Nullable Collection ks) throws IOException { - if (isBufferingMode.get()) { - tempMap.put(integer, ks == null? Collections.emptySet() : ks); - } - else { - super.doPut(integer, ks); - } - } - - @Override - protected void doRemove(Integer integer) throws IOException { - if (isBufferingMode.get()) { - tempMap.put(integer, Collections.emptySet()); - } - else { - super.doRemove(integer); - } - } - }; - - storage.addBufferingStateListsner(new MemoryIndexStorage.BufferingStateListener() { - @Override - public void bufferingStateChanged(boolean newState) { - synchronized (map) { - isBufferingMode.set(newState); - } - } - @Override - public void memoryStorageCleared() { - synchronized (map) { - tempMap.clear(); - } - } - }); - return map; - } - - @Override - @NonNls - @NotNull - public String getComponentName() { - return "FileBasedIndex"; - } - - @Override - public void disposeComponent() { - performShutdown(); - } - - private final AtomicBoolean myShutdownPerformed = new AtomicBoolean(false); - - private void performShutdown() { - if (!myShutdownPerformed.compareAndSet(false, true)) { - return; // already shut down - } - try { - if (myFlushingFuture != null) { - myFlushingFuture.cancel(false); - myFlushingFuture = null; - } - - myFileDocumentManager.saveAllDocuments(); - } - finally { - LOG.info("START INDEX SHUTDOWN"); - try { - myChangedFilesCollector.forceUpdate(null, null, null, true); - - for (ID indexId : myIndices.keySet()) { - final UpdatableIndex index = getIndex(indexId); - assert index != null; - checkRebuild(indexId, true); // if the index was scheduled for rebuild, only clean it - //LOG.info("DISPOSING " + indexId); - index.dispose(); - } - - myVfManager.removeVirtualFileListener(myChangedFilesCollector); - - //FileUtil.delete(getMarkerFile()); - } - catch (Throwable e) { - LOG.info("Problems during index shutdown", e); - throw new RuntimeException(e); - } - LOG.info("END INDEX SHUTDOWN"); - } - } - - private void flushAllIndices(final long modCount) { - if (HeavyProcessLatch.INSTANCE.isRunning()) { - return; - } - IndexingStamp.flushCache(); - for (ID indexId : new ArrayList>(myIndices.keySet())) { - if (HeavyProcessLatch.INSTANCE.isRunning() || modCount != myLocalModCount) { - return; // do not interfere with 'main' jobs - } - try { - final UpdatableIndex index = getIndex(indexId); - if (index != null) { - index.flush(); - } - } - catch (StorageException e) { - LOG.info(e); - requestRebuild(indexId); - } - } - - if (!HeavyProcessLatch.INSTANCE.isRunning() && modCount == myLocalModCount) { // do not interfere with 'main' jobs - SerializationManager.getInstance().flushNameStorage(); - } - } - - /** - * @param project it is guaranteed to return data which is up-to-date withing the project - * Keys obtained from the files which do not belong to the project specified may not be up-to-date or even exist - */ - @NotNull - public Collection getAllKeys(@NotNull final ID indexId, @NotNull Project project) { - Set allKeys = new THashSet(); - processAllKeys(indexId, new CommonProcessors.CollectProcessor(allKeys), project); - return allKeys; - } - - /** - * @param project it is guaranteed to return data which is up-to-date withing the project - * Keys obtained from the files which do not belong to the project specified may not be up-to-date or even exist - */ - public boolean processAllKeys(@NotNull final ID indexId, Processor processor, @Nullable Project project) { - try { - final UpdatableIndex index = getIndex(indexId); - if (index == null) { - return true; - } - ensureUpToDate(indexId, project, project != null? GlobalSearchScope.allScope(project) : new EverythingGlobalScope()); - return index.processAllKeys(processor); - } - catch (StorageException e) { - scheduleRebuild(indexId, e); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof StorageException || cause instanceof IOException) { - scheduleRebuild(indexId, cause); - } - else { - throw e; - } - } - - return false; - } - - private static final ThreadLocal myUpToDateCheckState = new ThreadLocal(); - - public static void disableUpToDateCheckForCurrentThread() { - final Integer currentValue = myUpToDateCheckState.get(); - myUpToDateCheckState.set(currentValue == null? 1 : currentValue.intValue() + 1); - } - - public static void enableUpToDateCheckForCurrentThread() { - final Integer currentValue = myUpToDateCheckState.get(); - if (currentValue != null) { - final int newValue = currentValue.intValue() - 1; - if (newValue != 0) { - myUpToDateCheckState.set(newValue); - } - else { - myUpToDateCheckState.remove(); - } - } - } - - private static boolean isUpToDateCheckEnabled() { - final Integer value = myUpToDateCheckState.get(); - return value == null || value.intValue() == 0; - } - - - private final ThreadLocal myReentrancyGuard = new ThreadLocal() { - @Override - protected Boolean initialValue() { - return Boolean.FALSE; - } - }; - - /** - * DO NOT CALL DIRECTLY IN CLIENT CODE - * The method is internal to indexing engine end is called internally. The method is public due to implementation details - */ - public void ensureUpToDate(@NotNull final ID indexId, @Nullable Project project, @Nullable GlobalSearchScope filter) { - ensureUpToDate(indexId, project, filter, null); - } - - private void ensureUpToDate(@NotNull final ID indexId, @Nullable Project project, @Nullable GlobalSearchScope filter, - @Nullable VirtualFile restrictedFile) { - if (!needsFileContentLoading(indexId)) { - return; //indexed eagerly in foreground while building unindexed file list - } - if (isDumb(project)) { - handleDumbMode(project); - } - - if (myReentrancyGuard.get().booleanValue()) { - //assert false : "ensureUpToDate() is not reentrant!"; - return; - } - myReentrancyGuard.set(Boolean.TRUE); - - try { - myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); - if (isUpToDateCheckEnabled()) { - try { - checkRebuild(indexId, false); - myChangedFilesCollector.forceUpdate(project, filter, restrictedFile, false); - indexUnsavedDocuments(indexId, project, filter, restrictedFile); - } - catch (StorageException e) { - scheduleRebuild(indexId, e); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof StorageException || cause instanceof IOException) { - scheduleRebuild(indexId, e); - } - else { - throw e; - } - } - } - } - finally { - myReentrancyGuard.set(Boolean.FALSE); - } - } - - private static void handleDumbMode(@Nullable Project project) { - ProgressManager.checkCanceled(); // DumbModeAction.CANCEL - - if (project != null) { - final ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator(); - if (progressIndicator instanceof BackgroundableProcessIndicator) { - final BackgroundableProcessIndicator indicator = (BackgroundableProcessIndicator)progressIndicator; - if (indicator.getDumbModeAction() == DumbModeAction.WAIT) { - assert !ApplicationManager.getApplication().isDispatchThread(); - DumbService.getInstance(project).waitForSmartMode(); - return; - } - } - } - - throw new IndexNotReadyException(); - } - - private static boolean isDumb(@Nullable Project project) { - if (project != null) { - return DumbServiceImpl.getInstance(project).isDumb(); - } - for (Project proj : ProjectManager.getInstance().getOpenProjects()) { - if (DumbServiceImpl.getInstance(proj).isDumb()) { - return true; - } - } - return false; - } - - @NotNull - public List getValues(@NotNull final ID indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) { - final List values = new SmartList(); - processValuesImpl(indexId, dataKey, true, null, new ValueProcessor() { - @Override - public boolean process(final VirtualFile file, final V value) { - values.add(value); - return true; - } - }, filter); - return values; - } - - @NotNull - public Collection getContainingFiles(@NotNull final ID indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) { - final Set files = new THashSet(); - processValuesImpl(indexId, dataKey, false, null, new ValueProcessor() { - @Override - public boolean process(final VirtualFile file, final V value) { - files.add(file); - return true; - } - }, filter); - return files; - } - - - public interface ValueProcessor { - /** - * @param value a value to process - * @param file the file the value came from - * @return false if no further processing is needed, true otherwise - */ - boolean process(VirtualFile file, V value); - } - - /** - * @return false if ValueProcessor.process() returned false; true otherwise or if ValueProcessor was not called at all - */ - public boolean processValues(@NotNull final ID indexId, @NotNull final K dataKey, @Nullable final VirtualFile inFile, - @NotNull ValueProcessor processor, @NotNull final GlobalSearchScope filter) { - return processValuesImpl(indexId, dataKey, false, inFile, processor, filter); - } - - - - - @Nullable - private R processExceptions(@NotNull final ID indexId, - @Nullable final VirtualFile restrictToFile, - @NotNull final GlobalSearchScope filter, - @NotNull ThrowableConvertor, R, StorageException> computable) { - try { - final UpdatableIndex index = getIndex(indexId); - if (index == null) { - return null; - } - final Project project = filter.getProject(); - //assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries"; - ensureUpToDate(indexId, project, filter, restrictToFile); - - try { - index.getReadLock().lock(); - return computable.convert(index); - } - finally { - index.getReadLock().unlock(); - } - } - catch (StorageException e) { - scheduleRebuild(indexId, e); - } - catch (RuntimeException e) { - final Throwable cause = getCauseToRebuildIndex(e); - if (cause != null) { - scheduleRebuild(indexId, cause); - } - else { - throw e; - } - } - return null; - } - - private boolean processValuesImpl(@NotNull final ID indexId, final K dataKey, final boolean ensureValueProcessedOnce, - @Nullable final VirtualFile restrictToFile, @NotNull final ValueProcessor processor, - @NotNull final GlobalSearchScope filter) { - ThrowableConvertor, Boolean, StorageException> keyProcessor = new ThrowableConvertor, Boolean, StorageException>() { - @Override - public Boolean convert(@NotNull UpdatableIndex index) throws StorageException { - final ValueContainer container = index.getData(dataKey); - - boolean shouldContinue = true; - - if (restrictToFile != null) { - if (restrictToFile instanceof VirtualFileWithId) { - final int restrictedFileId = getFileId(restrictToFile); - for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext(); ) { - final V value = valueIt.next(); - if (container.isAssociated(value, restrictedFileId)) { - shouldContinue = processor.process(restrictToFile, value); - if (!shouldContinue) { - break; - } - } - } - } - } - else { - final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); - ProjectIndexableFilesFilter projectFilesSet = projectIndexableFiles(filter.getProject()); - VALUES_LOOP: for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext();) { - final V value = valueIt.next(); - for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) { - final int id = inputIdsIterator.next(); - if (projectFilesSet != null && !projectFilesSet.contains(id)) continue; - VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); - if (file != null && filter.accept(file)) { - shouldContinue = processor.process(file, value); - if (!shouldContinue) { - break VALUES_LOOP; - } - if (ensureValueProcessedOnce) { - break; // continue with the next value - } - } - } - } - } - return shouldContinue; - } - }; - final Boolean result = processExceptions(indexId, restrictToFile, filter, keyProcessor); - return result == null || result.booleanValue(); - } - - public boolean processFilesContainingAllKeys(@NotNull final ID indexId, - @NotNull final Collection dataKeys, - @NotNull final GlobalSearchScope filter, - @Nullable Condition valueChecker, - @NotNull final Processor processor) { - ProjectIndexableFilesFilter filesSet = projectIndexableFiles(filter.getProject()); - final TIntHashSet set = collectFileIdsContainingAllKeys(indexId, dataKeys, filter, valueChecker, filesSet); - return set != null && processVirtualFiles(set, filter, processor); - } - - private static final Key> ourProjectFilesSetKey = Key.create("projectFiles"); - - public static final class ProjectIndexableFilesFilter { - private static final int SHIFT = 6; - private static final int MASK = (1 << SHIFT) - 1; - private final long[] myBitMask; - private final int myModificationCount; - private final int myMinId; - private final int myMaxId; - - private ProjectIndexableFilesFilter(@NotNull TIntHashSet set, int modificationCount) { - myModificationCount = modificationCount; - final int[] minMax = new int[2]; - if (set.size() > 0) { - minMax[0] = minMax[1] = set.iterator().next(); - } - set.forEach(new TIntProcedure() { - @Override - public boolean execute(int value) { - minMax[0] = Math.min(minMax[0], value); - minMax[1] = Math.max(minMax[1], value); - return true; - } - }); - myMaxId = minMax[1]; - myMinId = minMax[0]; - myBitMask = new long[((myMaxId - myMinId) >> SHIFT) + 1]; - set.forEach(new TIntProcedure() { - @Override - public boolean execute(int value) { - value = value - myMinId; - myBitMask[value >> SHIFT] |= (1L << (value & MASK)); - return true; - } - }); - } - - public boolean contains(int id) { - if (id < myMinId) return false; - if (id > myMaxId) return false; - id -= myMinId; - return (myBitMask[id >> SHIFT] & (1L << (id & MASK))) != 0; - } - } - - @Nullable - public ProjectIndexableFilesFilter projectIndexableFiles(@Nullable Project project) { - if (project == null) return null; - - SoftReference reference = project.getUserData(ourProjectFilesSetKey); - ProjectIndexableFilesFilter data = reference != null ? reference.get() : null; - if (data != null && data.myModificationCount == myFilesModCount) return data; - - final TIntHashSet filesSet = new TIntHashSet(); - iterateIndexableFiles(new ContentIterator() { - @Override - public boolean processFile(@NotNull VirtualFile fileOrDir) { - filesSet.add(((VirtualFileWithId)fileOrDir).getId()); - return true; - } - }, project, ProgressManager.getInstance().getProgressIndicator()); - ProjectIndexableFilesFilter files = new ProjectIndexableFilesFilter(filesSet, myFilesModCount); - project.putUserData(ourProjectFilesSetKey, new SoftReference(files)); - return files; - } - - @Nullable - private TIntHashSet collectFileIdsContainingAllKeys(@NotNull final ID indexId, - @NotNull final Collection dataKeys, - @NotNull final GlobalSearchScope filter, - @Nullable final Condition valueChecker, - @Nullable final ProjectIndexableFilesFilter projectFilesFilter) { - final ThrowableConvertor, TIntHashSet, StorageException> convertor = - new ThrowableConvertor, TIntHashSet, StorageException>() { - @Nullable - @Override - public TIntHashSet convert(@NotNull UpdatableIndex index) throws StorageException { - TIntHashSet mainIntersection = null; - - for (K dataKey : dataKeys) { - ProgressManager.checkCanceled(); - final TIntHashSet copy = new TIntHashSet(); - final ValueContainer container = index.getData(dataKey); - - for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext(); ) { - final V value = valueIt.next(); - if (valueChecker != null && !valueChecker.value(value)) { - continue; - } - - ValueContainer.IntIterator iterator = container.getInputIdsIterator(value); - - if (mainIntersection == null || iterator.size() < mainIntersection.size()) { - for (final ValueContainer.IntIterator inputIdsIterator = iterator; inputIdsIterator.hasNext(); ) { - final int id = inputIdsIterator.next(); - if (mainIntersection == null && (projectFilesFilter == null || projectFilesFilter.contains(id)) || - mainIntersection != null && mainIntersection.contains(id) - ) { - copy.add(id); - } - } - } else { - mainIntersection.forEach(new TIntProcedure() { - final ValueContainer.IntPredicate predicate = container.getValueAssociationPredicate(value); - @Override - public boolean execute(int id) { - if (predicate.contains(id)) copy.add(id); - return true; - } - }); - } - } - - mainIntersection = copy; - if (mainIntersection.isEmpty()) { - return new TIntHashSet(); - } - } - - return mainIntersection; - } - }; - - - return processExceptions(indexId, null, filter, convertor); - } - - private static boolean processVirtualFiles(@NotNull TIntHashSet ids, - @NotNull final GlobalSearchScope filter, - @NotNull final Processor processor) { - final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); - return ids.forEach(new TIntProcedure() { - @Override - public boolean execute(int id) { - ProgressManager.checkCanceled(); - VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); - if (file != null && filter.accept(file)) { - return processor.process(file); - } - return true; - } - }); - } - - @Nullable - public static Throwable getCauseToRebuildIndex(@NotNull RuntimeException e) { - Throwable cause = e.getCause(); - if (cause instanceof StorageException || cause instanceof IOException || - cause instanceof IllegalArgumentException) return cause; - return null; - } - - public boolean getFilesWithKey(@NotNull final ID indexId, - @NotNull final Set dataKeys, - @NotNull Processor processor, - @NotNull GlobalSearchScope filter) { - try { - final UpdatableIndex index = getIndex(indexId); - if (index == null) { - return true; - } - final Project project = filter.getProject(); - //assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries"; - ensureUpToDate(indexId, project, filter); - - try { - index.getReadLock().lock(); - final List locals = new ArrayList(); - for (K dataKey : dataKeys) { - TIntHashSet local = new TIntHashSet(); - locals.add(local); - final ValueContainer container = index.getData(dataKey); - - for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext();) { - final V value = valueIt.next(); - for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) { - final int id = inputIdsIterator.next(); - local.add(id); - } - } - } - - if (locals.isEmpty()) { - return true; - } - - Collections.sort(locals, new Comparator() { - @Override - public int compare(TIntHashSet o1, TIntHashSet o2) { - return o1.size() - o2.size(); - } - }); - - final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); - TIntIterator ids = join(locals).iterator(); - ProjectIndexableFilesFilter projectIndexableFilesFilter = projectIndexableFiles(project); - while (ids.hasNext()) { - int id = ids.next(); - if (projectIndexableFilesFilter != null && !projectIndexableFilesFilter.contains(id)) continue; - //VirtualFile file = IndexInfrastructure.findFileById(fs, id); - VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); - if (file != null && filter.accept(file)) { - if (!processor.process(file)) { - return false; - } - } - } - } - finally { - index.getReadLock().unlock(); - } - } - catch (StorageException e) { - scheduleRebuild(indexId, e); - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof StorageException || cause instanceof IOException) { - scheduleRebuild(indexId, cause); - } - else { - throw e; - } - } - return true; - } - - @NotNull - private static TIntHashSet join(@NotNull List locals) { - TIntHashSet result = locals.get(0); - if (locals.size() > 1) { - TIntIterator it = result.iterator(); - - while (it.hasNext()) { - int id = it.next(); - for (int i = 1; i < locals.size(); i++) { - if (!locals.get(i).contains(id)) { - it.remove(); - break; - } - } - } - } - return result; - } - - public void scheduleRebuild(@NotNull final ID indexId, @NotNull final Throwable e) { - LOG.info(e); - requestRebuild(indexId); - try { - checkRebuild(indexId, false); - } - catch (ProcessCanceledException ignored) { - } - } - - private void checkRebuild(@NotNull final ID indexId, final boolean cleanupOnly) { - final AtomicInteger status = ourRebuildStatus.get(indexId); - if (status.get() == OK) return; - if (status.compareAndSet(REQUIRES_REBUILD, REBUILD_IN_PROGRESS)) { - cleanupProcessedFlag(); - - final Runnable rebuildRunnable = new Runnable() { - @Override - public void run() { - try { - clearIndex(indexId); - if (!cleanupOnly) { - scheduleIndexRebuild(false); - } - } - catch (StorageException e) { - requestRebuild(indexId); - LOG.info(e); - } - finally { - status.compareAndSet(REBUILD_IN_PROGRESS, OK); - } - } - }; - - if (cleanupOnly || myIsUnitTestMode) { - rebuildRunnable.run(); - } - else { - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - new Task.Modal(null, "Updating index", false) { - @Override - public void run(@NotNull final ProgressIndicator indicator) { - indicator.setIndeterminate(true); - rebuildRunnable.run(); - } - }.queue(); - } - }); - } - } - - if (status.get() == REBUILD_IN_PROGRESS) { - throw new ProcessCanceledException(); - } - } - - private void scheduleIndexRebuild(boolean forceDumbMode) { - for (Project project : ProjectManager.getInstance().getOpenProjects()) { - final Set updatersToRun = Collections.singleton(new UnindexedFilesUpdater(project, this)); - final DumbServiceImpl service = DumbServiceImpl.getInstance(project); - if (forceDumbMode) { - service.queueCacheUpdateInDumbMode(updatersToRun); - } - else { - service.queueCacheUpdate(updatersToRun); - } - } - } - - private void clearIndex(@NotNull final ID indexId) throws StorageException { - final UpdatableIndex index = getIndex(indexId); - assert index != null: "Index with key " + indexId + " not found or not registered properly"; - index.clear(); - try { - IndexInfrastructure.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId)); - } - catch (IOException e) { - LOG.error(e); - } - } - - @NotNull - private Set getUnsavedOrTransactedDocuments() { - final Set docs = new THashSet(Arrays.asList(myFileDocumentManager.getUnsavedDocuments())); - synchronized (myTransactionMap) { - docs.addAll(myTransactionMap.keySet()); - } - return docs; - } - - private void indexUnsavedDocuments(@NotNull ID indexId, - @Nullable Project project, - GlobalSearchScope filter, - VirtualFile restrictedFile) throws StorageException { - if (myUpToDateIndices.contains(indexId)) { - return; // no need to index unsaved docs - } - - final Set documents = getUnsavedOrTransactedDocuments(); - if (!documents.isEmpty()) { - // now index unsaved data - final StorageGuard.Holder guard = setDataBufferingEnabled(true); - try { - final Semaphore semaphore = myUnsavedDataIndexingSemaphores.get(indexId); - - assert semaphore != null : "Semaphore for unsaved data indexing was not initialized for index " + indexId; - - semaphore.down(); - boolean allDocsProcessed = true; - try { - for (Document document : documents) { - allDocsProcessed &= indexUnsavedDocument(document, indexId, project, filter, restrictedFile); - } - } - finally { - semaphore.up(); - - while (!semaphore.waitFor(500)) { // may need to wait until another thread is done with indexing - if (Thread.holdsLock(PsiLock.LOCK)) { - break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding. - } - } - if (allDocsProcessed && !hasActiveTransactions()) { - myUpToDateIndices.add(indexId); // safe to set the flag here, because it will be cleared under the WriteAction - } - } - } - finally { - guard.leave(); - } - } - } - - private boolean hasActiveTransactions() { - synchronized (myTransactionMap) { - return !myTransactionMap.isEmpty(); - } - } - - private interface DocumentContent { - String getText(); - long getModificationStamp(); - } - - private static class AuthenticContent implements DocumentContent { - private final Document myDocument; - - private AuthenticContent(final Document document) { - myDocument = document; - } - - @Override - public String getText() { - return myDocument.getText(); - } - - @Override - public long getModificationStamp() { - return myDocument.getModificationStamp(); - } - } - - private static class PsiContent implements DocumentContent { - private final Document myDocument; - private final PsiFile myFile; - - private PsiContent(final Document document, final PsiFile file) { - myDocument = document; - myFile = file; - } - - @Override - public String getText() { - if (myFile.getModificationStamp() != myDocument.getModificationStamp()) { - final ASTNode node = myFile.getNode(); - assert node != null; - return node.getText(); - } - return myDocument.getText(); - } - - @Override - public long getModificationStamp() { - return myFile.getModificationStamp(); - } - } - -// returns false if doc was not indexed because the file does not fit in scope - private boolean indexUnsavedDocument(@NotNull final Document document, @NotNull final ID requestedIndexId, final Project project, - @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedFile) throws StorageException { - final VirtualFile vFile = myFileDocumentManager.getFile(document); - if (!(vFile instanceof VirtualFileWithId) || !vFile.isValid()) { - return true; - } - - if (restrictedFile != null) { - if(vFile != restrictedFile) { - return false; - } - } - else if (filter != null && !filter.accept(vFile)) { - return false; - } - - final PsiFile dominantContentFile = findDominantPsiForDocument(document, project); - - final DocumentContent content; - if (dominantContentFile != null && dominantContentFile.getModificationStamp() != document.getModificationStamp()) { - content = new PsiContent(document, dominantContentFile); - } - else { - content = new AuthenticContent(document); - } - - final long currentDocStamp = content.getModificationStamp(); - if (currentDocStamp != myLastIndexedDocStamps.getAndSet(document, requestedIndexId, currentDocStamp)) { - final Ref exRef = new Ref(null); - ProgressManager.getInstance().executeNonCancelableSection(new Runnable() { - @Override - public void run() { - try { - final String contentText = content.getText(); - if (isTooLarge(vFile, contentText.length())) { - return; - } - - final FileContentImpl newFc = new FileContentImpl(vFile, contentText, vFile.getCharset()); - - if (dominantContentFile != null) { - dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, true); - newFc.putUserData(IndexingDataKeys.PSI_FILE, dominantContentFile); - } - - if (content instanceof AuthenticContent) { - newFc.putUserData(EDITOR_HIGHLIGHTER, EditorHighlighterCache.getEditorHighlighterForCachesBuilding(document)); - } - - if (getInputFilter(requestedIndexId).acceptInput(vFile)) { - newFc.putUserData(IndexingDataKeys.PROJECT, project); - final int inputId = Math.abs(getFileId(vFile)); - getIndex(requestedIndexId).update(inputId, newFc); - } - - if (dominantContentFile != null) { - dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, null); - } - } - catch (StorageException e) { - exRef.set(e); - } - } - }); - final StorageException storageException = exRef.get(); - if (storageException != null) { - throw storageException; - } - } - return true; - } - - public static final Key EDITOR_HIGHLIGHTER = new Key("Editor"); - - @Nullable - private PsiFile findDominantPsiForDocument(@NotNull Document document, @Nullable Project project) { - synchronized (myTransactionMap) { - PsiFile psiFile = myTransactionMap.get(document); - if (psiFile != null) return psiFile; - } - - return project == null ? null : findLatestKnownPsiForUncomittedDocument(document, project); - } - - private final StorageGuard myStorageLock = new StorageGuard(); - - @NotNull - private StorageGuard.Holder setDataBufferingEnabled(final boolean enabled) { - final StorageGuard.Holder holder = myStorageLock.enter(enabled); - for (ID indexId : myIndices.keySet()) { - final MapReduceIndex index = (MapReduceIndex)getIndex(indexId); - assert index != null; - final IndexStorage indexStorage = index.getStorage(); - ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled); - } - return holder; - } - - private void cleanupMemoryStorage() { - myLastIndexedDocStamps.clear(); - for (ID indexId : myIndices.keySet()) { - final MapReduceIndex index = (MapReduceIndex)getIndex(indexId); - assert index != null; - final MemoryIndexStorage memStorage = (MemoryIndexStorage)index.getStorage(); - index.getWriteLock().lock(); - try { - memStorage.clearMemoryMap(); - } - finally { - index.getWriteLock().unlock(); - } - memStorage.fireMemoryStorageCleared(); - } - } - - private void dropUnregisteredIndices() { - final Set indicesToDrop = readRegisteredIndexNames(); - for (ID key : myIndices.keySet()) { - indicesToDrop.remove(key.toString()); - } - for (String s : indicesToDrop) { - FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s))); - } - } - - public static void requestRebuild(ID indexId) { - requestRebuild(indexId, new Throwable()); - } - - public static void requestRebuild(ID indexId, Throwable throwable) { - cleanupProcessedFlag(); - LOG.info("Rebuild requested for index " + indexId, throwable); - ourRebuildStatus.get(indexId).set(REQUIRES_REBUILD); - } - - private UpdatableIndex getIndex(ID indexId) { - final Pair, InputFilter> pair = myIndices.get(indexId); - - assert pair != null : "Index data is absent for index " + indexId; - - //noinspection unchecked - return (UpdatableIndex)pair.getFirst(); - } - - private InputFilter getInputFilter(ID indexId) { - final Pair, InputFilter> pair = myIndices.get(indexId); - - assert pair != null : "Index data is absent for index " + indexId; - - return pair.getSecond(); - } - - public int getNumberOfPendingInvalidations() { - return myChangedFilesCollector.getNumberOfPendingInvalidations(); - } - - @NotNull - public Collection getFilesToUpdate(final Project project) { - return ContainerUtil.findAll(myChangedFilesCollector.getAllFilesToUpdate(), new Condition() { - @Override - public boolean value(VirtualFile virtualFile) { - for (IndexableFileSet set : myIndexableSets) { - final Project proj = myIndexableSetToProjectMap.get(set); - if (proj != null && !proj.equals(project)) { - continue; // skip this set as associated with a different project - } - if (set.isInSet(virtualFile)) { - return true; - } - } - return false; - } - }); - } - - public void processRefreshedFile(@NotNull Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) { - myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); - myChangedFilesCollector.processFileImpl(project, fileContent, false); - } - - public void indexFileContent(@Nullable Project project, @NotNull com.intellij.ide.caches.FileContent content) { - myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); - final VirtualFile file = content.getVirtualFile(); - FileContentImpl fc = null; - - PsiFile psiFile = null; - - FileTypeManagerImpl.cacheFileType(file, file.getFileType()); - try { - for (final ID indexId : myIndices.keySet()) { - if (shouldIndexFile(file, indexId)) { - if (fc == null) { - byte[] currentBytes; - try { - currentBytes = content.getBytes(); - } - catch (IOException e) { - currentBytes = ArrayUtil.EMPTY_BYTE_ARRAY; - } - fc = new FileContentImpl(file, currentBytes); - - psiFile = content.getUserData(IndexingDataKeys.PSI_FILE); - if (psiFile != null) { - psiFile.putUserData(PsiFileImpl.BUILDING_STUB, true); - fc.putUserData(IndexingDataKeys.PSI_FILE, psiFile); - } - if (project == null) { - project = ProjectUtil.guessProjectForFile(file); - } - fc.putUserData(IndexingDataKeys.PROJECT, project); - } - - try { - ProgressManager.checkCanceled(); - updateSingleIndex(indexId, file, fc); - } - catch (ProcessCanceledException e) { - myChangedFilesCollector.scheduleForUpdate(file); - throw e; - } - catch (StorageException e) { - requestRebuild(indexId); - LOG.info(e); - } - } - } - - if (psiFile != null) { - psiFile.putUserData(PsiFileImpl.BUILDING_STUB, null); - } - } finally { - FileTypeManagerImpl.cacheFileType(file, null); - } - } - - private void updateSingleIndex(final ID indexId, @NotNull final VirtualFile file, @Nullable final FileContent currentFC) - throws StorageException { - if (ourRebuildStatus.get(indexId).get() == REQUIRES_REBUILD) { - return; // the index is scheduled for rebuild, no need to update - } - myLocalModCount++; - - final StorageGuard.Holder lock = setDataBufferingEnabled(false); - - try { - final int inputId = Math.abs(getFileId(file)); - - final UpdatableIndex index = getIndex(indexId); - assert index != null; - - final Ref exRef = new Ref(null); - ProgressManager.getInstance().executeNonCancelableSection(new Runnable() { - @Override - public void run() { - try { - index.update(inputId, currentFC); - } - catch (StorageException e) { - exRef.set(e); - } - } - }); - final StorageException storageException = exRef.get(); - if (storageException != null) { - throw storageException; - } - ApplicationManager.getApplication().runReadAction(new Runnable() { - @Override - public void run() { - if (file.isValid()) { - if (currentFC != null) { - IndexingStamp.update(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)); - } - else { - // mark the file as unindexed - IndexingStamp.update(file, indexId, -1L); - } - } - } - }); - } - finally { - lock.leave(); - } + return ApplicationManager.getApplication().getComponent(FileBasedIndex.class); } public static int getFileId(@NotNull final VirtualFile file) { @@ -1693,653 +47,68 @@ public class FileBasedIndex implements ApplicationComponent { throw new IllegalArgumentException("Virtual file doesn't support id: " + file + ", implementation class: " + file.getClass().getName()); } - private boolean needsFileContentLoading(ID indexId) { - return !myNotRequiringContentIndices.contains(indexId); + public void requestRebuild(ID indexId) { + requestRebuild(indexId, new Throwable()); } - private abstract static class InvalidationTask implements Runnable { - private final VirtualFile mySubj; - protected InvalidationTask(final VirtualFile subj) { - mySubj = subj; - } - - public VirtualFile getSubj() { - return mySubj; - } - } - - private final class ChangedFilesCollector extends VirtualFileAdapter { - private final Set myFilesToUpdate = new ConcurrentHashSet(); - private final Queue myFutureInvalidations = new ConcurrentLinkedQueue(); - - private final ManagingFS myManagingFS = ManagingFS.getInstance(); - // No need to react on movement events since files stay valid, their ids don't change and all associated attributes remain intact. - - @Override - public void fileCreated(@NotNull final VirtualFileEvent event) { - markDirty(event, false); - } - - @Override - public void fileDeleted(@NotNull final VirtualFileEvent event) { - myFilesToUpdate.remove(event.getFile()); // no need to update it anymore - } - - @Override - public void fileCopied(@NotNull final VirtualFileCopyEvent event) { - markDirty(event, false); - } - - @Override - public void beforeFileDeletion(@NotNull final VirtualFileEvent event) { - invalidateIndices(event.getFile(), false); - } - - @Override - public void beforeContentsChange(@NotNull final VirtualFileEvent event) { - invalidateIndices(event.getFile(), true); - } - - @Override - public void contentsChanged(@NotNull final VirtualFileEvent event) { - markDirty(event, true); - } - - @Override - public void beforePropertyChange(@NotNull final VirtualFilePropertyEvent event) { - if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) { - // indexes may depend on file name - final VirtualFile file = event.getFile(); - if (!file.isDirectory()) { - // name change may lead to filetype change so the file might become not indexable - // in general case have to 'unindex' the file and index it again if needed after the name has been changed - invalidateIndices(file, false); - } - } - } - - @Override - public void propertyChanged(@NotNull final VirtualFilePropertyEvent event) { - if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) { - // indexes may depend on file name - if (!event.getFile().isDirectory()) { - markDirty(event, false); - } - } - } - - private void markDirty(@NotNull final VirtualFileEvent event, final boolean contentChange) { - final VirtualFile eventFile = event.getFile(); - cleanProcessedFlag(eventFile); - iterateIndexableFiles(eventFile, new Processor() { - @Override - public boolean process(@NotNull final VirtualFile file) { - if (!contentChange) ++myFilesModCount; - FileContent fileContent = null; - // handle 'content-less' indices separately - for (ID indexId : myNotRequiringContentIndices) { - if (getInputFilter(indexId).acceptInput(file)) { - try { - if (fileContent == null) { - fileContent = new FileContentImpl(file); - } - updateSingleIndex(indexId, file, fileContent); - } - catch (StorageException e) { - LOG.info(e); - requestRebuild(indexId); - } - } - } - // For 'normal indices' schedule the file for update and stop iteration if at least one index accepts it - if (!isTooLarge(file)) { - for (ID indexId : myIndices.keySet()) { - if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) { - scheduleForUpdate(file); - break; // no need to iterate further, as the file is already marked - } - } - } - - return true; - } - }); - IndexingStamp.flushCache(); - } - - public void scheduleForUpdate(VirtualFile file) { - myFilesToUpdate.add(file); - } - - void invalidateIndices(@NotNull final VirtualFile file, final boolean markForReindex) { - if (isUnderConfigOrSystem(file)) { - return; - } - if (file.isDirectory()) { - if (isMock(file) || myManagingFS.wereChildrenAccessed(file)) { - final Iterable children = file instanceof NewVirtualFile - ? ((NewVirtualFile)file).iterInDbChildren() : Arrays.asList(file.getChildren()); - for (VirtualFile child : children) { - invalidateIndices(child, markForReindex); - } - } - } - else { - cleanProcessedFlag(file); - IndexingStamp.flushCache(); - final List> affectedIndices = new ArrayList>(myIndices.size()); - - for (final ID indexId : myIndices.keySet()) { - try { - if (!needsFileContentLoading(indexId)) { - if (shouldUpdateIndex(file, indexId)) { - updateSingleIndex(indexId, file, null); - } - } - else { // the index requires file content - if (shouldUpdateIndex(file, indexId)) { - affectedIndices.add(indexId); - } - } - } - catch (StorageException e) { - LOG.info(e); - requestRebuild(indexId); - } - } - - if (!affectedIndices.isEmpty()) { - if (markForReindex && !isTooLarge(file)) { - // only mark the file as unindexed, reindex will be done lazily - ApplicationManager.getApplication().runReadAction(new Runnable() { - @Override - public void run() { - for (ID indexId : affectedIndices) { - IndexingStamp.update(file, indexId, -2L); - } - } - }); - // the file is for sure not a dir and it was previously indexed by at least one index - scheduleForUpdate(file); - } - else { - myFutureInvalidations.offer(new InvalidationTask(file) { - @Override - public void run() { - removeFileDataFromIndices(affectedIndices, file); - } - }); - } - } - if (!markForReindex) { - final boolean removedFromUpdateQueue = myFilesToUpdate.remove(file);// no need to update it anymore - if (removedFromUpdateQueue && affectedIndices.isEmpty()) { - // Currently the file is about to be deleted and previously it was scheduled for update and not processed up to now. - // Because the file was scheduled for update, at the moment of scheduling it was marked as unindexed, - // so, to be on the safe side, we have to schedule data invalidation from all content-requiring indices for this file - myFutureInvalidations.offer(new InvalidationTask(file) { - @Override - public void run() { - removeFileDataFromIndices(myRequiringContentIndices, file); - } - }); - } - } - - IndexingStamp.flushCache(); - } - } - - private void removeFileDataFromIndices(@NotNull Collection> affectedIndices, @NotNull VirtualFile file) { - Throwable unexpectedError = null; - for (ID indexId : affectedIndices) { - try { - updateSingleIndex(indexId, file, null); - } - catch (StorageException e) { - LOG.info(e); - requestRebuild(indexId); - } - catch (ProcessCanceledException ignored) { - } - catch (Throwable e) { - LOG.info(e); - if (unexpectedError == null) { - unexpectedError = e; - } - } - } - IndexingStamp.flushCache(); - if (unexpectedError != null) { - LOG.error(unexpectedError); - } - } - - public int getNumberOfPendingInvalidations() { - return myFutureInvalidations.size(); - } - - public void ensureAllInvalidateTasksCompleted() { - final int size = getNumberOfPendingInvalidations(); - if (size == 0) { - return; - } - final ProgressIndicator current = ProgressManager.getInstance().getProgressIndicator(); - final ProgressIndicator indicator = current != null ? current : new EmptyProgressIndicator(); - indicator.setText(""); - int count = 0; - while (true) { - InvalidationTask task = myFutureInvalidations.poll(); - - if (task == null) { - break; - } - indicator.setFraction((double)count++ /size); - indicator.setText2(task.getSubj().getPresentableUrl()); - task.run(); - } - } - - private void iterateIndexableFiles(@NotNull final VirtualFile file, @NotNull final Processor processor) { - if (file.isDirectory()) { - final ContentIterator iterator = new ContentIterator() { - @Override - public boolean processFile(@NotNull final VirtualFile fileOrDir) { - if (!fileOrDir.isDirectory()) { - processor.process(fileOrDir); - } - return true; - } - }; - - for (IndexableFileSet set : myIndexableSets) { - if (set.isInSet(file)) { - set.iterateIndexableFilesIn(file, iterator); - } - } - } - else { - for (IndexableFileSet set : myIndexableSets) { - if (set.isInSet(file)) { - processor.process(file); - break; - } - } - } - } - - public Collection getAllFilesToUpdate() { - if (myFilesToUpdate.isEmpty()) { - return Collections.emptyList(); - } - return new ArrayList(myFilesToUpdate); - } - - private final Semaphore myForceUpdateSemaphore = new Semaphore(); - - private void forceUpdate(@Nullable Project project, @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedTo, - boolean onlyRemoveOutdatedData) { - myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); - for (VirtualFile file: getAllFilesToUpdate()) { - if (filter == null || filter.accept(file) || file == restrictedTo) { - try { - myForceUpdateSemaphore.down(); - // process only files that can affect result - processFileImpl(project, new com.intellij.ide.caches.FileContent(file), onlyRemoveOutdatedData); - } - finally { - myForceUpdateSemaphore.up(); - } - } - } - - // If several threads entered the method at the same time and there were files to update, - // all the threads should leave the method synchronously after all the files scheduled for update are reindexed, - // no matter which thread will do reindexing job. - // Thus we ensure that all the threads that entered the method will get the most recent data - - while (!myForceUpdateSemaphore.waitFor(500)) { // may need to wait until another thread is done with indexing - if (Thread.holdsLock(PsiLock.LOCK)) { - break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding. - } - } - } - - private void processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent, boolean onlyRemoveOutdatedData) { - final VirtualFile file = fileContent.getVirtualFile(); - final boolean reallyRemoved = myFilesToUpdate.remove(file); - if (reallyRemoved && file.isValid()) { - if (onlyRemoveOutdatedData) { - // on shutdown there is no need to re-index the file, just remove outdated data from indices - final List> affected = new ArrayList>(); - for (final ID indexId : myIndices.keySet()) { - if (getInputFilter(indexId).acceptInput(file)) { - affected.add(indexId); - } - } - removeFileDataFromIndices(affected, file); - } - else { - indexFileContent(project, fileContent); - } - IndexingStamp.flushCache(); - } - } - } - - private class UnindexedFilesFinder implements CollectingContentIterator { - private final List myFiles = new ArrayList(); - private final ProgressIndicator myProgressIndicator; - - private UnindexedFilesFinder() { - myProgressIndicator = ProgressManager.getInstance().getProgressIndicator(); - } - - @NotNull - @Override - public List getFiles() { - return myFiles; - } - - @Override - public boolean processFile(@NotNull final VirtualFile file) { - if (!file.isValid()) { - return true; - } - if (!file.isDirectory()) { - if (file instanceof NewVirtualFile && ((NewVirtualFile)file).getFlag(ALREADY_PROCESSED)) { - return true; - } - - if (file instanceof VirtualFileWithId) { - try { - FileTypeManagerImpl.cacheFileType(file, file.getFileType()); - - boolean oldStuff = true; - if (!isTooLarge(file)) { - for (ID indexId : myIndices.keySet()) { - try { - if (needsFileContentLoading(indexId) && shouldIndexFile(file, indexId)) { - myFiles.add(file); - oldStuff = false; - break; - } - } - catch (RuntimeException e) { - final Throwable cause = e.getCause(); - if (cause instanceof IOException || cause instanceof StorageException) { - LOG.info(e); - requestRebuild(indexId); - } - else { - throw e; - } - } - } - } - FileContent fileContent = null; - for (ID indexId : myNotRequiringContentIndices) { - if (shouldIndexFile(file, indexId)) { - oldStuff = false; - try { - if (fileContent == null) { - fileContent = new FileContentImpl(file); - } - updateSingleIndex(indexId, file, fileContent); - } - catch (StorageException e) { - LOG.info(e); - requestRebuild(indexId); - } - } - } - IndexingStamp.flushCache(); - - if (oldStuff && file instanceof NewVirtualFile) { - ((NewVirtualFile)file).setFlag(ALREADY_PROCESSED, true); - } - } - finally { - FileTypeManagerImpl.cacheFileType(file, null); - } - } - } - else { - if (myProgressIndicator != null) { - myProgressIndicator.setText("Scanning files to index"); - myProgressIndicator.setText2(file.getPresentableUrl()); - } - } - return true; - } - } - - private boolean shouldUpdateIndex(final VirtualFile file, final ID indexId) { - return getInputFilter(indexId).acceptInput(file) && - (isMock(file) || IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId))); - } - - private boolean shouldIndexFile(final VirtualFile file, final ID indexId) { - return getInputFilter(indexId).acceptInput(file) && - (isMock(file) || !IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId))); - } - - private boolean isUnderConfigOrSystem(@NotNull VirtualFile file) { - final String filePath = file.getPath(); - return myConfigPath != null && FileUtil.startsWith(filePath, myConfigPath) || - mySystemPath != null && FileUtil.startsWith(filePath, mySystemPath); - } - - private static boolean isMock(final VirtualFile file) { - return !(file instanceof NewVirtualFile); - } - - private boolean isTooLarge(@NotNull VirtualFile file) { - if (SingleRootFileViewProvider.isTooLarge(file)) { - final FileType type = file.getFileType(); - return !myNoLimitCheckTypes.contains(type); - } - return false; - } - - private boolean isTooLarge(@NotNull VirtualFile file, long contentSize) { - if (SingleRootFileViewProvider.isTooLarge(file, contentSize)) { - final FileType type = file.getFileType(); - return !myNoLimitCheckTypes.contains(type); - } - return false; + @NonNls + @NotNull + public String getComponentName() { + return "FileBasedIndex"; } @NotNull - public CollectingContentIterator createContentIterator() { - ++myFilesModCount; - return new UnindexedFilesFinder(); - } + public abstract List getValues(@NotNull ID indexId, @NotNull K dataKey, @NotNull GlobalSearchScope filter); - public void registerIndexableSet(@NotNull IndexableFileSet set, @Nullable Project project) { - myIndexableSets.add(set); - myIndexableSetToProjectMap.put(set, project); - } + @NotNull + public abstract Collection getContainingFiles(@NotNull ID indexId, + @NotNull K dataKey, + @NotNull GlobalSearchScope filter); - public void removeIndexableSet(@NotNull IndexableFileSet set) { - myChangedFilesCollector.forceUpdate(null, null, null, true); - myIndexableSets.remove(set); - myIndexableSetToProjectMap.remove(set); - } + public abstract boolean processValues(@NotNull ID indexId, + @NotNull K dataKey, + @Nullable VirtualFile inFile, + @NotNull FileBasedIndex.ValueProcessor processor, + @NotNull GlobalSearchScope filter); - @Nullable - private static PsiFile findLatestKnownPsiForUncomittedDocument(@NotNull Document doc, @NotNull Project project) { - return PsiDocumentManager.getInstance(project).getCachedPsiFile(doc); - } - - private static class IndexableFilesFilter implements InputFilter { - private final InputFilter myDelegate; + public abstract boolean processFilesContainingAllKeys(@NotNull ID indexId, + @NotNull Collection dataKeys, + @NotNull GlobalSearchScope filter, + @Nullable Condition valueChecker, + @NotNull Processor processor); - private IndexableFilesFilter(InputFilter delegate) { - myDelegate = delegate; - } + @NotNull + public abstract Collection getAllKeys(@NotNull ID indexId, @NotNull Project project); - @Override - public boolean acceptInput(final VirtualFile file) { - return file instanceof VirtualFileWithId && myDelegate.acceptInput(file); - } - } + public abstract void ensureUpToDate(@NotNull ID indexId, @Nullable Project project, @Nullable GlobalSearchScope filter); - private static void cleanupProcessedFlag() { - final VirtualFile[] roots = ManagingFS.getInstance().getRoots(); - for (VirtualFile root : roots) { - cleanProcessedFlag(root); - } - } + protected abstract void ensureUpToDate(@NotNull ID indexId, + @Nullable Project project, + @Nullable GlobalSearchScope filter, + @Nullable VirtualFile restrictedFile); - private static void cleanProcessedFlag(@NotNull final VirtualFile file) { - if (!(file instanceof NewVirtualFile)) return; - - final NewVirtualFile nvf = (NewVirtualFile)file; - if (file.isDirectory()) { - for (VirtualFile child : nvf.getCachedChildren()) { - cleanProcessedFlag(child); - } - } - else { - nvf.setFlag(ALREADY_PROCESSED, false); - } - } + public abstract void requestRebuild(ID indexId, Throwable throwable); - public static void iterateIndexableFiles(@NotNull final ContentIterator processor, @NotNull Project project, ProgressIndicator indicator) { - if (project.isDisposed()) { - return; - } - final ProjectFileIndex projectFileIndex = ProjectRootManager.getInstance(project).getFileIndex(); - // iterate project content - projectFileIndex.iterateContent(processor); + public abstract void scheduleRebuild(@NotNull ID indexId, @NotNull Throwable e); - if (project.isDisposed()) { - return; - } + public abstract void requestReindex(@NotNull VirtualFile file); - Set visitedRoots = new THashSet(); - for (IndexedRootsProvider provider : Extensions.getExtensions(IndexedRootsProvider.EP_NAME)) { - //important not to depend on project here, to support per-project background reindex - // each client gives a project to FileBasedIndex - if (project.isDisposed()) { - return; - } - for (VirtualFile root : IndexableSetContributor.getRootsToIndex(provider)) { - if (visitedRoots.add(root)) { - iterateRecursively(root, processor, indicator); - } - } - for (VirtualFile root : IndexableSetContributor.getProjectRootsToIndex(provider, project)) { - if (visitedRoots.add(root)) { - iterateRecursively(root, processor, indicator); - } - } - } + public abstract void requestReindexExcluded(@NotNull VirtualFile file); - if (project.isDisposed()) { - return; - } - // iterate associated libraries - for (Module module : ModuleManager.getInstance(project).getModules()) { - if (module.isDisposed()) { - return; - } - OrderEntry[] orderEntries = ModuleRootManager.getInstance(module).getOrderEntries(); - for (OrderEntry orderEntry : orderEntries) { - if (orderEntry instanceof LibraryOrderEntry || orderEntry instanceof JdkOrderEntry) { - if (orderEntry.isValid()) { - final VirtualFile[] libSources = orderEntry.getFiles(OrderRootType.SOURCES); - final VirtualFile[] libClasses = orderEntry.getFiles(OrderRootType.CLASSES); - for (VirtualFile[] roots : new VirtualFile[][]{libSources, libClasses}) { - for (VirtualFile root : roots) { - if (visitedRoots.add(root)) { - iterateRecursively(root, processor, indicator); - } - } - } - } - } - } - } - } + public abstract boolean getFilesWithKey(@NotNull ID indexId, + @NotNull Set dataKeys, + @NotNull Processor processor, + @NotNull GlobalSearchScope filter); - private static void iterateRecursively(@Nullable final VirtualFile root, @NotNull final ContentIterator processor, @Nullable ProgressIndicator indicator) { - if (root != null) { - if (indicator != null) { - indicator.checkCanceled(); - indicator.setText2(root.getPresentableUrl()); - } - - if (root.isDirectory()) { - for (VirtualFile file : root.getChildren()) { - if (file.isDirectory()) { - iterateRecursively(file, processor, indicator); - } - else { - processor.processFile(file); - } - } - } - else { - processor.processFile(root); - } - } - } - - private static class StorageGuard { - private int myHolds = 0; - - public interface Holder { - void leave(); - } - - private final Holder myTrueHolder = new Holder() { - @Override - public void leave() { - StorageGuard.this.leave(true); - } - }; - private final Holder myFalseHolder = new Holder() { - @Override - public void leave() { - StorageGuard.this.leave(false); - } - }; - - @NotNull - public synchronized Holder enter(boolean mode) { - if (mode) { - while (myHolds < 0) { - try { - wait(); - } - catch (InterruptedException ignored) { - } - } - myHolds++; - return myTrueHolder; - } - else { - while (myHolds > 0) { - try { - wait(); - } - catch (InterruptedException ignored) { - } - } - myHolds--; - return myFalseHolder; - } - } - - private synchronized void leave(boolean mode) { - myHolds += mode? -1 : 1; - if (myHolds == 0) { - notifyAll(); - } - } + public abstract boolean processAllKeys(@NotNull ID indexId, Processor processor, @Nullable Project project); + public interface ValueProcessor { + /** + * @param value a value to process + * @param file the file the value came from + * @return false if no further processing is needed, true otherwise + */ + boolean process(VirtualFile file, V value); } } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java new file mode 100644 index 000000000000..1cf3b2677d89 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -0,0 +1,2327 @@ +/* + * Copyright 2000-2012 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intellij.util.indexing; + +import com.intellij.AppTopics; +import com.intellij.history.LocalHistory; +import com.intellij.ide.caches.CacheUpdater; +import com.intellij.lang.ASTNode; +import com.intellij.notification.NotificationDisplayType; +import com.intellij.notification.NotificationGroup; +import com.intellij.notification.NotificationType; +import com.intellij.openapi.application.ApplicationAdapter; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.application.PathManager; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.editor.Document; +import com.intellij.openapi.editor.highlighter.EditorHighlighter; +import com.intellij.openapi.editor.impl.EditorHighlighterCache; +import com.intellij.openapi.extensions.Extensions; +import com.intellij.openapi.fileEditor.FileDocumentManager; +import com.intellij.openapi.fileEditor.FileDocumentManagerAdapter; +import com.intellij.openapi.fileTypes.*; +import com.intellij.openapi.fileTypes.impl.FileTypeManagerImpl; +import com.intellij.openapi.module.Module; +import com.intellij.openapi.module.ModuleManager; +import com.intellij.openapi.progress.*; +import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator; +import com.intellij.openapi.project.*; +import com.intellij.openapi.roots.*; +import com.intellij.openapi.util.*; +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.registry.Registry; +import com.intellij.openapi.vfs.*; +import com.intellij.openapi.vfs.ex.VirtualFileManagerEx; +import com.intellij.openapi.vfs.newvfs.BulkFileListener; +import com.intellij.openapi.vfs.newvfs.ManagingFS; +import com.intellij.openapi.vfs.newvfs.NewVirtualFile; +import com.intellij.openapi.vfs.newvfs.events.VFileEvent; +import com.intellij.openapi.vfs.newvfs.persistent.FlushingDaemon; +import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS; +import com.intellij.psi.*; +import com.intellij.psi.impl.PsiDocumentTransactionListener; +import com.intellij.psi.impl.source.PsiFileImpl; +import com.intellij.psi.search.EverythingGlobalScope; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.psi.stubs.SerializationManager; +import com.intellij.util.*; +import com.intellij.util.concurrency.Semaphore; +import com.intellij.util.containers.ConcurrentHashSet; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.io.*; +import com.intellij.util.io.DataOutputStream; +import com.intellij.util.io.storage.HeavyProcessLatch; +import com.intellij.util.messages.MessageBus; +import com.intellij.util.messages.MessageBusConnection; +import gnu.trove.*; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import javax.swing.*; +import java.io.*; +import java.lang.ref.SoftReference; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Eugene Zhuravlev + * Date: Dec 20, 2007 + */ + +public class FileBasedIndexImpl extends FileBasedIndex { + private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.FileBasedIndexImpl"); + @NonNls + private static final String CORRUPTION_MARKER_NAME = "corruption.marker"; + private final Map, Pair, InputFilter>> myIndices = new THashMap, Pair, InputFilter>>(); + private final Map, Semaphore> myUnsavedDataIndexingSemaphores = new THashMap, Semaphore>(); + private final TObjectIntHashMap> myIndexIdToVersionMap = new TObjectIntHashMap>(); + private final Set> myNotRequiringContentIndices = new THashSet>(); + private final Set> myRequiringContentIndices = new THashSet>(); + private final Set myNoLimitCheckTypes = new THashSet(); + + private final PerIndexDocumentVersionMap myLastIndexedDocStamps = new PerIndexDocumentVersionMap(); + @NotNull private final ChangedFilesCollector myChangedFilesCollector; + + private final List myIndexableSets = ContainerUtil.createEmptyCOWList(); + private final Map myIndexableSetToProjectMap = new THashMap(); + + private static final int OK = 1; + private static final int REQUIRES_REBUILD = 2; + private static final int REBUILD_IN_PROGRESS = 3; + private static final Map, AtomicInteger> ourRebuildStatus = new THashMap, AtomicInteger>(); + + private final VirtualFileManagerEx myVfManager; + private final FileDocumentManager myFileDocumentManager; + private final FileTypeManager myFileTypeManager; + private final ConcurrentHashSet> myUpToDateIndices = new ConcurrentHashSet>(); + private final Map myTransactionMap = new THashMap(); + + private static final int ALREADY_PROCESSED = 0x04; + + @Nullable private final String myConfigPath; + @Nullable private final String mySystemPath; + private final boolean myIsUnitTestMode; + @Nullable private ScheduledFuture myFlushingFuture; + private volatile int myLocalModCount; + private volatile int myFilesModCount; + + @Override + public void requestReindex(@NotNull final VirtualFile file) { + myChangedFilesCollector.invalidateIndices(file, true); + } + + @Override + public void requestReindexExcluded(@NotNull final VirtualFile file) { + myChangedFilesCollector.invalidateIndices(file, false); + } + + public FileBasedIndexImpl(final VirtualFileManagerEx vfManager, + FileDocumentManager fdm, + FileTypeManager fileTypeManager, + @NotNull MessageBus bus, + SerializationManager sm + /*need this parameter to ensure component dependency*/) throws IOException { + myVfManager = vfManager; + myFileDocumentManager = fdm; + myFileTypeManager = fileTypeManager; + myIsUnitTestMode = ApplicationManager.getApplication().isUnitTestMode(); + myConfigPath = calcConfigPath(PathManager.getConfigPath()); + mySystemPath = calcConfigPath(PathManager.getSystemPath()); + + final MessageBusConnection connection = bus.connect(); + connection.subscribe(PsiDocumentTransactionListener.TOPIC, new PsiDocumentTransactionListener() { + @Override + public void transactionStarted(final Document doc, final PsiFile file) { + if (file != null) { + synchronized (myTransactionMap) { + myTransactionMap.put(doc, file); + } + myUpToDateIndices.clear(); + } + } + + @Override + public void transactionCompleted(final Document doc, final PsiFile file) { + synchronized (myTransactionMap) { + myTransactionMap.remove(doc); + } + } + }); + + connection.subscribe(FileTypeManager.TOPIC, new FileTypeListener() { + @Nullable private Map> myTypeToExtensionMap; + @Override + public void beforeFileTypesChanged(final FileTypeEvent event) { + cleanupProcessedFlag(); + myTypeToExtensionMap = new THashMap>(); + for (FileType type : myFileTypeManager.getRegisteredFileTypes()) { + myTypeToExtensionMap.put(type, getExtensions(type)); + } + } + + @Override + public void fileTypesChanged(final FileTypeEvent event) { + final Map> oldExtensions = myTypeToExtensionMap; + myTypeToExtensionMap = null; + if (oldExtensions != null) { + final Map> newExtensions = new THashMap>(); + for (FileType type : myFileTypeManager.getRegisteredFileTypes()) { + newExtensions.put(type, getExtensions(type)); + } + // we are interested only in extension changes or removals. + // addition of an extension is handled separately by RootsChanged event + if (!newExtensions.keySet().containsAll(oldExtensions.keySet())) { + rebuildAllIndices(); + return; + } + for (Map.Entry> entry : oldExtensions.entrySet()) { + FileType fileType = entry.getKey(); + Set strings = entry.getValue(); + if (!newExtensions.get(fileType).containsAll(strings)) { + rebuildAllIndices(); + return; + } + } + } + } + + @NotNull + private Set getExtensions(@NotNull FileType type) { + final Set set = new THashSet(); + for (FileNameMatcher matcher : myFileTypeManager.getAssociations(type)) { + set.add(matcher.getPresentableString()); + } + return set; + } + + private void rebuildAllIndices() { + for (ID indexId : myIndices.keySet()) { + try { + clearIndex(indexId); + } + catch (StorageException e) { + LOG.info(e); + } + } + scheduleIndexRebuild(true); + } + }); + + connection.subscribe(VirtualFileManager.VFS_CHANGES, new BulkFileListener() { + @Override + public void before(@NotNull List events) { + for (VFileEvent event : events) { + final Object requestor = event.getRequestor(); + if (requestor instanceof FileDocumentManager || requestor instanceof PsiManager || requestor == LocalHistory.VFS_EVENT_REQUESTOR) { + cleanupMemoryStorage(); + break; + } + } + } + + @Override + public void after(@NotNull List events) { + } + }); + + connection.subscribe(AppTopics.FILE_DOCUMENT_SYNC, new FileDocumentManagerAdapter() { + @Override + public void fileContentReloaded(VirtualFile file, @NotNull Document document) { + cleanupMemoryStorage(); + } + + @Override + public void unsavedDocumentsDropped() { + cleanupMemoryStorage(); + } + }); + + ApplicationManager.getApplication().addApplicationListener(new ApplicationAdapter() { + @Override + public void writeActionStarted(Object action) { + myUpToDateIndices.clear(); + } + }); + + myChangedFilesCollector = new ChangedFilesCollector(); + + /* + final File workInProgressFile = getMarkerFile(); + if (workInProgressFile.exists()) { + // previous IDEA session was closed incorrectly, so drop all indices + FileUtil.delete(PathManager.getIndexRoot()); + } + */ + + try { + final FileBasedIndexExtension[] extensions = Extensions.getExtensions(FileBasedIndexExtension.EXTENSION_POINT_NAME); + for (FileBasedIndexExtension extension : extensions) { + ourRebuildStatus.put(extension.getName(), new AtomicInteger(OK)); + } + + final File corruptionMarker = new File(PathManager.getIndexRoot(), CORRUPTION_MARKER_NAME); + final boolean currentVersionCorrupted = corruptionMarker.exists(); + boolean versionChanged = false; + for (FileBasedIndexExtension extension : extensions) { + versionChanged |= registerIndexer(extension, currentVersionCorrupted); + } + FileUtil.delete(corruptionMarker); + + String rebuildNotification = null; + if (currentVersionCorrupted) { + rebuildNotification = "Index files on disk are corrupted. Indices will be rebuilt."; + } + else if (versionChanged) { + rebuildNotification = "Index file format has changed for some indices. These indices will be rebuilt."; + } + if (rebuildNotification != null + && !ApplicationManager.getApplication().isHeadlessEnvironment() + && Registry.is("ide.showIndexRebuildMessage")) { + new NotificationGroup("Indexing", NotificationDisplayType.BALLOON, false) + .createNotification("Index Rebuild", rebuildNotification, NotificationType.INFORMATION, null).notify(null); + } + + dropUnregisteredIndices(); + + // check if rebuild was requested for any index during registration + for (ID indexId : myIndices.keySet()) { + if (ourRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, OK)) { + try { + clearIndex(indexId); + } + catch (StorageException e) { + requestRebuild(indexId); + LOG.error(e); + } + } + } + + myVfManager.addVirtualFileListener(myChangedFilesCollector); + + registerIndexableSet(new AdditionalIndexableFileSet(), null); + } + finally { + ShutDownTracker.getInstance().registerShutdownTask(new Runnable() { + @Override + public void run() { + performShutdown(); + } + }); + //FileUtil.createIfDoesntExist(workInProgressFile); + saveRegisteredIndices(myIndices.keySet()); + myFlushingFuture = FlushingDaemon.everyFiveSeconds(new Runnable() { + int lastModCount = 0; + @Override + public void run() { + if (lastModCount == myLocalModCount) { + flushAllIndices(lastModCount); + } + lastModCount = myLocalModCount; + } + }); + + } + } + + @Override + public void initComponent() { + } + + @Nullable + private static String calcConfigPath(final String path) { + try { + final String _path = FileUtil.toSystemIndependentName(new File(path).getCanonicalPath()); + return _path.endsWith("/")? _path : _path + "/" ; + } + catch (IOException e) { + LOG.info(e); + return null; + } + } + + /** + * @return true if registered index requires full rebuild for some reason, e.g. is just created or corrupted + * + * @param extension + * @param isCurrentVersionCorrupted + */ + private boolean registerIndexer(@NotNull final FileBasedIndexExtension extension, final boolean isCurrentVersionCorrupted) throws IOException { + final ID name = extension.getName(); + final int version = extension.getVersion(); + final File versionFile = IndexInfrastructure.getVersionFile(name); + final boolean versionFileExisted = versionFile.exists(); + boolean versionChanged = false; + if (isCurrentVersionCorrupted || IndexInfrastructure.versionDiffers(versionFile, version)) { + if (!isCurrentVersionCorrupted && versionFileExisted) { + versionChanged = true; + LOG.info("Version has changed for index " + name + ". The index will be rebuilt."); + } + FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); + IndexInfrastructure.rewriteVersion(versionFile, version); + } + + for (int attempt = 0; attempt < 2; attempt++) { + try { + final MapIndexStorage storage = new MapIndexStorage(IndexInfrastructure.getStorageFile(name), extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getCacheSize()); + final MemoryIndexStorage memStorage = new MemoryIndexStorage(storage); + final UpdatableIndex index = createIndex(name, extension, memStorage); + final InputFilter inputFilter = extension.getInputFilter(); + + assert inputFilter != null : "Index extension " + name + " must provide non-null input filter"; + + myIndices.put(name, new Pair, InputFilter>(index, new IndexableFilesFilter(inputFilter))); + myUnsavedDataIndexingSemaphores.put(name, new Semaphore()); + myIndexIdToVersionMap.put(name, version); + if (!extension.dependsOnFileContent()) { + myNotRequiringContentIndices.add(name); + } + else { + myRequiringContentIndices.add(name); + } + myNoLimitCheckTypes.addAll(extension.getFileTypesWithSizeLimitNotApplicable()); + break; + } + catch (IOException e) { + LOG.info(e); + FileUtil.delete(IndexInfrastructure.getIndexRootDir(name)); + IndexInfrastructure.rewriteVersion(versionFile, version); + } + } + return versionChanged; + } + + private static void saveRegisteredIndices(@NotNull Collection> ids) { + final File file = getRegisteredIndicesFile(); + try { + FileUtil.createIfDoesntExist(file); + final DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file))); + try { + os.writeInt(ids.size()); + for (ID id : ids) { + IOUtil.writeString(id.toString(), os); + } + } + finally { + os.close(); + } + } + catch (IOException ignored) { + } + } + + @NotNull + private static Set readRegisteredIndexNames() { + final Set result = new THashSet(); + try { + final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(getRegisteredIndicesFile()))); + try { + final int size = in.readInt(); + for (int idx = 0; idx < size; idx++) { + result.add(IOUtil.readString(in)); + } + } + finally { + in.close(); + } + } + catch (IOException ignored) { + } + return result; + } + + @NotNull + private static File getRegisteredIndicesFile() { + return new File(PathManager.getIndexRoot(), "registered"); + } + + @NotNull + private UpdatableIndex createIndex(@NotNull final ID indexId, @NotNull final FileBasedIndexExtension extension, @NotNull final MemoryIndexStorage storage) throws IOException { + final MapReduceIndex index; + if (extension instanceof CustomImplementationFileBasedIndexExtension) { + final UpdatableIndex custom = ((CustomImplementationFileBasedIndexExtension)extension).createIndexImplementation(indexId, this, storage); + + assert custom != null : "Custom index implementation must not be null; index: " + indexId; + + if (!(custom instanceof MapReduceIndex)) { + return custom; + } + index = (MapReduceIndex)custom; + } + else { + index = new MapReduceIndex(indexId, extension.getIndexer(), storage); + } + + final KeyDescriptor keyDescriptor = extension.getKeyDescriptor(); + index.setInputIdToDataKeysIndex(new Factory>>() { + @Override + public PersistentHashMap> create() { + try { + return createIdToDataKeysIndex(indexId, keyDescriptor, storage); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + + return index; + } + + @NotNull + private static PersistentHashMap> createIdToDataKeysIndex(@NotNull final ID indexId, + @NotNull final KeyDescriptor keyDescriptor, + @NotNull MemoryIndexStorage storage) throws IOException { + final File indexStorageFile = IndexInfrastructure.getInputIndexStorageFile(indexId); + final Ref isBufferingMode = new Ref(false); + final TIntObjectHashMap> tempMap = new TIntObjectHashMap>(); + + final DataExternalizer> dataExternalizer = new DataExternalizer>() { + @Override + public void save(DataOutput out, @NotNull Collection value) throws IOException { + try { + DataInputOutputUtil.writeINT(out, value.size()); + for (K key : value) { + keyDescriptor.save(out, key); + } + } + catch (IllegalArgumentException e) { + throw new IOException("Error saving data for index " + indexId, e); + } + } + + @NotNull + @Override + public Collection read(DataInput in) throws IOException { + try { + final int size = DataInputOutputUtil.readINT(in); + final List list = new ArrayList(size); + for (int idx = 0; idx < size; idx++) { + list.add(keyDescriptor.read(in)); + } + return list; + } + catch (IllegalArgumentException e) { + throw new IOException("Error reading data for index " + indexId, e); + } + } + }; + + // Important! Update IdToDataKeysIndex depending on the sate of "buffering" flag from the MemoryStorage. + // If buffering is on, all changes should be done in memory (similar to the way it is done in memory storage). + // Otherwise data in IdToDataKeysIndex will not be in sync with the 'main' data in the index on disk and index updates will be based on the + // wrong sets of keys for the given file. This will lead to unpredictable results in main index because it will not be + // cleared properly before updating (removed data will still be present on disk). See IDEA-52223 for illustration of possible effects. + + final PersistentHashMap> map = new PersistentHashMap>( + indexStorageFile, EnumeratorIntegerDescriptor.INSTANCE, dataExternalizer + ) { + + @Override + protected Collection doGet(Integer integer) throws IOException { + if (isBufferingMode.get()) { + final Collection collection = tempMap.get(integer); + if (collection != null) { + return collection; + } + } + return super.doGet(integer); + } + + @Override + protected void doPut(Integer integer, @Nullable Collection ks) throws IOException { + if (isBufferingMode.get()) { + tempMap.put(integer, ks == null? Collections.emptySet() : ks); + } + else { + super.doPut(integer, ks); + } + } + + @Override + protected void doRemove(Integer integer) throws IOException { + if (isBufferingMode.get()) { + tempMap.put(integer, Collections.emptySet()); + } + else { + super.doRemove(integer); + } + } + }; + + storage.addBufferingStateListsner(new MemoryIndexStorage.BufferingStateListener() { + @Override + public void bufferingStateChanged(boolean newState) { + synchronized (map) { + isBufferingMode.set(newState); + } + } + @Override + public void memoryStorageCleared() { + synchronized (map) { + tempMap.clear(); + } + } + }); + return map; + } + + @Override + public void disposeComponent() { + performShutdown(); + } + + private final AtomicBoolean myShutdownPerformed = new AtomicBoolean(false); + + private void performShutdown() { + if (!myShutdownPerformed.compareAndSet(false, true)) { + return; // already shut down + } + try { + if (myFlushingFuture != null) { + myFlushingFuture.cancel(false); + myFlushingFuture = null; + } + + myFileDocumentManager.saveAllDocuments(); + } + finally { + LOG.info("START INDEX SHUTDOWN"); + try { + myChangedFilesCollector.forceUpdate(null, null, null, true); + + for (ID indexId : myIndices.keySet()) { + final UpdatableIndex index = getIndex(indexId); + assert index != null; + checkRebuild(indexId, true); // if the index was scheduled for rebuild, only clean it + //LOG.info("DISPOSING " + indexId); + index.dispose(); + } + + myVfManager.removeVirtualFileListener(myChangedFilesCollector); + + //FileUtil.delete(getMarkerFile()); + } + catch (Throwable e) { + LOG.info("Problems during index shutdown", e); + throw new RuntimeException(e); + } + LOG.info("END INDEX SHUTDOWN"); + } + } + + private void flushAllIndices(final long modCount) { + if (HeavyProcessLatch.INSTANCE.isRunning()) { + return; + } + IndexingStamp.flushCache(); + for (ID indexId : new ArrayList>(myIndices.keySet())) { + if (HeavyProcessLatch.INSTANCE.isRunning() || modCount != myLocalModCount) { + return; // do not interfere with 'main' jobs + } + try { + final UpdatableIndex index = getIndex(indexId); + if (index != null) { + index.flush(); + } + } + catch (StorageException e) { + LOG.info(e); + requestRebuild(indexId); + } + } + + if (!HeavyProcessLatch.INSTANCE.isRunning() && modCount == myLocalModCount) { // do not interfere with 'main' jobs + SerializationManager.getInstance().flushNameStorage(); + } + } + + /** + * @param project it is guaranteed to return data which is up-to-date withing the project + * Keys obtained from the files which do not belong to the project specified may not be up-to-date or even exist + */ + @Override + @NotNull + public Collection getAllKeys(@NotNull final ID indexId, @NotNull Project project) { + Set allKeys = new THashSet(); + processAllKeys(indexId, new CommonProcessors.CollectProcessor(allKeys), project); + return allKeys; + } + + /** + * @param project it is guaranteed to return data which is up-to-date withing the project + * Keys obtained from the files which do not belong to the project specified may not be up-to-date or even exist + */ + @Override + public boolean processAllKeys(@NotNull final ID indexId, Processor processor, @Nullable Project project) { + try { + final UpdatableIndex index = getIndex(indexId); + if (index == null) { + return true; + } + ensureUpToDate(indexId, project, project != null? GlobalSearchScope.allScope(project) : new EverythingGlobalScope()); + return index.processAllKeys(processor); + } + catch (StorageException e) { + scheduleRebuild(indexId, e); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof StorageException || cause instanceof IOException) { + scheduleRebuild(indexId, cause); + } + else { + throw e; + } + } + + return false; + } + + private static final ThreadLocal myUpToDateCheckState = new ThreadLocal(); + + public static void disableUpToDateCheckForCurrentThread() { + final Integer currentValue = myUpToDateCheckState.get(); + myUpToDateCheckState.set(currentValue == null? 1 : currentValue.intValue() + 1); + } + + public static void enableUpToDateCheckForCurrentThread() { + final Integer currentValue = myUpToDateCheckState.get(); + if (currentValue != null) { + final int newValue = currentValue.intValue() - 1; + if (newValue != 0) { + myUpToDateCheckState.set(newValue); + } + else { + myUpToDateCheckState.remove(); + } + } + } + + private static boolean isUpToDateCheckEnabled() { + final Integer value = myUpToDateCheckState.get(); + return value == null || value.intValue() == 0; + } + + + private final ThreadLocal myReentrancyGuard = new ThreadLocal() { + @Override + protected Boolean initialValue() { + return Boolean.FALSE; + } + }; + + /** + * DO NOT CALL DIRECTLY IN CLIENT CODE + * The method is internal to indexing engine end is called internally. The method is public due to implementation details + */ + @Override + public void ensureUpToDate(@NotNull final ID indexId, @Nullable Project project, @Nullable GlobalSearchScope filter) { + ensureUpToDate(indexId, project, filter, null); + } + + @Override + protected void ensureUpToDate(@NotNull final ID indexId, + @Nullable Project project, + @Nullable GlobalSearchScope filter, + @Nullable VirtualFile restrictedFile) { + if (!needsFileContentLoading(indexId)) { + return; //indexed eagerly in foreground while building unindexed file list + } + if (isDumb(project)) { + handleDumbMode(project); + } + + if (myReentrancyGuard.get().booleanValue()) { + //assert false : "ensureUpToDate() is not reentrant!"; + return; + } + myReentrancyGuard.set(Boolean.TRUE); + + try { + myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); + if (isUpToDateCheckEnabled()) { + try { + checkRebuild(indexId, false); + myChangedFilesCollector.forceUpdate(project, filter, restrictedFile, false); + indexUnsavedDocuments(indexId, project, filter, restrictedFile); + } + catch (StorageException e) { + scheduleRebuild(indexId, e); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof StorageException || cause instanceof IOException) { + scheduleRebuild(indexId, e); + } + else { + throw e; + } + } + } + } + finally { + myReentrancyGuard.set(Boolean.FALSE); + } + } + + private static void handleDumbMode(@Nullable Project project) { + ProgressManager.checkCanceled(); // DumbModeAction.CANCEL + + if (project != null) { + final ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator(); + if (progressIndicator instanceof BackgroundableProcessIndicator) { + final BackgroundableProcessIndicator indicator = (BackgroundableProcessIndicator)progressIndicator; + if (indicator.getDumbModeAction() == DumbModeAction.WAIT) { + assert !ApplicationManager.getApplication().isDispatchThread(); + DumbService.getInstance(project).waitForSmartMode(); + return; + } + } + } + + throw new IndexNotReadyException(); + } + + private static boolean isDumb(@Nullable Project project) { + if (project != null) { + return DumbServiceImpl.getInstance(project).isDumb(); + } + for (Project proj : ProjectManager.getInstance().getOpenProjects()) { + if (DumbServiceImpl.getInstance(proj).isDumb()) { + return true; + } + } + return false; + } + + @Override + @NotNull + public List getValues(@NotNull final ID indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) { + final List values = new SmartList(); + processValuesImpl(indexId, dataKey, true, null, new ValueProcessor() { + @Override + public boolean process(final VirtualFile file, final V value) { + values.add(value); + return true; + } + }, filter); + return values; + } + + @Override + @NotNull + public Collection getContainingFiles(@NotNull final ID indexId, @NotNull K dataKey, @NotNull final GlobalSearchScope filter) { + final Set files = new THashSet(); + processValuesImpl(indexId, dataKey, false, null, new ValueProcessor() { + @Override + public boolean process(final VirtualFile file, final V value) { + files.add(file); + return true; + } + }, filter); + return files; + } + + + /** + * @return false if ValueProcessor.process() returned false; true otherwise or if ValueProcessor was not called at all + */ + @Override + public boolean processValues(@NotNull final ID indexId, @NotNull final K dataKey, @Nullable final VirtualFile inFile, + @NotNull ValueProcessor processor, @NotNull final GlobalSearchScope filter) { + return processValuesImpl(indexId, dataKey, false, inFile, processor, filter); + } + + + + + @Nullable + private R processExceptions(@NotNull final ID indexId, + @Nullable final VirtualFile restrictToFile, + @NotNull final GlobalSearchScope filter, + @NotNull ThrowableConvertor, R, StorageException> computable) { + try { + final UpdatableIndex index = getIndex(indexId); + if (index == null) { + return null; + } + final Project project = filter.getProject(); + //assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries"; + ensureUpToDate(indexId, project, filter, restrictToFile); + + try { + index.getReadLock().lock(); + return computable.convert(index); + } + finally { + index.getReadLock().unlock(); + } + } + catch (StorageException e) { + scheduleRebuild(indexId, e); + } + catch (RuntimeException e) { + final Throwable cause = getCauseToRebuildIndex(e); + if (cause != null) { + scheduleRebuild(indexId, cause); + } + else { + throw e; + } + } + return null; + } + + private boolean processValuesImpl(@NotNull final ID indexId, final K dataKey, final boolean ensureValueProcessedOnce, + @Nullable final VirtualFile restrictToFile, @NotNull final ValueProcessor processor, + @NotNull final GlobalSearchScope filter) { + ThrowableConvertor, Boolean, StorageException> keyProcessor = new ThrowableConvertor, Boolean, StorageException>() { + @Override + public Boolean convert(@NotNull UpdatableIndex index) throws StorageException { + final ValueContainer container = index.getData(dataKey); + + boolean shouldContinue = true; + + if (restrictToFile != null) { + if (restrictToFile instanceof VirtualFileWithId) { + final int restrictedFileId = getFileId(restrictToFile); + for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext(); ) { + final V value = valueIt.next(); + if (container.isAssociated(value, restrictedFileId)) { + shouldContinue = processor.process(restrictToFile, value); + if (!shouldContinue) { + break; + } + } + } + } + } + else { + final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); + ProjectIndexableFilesFilter projectFilesSet = projectIndexableFiles(filter.getProject()); + VALUES_LOOP: for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext();) { + final V value = valueIt.next(); + for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) { + final int id = inputIdsIterator.next(); + if (projectFilesSet != null && !projectFilesSet.contains(id)) continue; + VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); + if (file != null && filter.accept(file)) { + shouldContinue = processor.process(file, value); + if (!shouldContinue) { + break VALUES_LOOP; + } + if (ensureValueProcessedOnce) { + break; // continue with the next value + } + } + } + } + } + return shouldContinue; + } + }; + final Boolean result = processExceptions(indexId, restrictToFile, filter, keyProcessor); + return result == null || result.booleanValue(); + } + + @Override + public boolean processFilesContainingAllKeys(@NotNull final ID indexId, + @NotNull final Collection dataKeys, + @NotNull final GlobalSearchScope filter, + @Nullable Condition valueChecker, + @NotNull final Processor processor) { + ProjectIndexableFilesFilter filesSet = projectIndexableFiles(filter.getProject()); + final TIntHashSet set = collectFileIdsContainingAllKeys(indexId, dataKeys, filter, valueChecker, filesSet); + return set != null && processVirtualFiles(set, filter, processor); + } + + private static final Key> ourProjectFilesSetKey = Key.create("projectFiles"); + + public static final class ProjectIndexableFilesFilter { + private static final int SHIFT = 6; + private static final int MASK = (1 << SHIFT) - 1; + private final long[] myBitMask; + private final int myModificationCount; + private final int myMinId; + private final int myMaxId; + + private ProjectIndexableFilesFilter(@NotNull TIntHashSet set, int modificationCount) { + myModificationCount = modificationCount; + final int[] minMax = new int[2]; + if (set.size() > 0) { + minMax[0] = minMax[1] = set.iterator().next(); + } + set.forEach(new TIntProcedure() { + @Override + public boolean execute(int value) { + minMax[0] = Math.min(minMax[0], value); + minMax[1] = Math.max(minMax[1], value); + return true; + } + }); + myMaxId = minMax[1]; + myMinId = minMax[0]; + myBitMask = new long[((myMaxId - myMinId) >> SHIFT) + 1]; + set.forEach(new TIntProcedure() { + @Override + public boolean execute(int value) { + value = value - myMinId; + myBitMask[value >> SHIFT] |= (1L << (value & MASK)); + return true; + } + }); + } + + public boolean contains(int id) { + if (id < myMinId) return false; + if (id > myMaxId) return false; + id -= myMinId; + return (myBitMask[id >> SHIFT] & (1L << (id & MASK))) != 0; + } + } + + @Nullable + public ProjectIndexableFilesFilter projectIndexableFiles(@Nullable Project project) { + if (project == null) return null; + + SoftReference reference = project.getUserData(ourProjectFilesSetKey); + ProjectIndexableFilesFilter data = reference != null ? reference.get() : null; + if (data != null && data.myModificationCount == myFilesModCount) return data; + + final TIntHashSet filesSet = new TIntHashSet(); + iterateIndexableFiles(new ContentIterator() { + @Override + public boolean processFile(@NotNull VirtualFile fileOrDir) { + filesSet.add(((VirtualFileWithId)fileOrDir).getId()); + return true; + } + }, project, ProgressManager.getInstance().getProgressIndicator()); + ProjectIndexableFilesFilter files = new ProjectIndexableFilesFilter(filesSet, myFilesModCount); + project.putUserData(ourProjectFilesSetKey, new SoftReference(files)); + return files; + } + + @Nullable + private TIntHashSet collectFileIdsContainingAllKeys(@NotNull final ID indexId, + @NotNull final Collection dataKeys, + @NotNull final GlobalSearchScope filter, + @Nullable final Condition valueChecker, + @Nullable final ProjectIndexableFilesFilter projectFilesFilter) { + final ThrowableConvertor, TIntHashSet, StorageException> convertor = + new ThrowableConvertor, TIntHashSet, StorageException>() { + @Nullable + @Override + public TIntHashSet convert(@NotNull UpdatableIndex index) throws StorageException { + TIntHashSet mainIntersection = null; + + for (K dataKey : dataKeys) { + ProgressManager.checkCanceled(); + final TIntHashSet copy = new TIntHashSet(); + final ValueContainer container = index.getData(dataKey); + + for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext(); ) { + final V value = valueIt.next(); + if (valueChecker != null && !valueChecker.value(value)) { + continue; + } + + ValueContainer.IntIterator iterator = container.getInputIdsIterator(value); + + if (mainIntersection == null || iterator.size() < mainIntersection.size()) { + for (final ValueContainer.IntIterator inputIdsIterator = iterator; inputIdsIterator.hasNext(); ) { + final int id = inputIdsIterator.next(); + if (mainIntersection == null && (projectFilesFilter == null || projectFilesFilter.contains(id)) || + mainIntersection != null && mainIntersection.contains(id) + ) { + copy.add(id); + } + } + } else { + mainIntersection.forEach(new TIntProcedure() { + final ValueContainer.IntPredicate predicate = container.getValueAssociationPredicate(value); + @Override + public boolean execute(int id) { + if (predicate.contains(id)) copy.add(id); + return true; + } + }); + } + } + + mainIntersection = copy; + if (mainIntersection.isEmpty()) { + return new TIntHashSet(); + } + } + + return mainIntersection; + } + }; + + + return processExceptions(indexId, null, filter, convertor); + } + + private static boolean processVirtualFiles(@NotNull TIntHashSet ids, + @NotNull final GlobalSearchScope filter, + @NotNull final Processor processor) { + final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); + return ids.forEach(new TIntProcedure() { + @Override + public boolean execute(int id) { + ProgressManager.checkCanceled(); + VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); + if (file != null && filter.accept(file)) { + return processor.process(file); + } + return true; + } + }); + } + + @Nullable + public static Throwable getCauseToRebuildIndex(@NotNull RuntimeException e) { + Throwable cause = e.getCause(); + if (cause instanceof StorageException || cause instanceof IOException || + cause instanceof IllegalArgumentException) return cause; + return null; + } + + @Override + public boolean getFilesWithKey(@NotNull final ID indexId, + @NotNull final Set dataKeys, + @NotNull Processor processor, + @NotNull GlobalSearchScope filter) { + try { + final UpdatableIndex index = getIndex(indexId); + if (index == null) { + return true; + } + final Project project = filter.getProject(); + //assert project != null : "GlobalSearchScope#getProject() should be not-null for all index queries"; + ensureUpToDate(indexId, project, filter); + + try { + index.getReadLock().lock(); + final List locals = new ArrayList(); + for (K dataKey : dataKeys) { + TIntHashSet local = new TIntHashSet(); + locals.add(local); + final ValueContainer container = index.getData(dataKey); + + for (final Iterator valueIt = container.getValueIterator(); valueIt.hasNext();) { + final V value = valueIt.next(); + for (final ValueContainer.IntIterator inputIdsIterator = container.getInputIdsIterator(value); inputIdsIterator.hasNext();) { + final int id = inputIdsIterator.next(); + local.add(id); + } + } + } + + if (locals.isEmpty()) { + return true; + } + + Collections.sort(locals, new Comparator() { + @Override + public int compare(TIntHashSet o1, TIntHashSet o2) { + return o1.size() - o2.size(); + } + }); + + final PersistentFS fs = (PersistentFS)ManagingFS.getInstance(); + TIntIterator ids = join(locals).iterator(); + ProjectIndexableFilesFilter projectIndexableFilesFilter = projectIndexableFiles(project); + while (ids.hasNext()) { + int id = ids.next(); + if (projectIndexableFilesFilter != null && !projectIndexableFilesFilter.contains(id)) continue; + //VirtualFile file = IndexInfrastructure.findFileById(fs, id); + VirtualFile file = IndexInfrastructure.findFileByIdIfCached(fs, id); + if (file != null && filter.accept(file)) { + if (!processor.process(file)) { + return false; + } + } + } + } + finally { + index.getReadLock().unlock(); + } + } + catch (StorageException e) { + scheduleRebuild(indexId, e); + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof StorageException || cause instanceof IOException) { + scheduleRebuild(indexId, cause); + } + else { + throw e; + } + } + return true; + } + + @NotNull + private static TIntHashSet join(@NotNull List locals) { + TIntHashSet result = locals.get(0); + if (locals.size() > 1) { + TIntIterator it = result.iterator(); + + while (it.hasNext()) { + int id = it.next(); + for (int i = 1; i < locals.size(); i++) { + if (!locals.get(i).contains(id)) { + it.remove(); + break; + } + } + } + } + return result; + } + + @Override + public void scheduleRebuild(@NotNull final ID indexId, @NotNull final Throwable e) { + LOG.info(e); + requestRebuild(indexId); + try { + checkRebuild(indexId, false); + } + catch (ProcessCanceledException ignored) { + } + } + + private void checkRebuild(@NotNull final ID indexId, final boolean cleanupOnly) { + final AtomicInteger status = ourRebuildStatus.get(indexId); + if (status.get() == OK) return; + if (status.compareAndSet(REQUIRES_REBUILD, REBUILD_IN_PROGRESS)) { + cleanupProcessedFlag(); + + final Runnable rebuildRunnable = new Runnable() { + @Override + public void run() { + try { + clearIndex(indexId); + if (!cleanupOnly) { + scheduleIndexRebuild(false); + } + } + catch (StorageException e) { + requestRebuild(indexId); + LOG.info(e); + } + finally { + status.compareAndSet(REBUILD_IN_PROGRESS, OK); + } + } + }; + + if (cleanupOnly || myIsUnitTestMode) { + rebuildRunnable.run(); + } + else { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + new Task.Modal(null, "Updating index", false) { + @Override + public void run(@NotNull final ProgressIndicator indicator) { + indicator.setIndeterminate(true); + rebuildRunnable.run(); + } + }.queue(); + } + }); + } + } + + if (status.get() == REBUILD_IN_PROGRESS) { + throw new ProcessCanceledException(); + } + } + + private void scheduleIndexRebuild(boolean forceDumbMode) { + for (Project project : ProjectManager.getInstance().getOpenProjects()) { + final Set updatersToRun = Collections.singleton(new UnindexedFilesUpdater(project, this)); + final DumbServiceImpl service = DumbServiceImpl.getInstance(project); + if (forceDumbMode) { + service.queueCacheUpdateInDumbMode(updatersToRun); + } + else { + service.queueCacheUpdate(updatersToRun); + } + } + } + + private void clearIndex(@NotNull final ID indexId) throws StorageException { + final UpdatableIndex index = getIndex(indexId); + assert index != null: "Index with key " + indexId + " not found or not registered properly"; + index.clear(); + try { + IndexInfrastructure.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myIndexIdToVersionMap.get(indexId)); + } + catch (IOException e) { + LOG.error(e); + } + } + + @NotNull + private Set getUnsavedOrTransactedDocuments() { + final Set docs = new THashSet(Arrays.asList(myFileDocumentManager.getUnsavedDocuments())); + synchronized (myTransactionMap) { + docs.addAll(myTransactionMap.keySet()); + } + return docs; + } + + private void indexUnsavedDocuments(@NotNull ID indexId, + @Nullable Project project, + GlobalSearchScope filter, + VirtualFile restrictedFile) throws StorageException { + if (myUpToDateIndices.contains(indexId)) { + return; // no need to index unsaved docs + } + + final Set documents = getUnsavedOrTransactedDocuments(); + if (!documents.isEmpty()) { + // now index unsaved data + final StorageGuard.Holder guard = setDataBufferingEnabled(true); + try { + final Semaphore semaphore = myUnsavedDataIndexingSemaphores.get(indexId); + + assert semaphore != null : "Semaphore for unsaved data indexing was not initialized for index " + indexId; + + semaphore.down(); + boolean allDocsProcessed = true; + try { + for (Document document : documents) { + allDocsProcessed &= indexUnsavedDocument(document, indexId, project, filter, restrictedFile); + } + } + finally { + semaphore.up(); + + while (!semaphore.waitFor(500)) { // may need to wait until another thread is done with indexing + if (Thread.holdsLock(PsiLock.LOCK)) { + break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding. + } + } + if (allDocsProcessed && !hasActiveTransactions()) { + myUpToDateIndices.add(indexId); // safe to set the flag here, because it will be cleared under the WriteAction + } + } + } + finally { + guard.leave(); + } + } + } + + private boolean hasActiveTransactions() { + synchronized (myTransactionMap) { + return !myTransactionMap.isEmpty(); + } + } + + private interface DocumentContent { + String getText(); + long getModificationStamp(); + } + + private static class AuthenticContent implements DocumentContent { + private final Document myDocument; + + private AuthenticContent(final Document document) { + myDocument = document; + } + + @Override + public String getText() { + return myDocument.getText(); + } + + @Override + public long getModificationStamp() { + return myDocument.getModificationStamp(); + } + } + + private static class PsiContent implements DocumentContent { + private final Document myDocument; + private final PsiFile myFile; + + private PsiContent(final Document document, final PsiFile file) { + myDocument = document; + myFile = file; + } + + @Override + public String getText() { + if (myFile.getModificationStamp() != myDocument.getModificationStamp()) { + final ASTNode node = myFile.getNode(); + assert node != null; + return node.getText(); + } + return myDocument.getText(); + } + + @Override + public long getModificationStamp() { + return myFile.getModificationStamp(); + } + } + +// returns false if doc was not indexed because the file does not fit in scope + private boolean indexUnsavedDocument(@NotNull final Document document, @NotNull final ID requestedIndexId, final Project project, + @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedFile) throws StorageException { + final VirtualFile vFile = myFileDocumentManager.getFile(document); + if (!(vFile instanceof VirtualFileWithId) || !vFile.isValid()) { + return true; + } + + if (restrictedFile != null) { + if(vFile != restrictedFile) { + return false; + } + } + else if (filter != null && !filter.accept(vFile)) { + return false; + } + + final PsiFile dominantContentFile = findDominantPsiForDocument(document, project); + + final DocumentContent content; + if (dominantContentFile != null && dominantContentFile.getModificationStamp() != document.getModificationStamp()) { + content = new PsiContent(document, dominantContentFile); + } + else { + content = new AuthenticContent(document); + } + + final long currentDocStamp = content.getModificationStamp(); + if (currentDocStamp != myLastIndexedDocStamps.getAndSet(document, requestedIndexId, currentDocStamp)) { + final Ref exRef = new Ref(null); + ProgressManager.getInstance().executeNonCancelableSection(new Runnable() { + @Override + public void run() { + try { + final String contentText = content.getText(); + if (isTooLarge(vFile, contentText.length())) { + return; + } + + final FileContentImpl newFc = new FileContentImpl(vFile, contentText, vFile.getCharset()); + + if (dominantContentFile != null) { + dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, true); + newFc.putUserData(IndexingDataKeys.PSI_FILE, dominantContentFile); + } + + if (content instanceof AuthenticContent) { + newFc.putUserData(EDITOR_HIGHLIGHTER, EditorHighlighterCache.getEditorHighlighterForCachesBuilding(document)); + } + + if (getInputFilter(requestedIndexId).acceptInput(vFile)) { + newFc.putUserData(IndexingDataKeys.PROJECT, project); + final int inputId = Math.abs(getFileId(vFile)); + getIndex(requestedIndexId).update(inputId, newFc); + } + + if (dominantContentFile != null) { + dominantContentFile.putUserData(PsiFileImpl.BUILDING_STUB, null); + } + } + catch (StorageException e) { + exRef.set(e); + } + } + }); + final StorageException storageException = exRef.get(); + if (storageException != null) { + throw storageException; + } + } + return true; + } + + public static final Key EDITOR_HIGHLIGHTER = new Key("Editor"); + + @Nullable + private PsiFile findDominantPsiForDocument(@NotNull Document document, @Nullable Project project) { + synchronized (myTransactionMap) { + PsiFile psiFile = myTransactionMap.get(document); + if (psiFile != null) return psiFile; + } + + return project == null ? null : findLatestKnownPsiForUncomittedDocument(document, project); + } + + private final StorageGuard myStorageLock = new StorageGuard(); + + @NotNull + private StorageGuard.Holder setDataBufferingEnabled(final boolean enabled) { + final StorageGuard.Holder holder = myStorageLock.enter(enabled); + for (ID indexId : myIndices.keySet()) { + final MapReduceIndex index = (MapReduceIndex)getIndex(indexId); + assert index != null; + final IndexStorage indexStorage = index.getStorage(); + ((MemoryIndexStorage)indexStorage).setBufferingEnabled(enabled); + } + return holder; + } + + private void cleanupMemoryStorage() { + myLastIndexedDocStamps.clear(); + for (ID indexId : myIndices.keySet()) { + final MapReduceIndex index = (MapReduceIndex)getIndex(indexId); + assert index != null; + final MemoryIndexStorage memStorage = (MemoryIndexStorage)index.getStorage(); + index.getWriteLock().lock(); + try { + memStorage.clearMemoryMap(); + } + finally { + index.getWriteLock().unlock(); + } + memStorage.fireMemoryStorageCleared(); + } + } + + private void dropUnregisteredIndices() { + final Set indicesToDrop = readRegisteredIndexNames(); + for (ID key : myIndices.keySet()) { + indicesToDrop.remove(key.toString()); + } + for (String s : indicesToDrop) { + FileUtil.delete(IndexInfrastructure.getIndexRootDir(ID.create(s))); + } + } + + @Override + public void requestRebuild(ID indexId, Throwable throwable) { + cleanupProcessedFlag(); + LOG.info("Rebuild requested for index " + indexId, throwable); + ourRebuildStatus.get(indexId).set(REQUIRES_REBUILD); + } + + private UpdatableIndex getIndex(ID indexId) { + final Pair, InputFilter> pair = myIndices.get(indexId); + + assert pair != null : "Index data is absent for index " + indexId; + + //noinspection unchecked + return (UpdatableIndex)pair.getFirst(); + } + + private InputFilter getInputFilter(ID indexId) { + final Pair, InputFilter> pair = myIndices.get(indexId); + + assert pair != null : "Index data is absent for index " + indexId; + + return pair.getSecond(); + } + + public int getNumberOfPendingInvalidations() { + return myChangedFilesCollector.getNumberOfPendingInvalidations(); + } + + @NotNull + public Collection getFilesToUpdate(final Project project) { + return ContainerUtil.findAll(myChangedFilesCollector.getAllFilesToUpdate(), new Condition() { + @Override + public boolean value(VirtualFile virtualFile) { + for (IndexableFileSet set : myIndexableSets) { + final Project proj = myIndexableSetToProjectMap.get(set); + if (proj != null && !proj.equals(project)) { + continue; // skip this set as associated with a different project + } + if (set.isInSet(virtualFile)) { + return true; + } + } + return false; + } + }); + } + + public void processRefreshedFile(@NotNull Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent) { + myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); + myChangedFilesCollector.processFileImpl(project, fileContent, false); + } + + public void indexFileContent(@Nullable Project project, @NotNull com.intellij.ide.caches.FileContent content) { + myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); + final VirtualFile file = content.getVirtualFile(); + FileContentImpl fc = null; + + PsiFile psiFile = null; + + FileTypeManagerImpl.cacheFileType(file, file.getFileType()); + try { + for (final ID indexId : myIndices.keySet()) { + if (shouldIndexFile(file, indexId)) { + if (fc == null) { + byte[] currentBytes; + try { + currentBytes = content.getBytes(); + } + catch (IOException e) { + currentBytes = ArrayUtil.EMPTY_BYTE_ARRAY; + } + fc = new FileContentImpl(file, currentBytes); + + psiFile = content.getUserData(IndexingDataKeys.PSI_FILE); + if (psiFile != null) { + psiFile.putUserData(PsiFileImpl.BUILDING_STUB, true); + fc.putUserData(IndexingDataKeys.PSI_FILE, psiFile); + } + if (project == null) { + project = ProjectUtil.guessProjectForFile(file); + } + fc.putUserData(IndexingDataKeys.PROJECT, project); + } + + try { + ProgressManager.checkCanceled(); + updateSingleIndex(indexId, file, fc); + } + catch (ProcessCanceledException e) { + myChangedFilesCollector.scheduleForUpdate(file); + throw e; + } + catch (StorageException e) { + requestRebuild(indexId); + LOG.info(e); + } + } + } + + if (psiFile != null) { + psiFile.putUserData(PsiFileImpl.BUILDING_STUB, null); + } + } finally { + FileTypeManagerImpl.cacheFileType(file, null); + } + } + + private void updateSingleIndex(final ID indexId, @NotNull final VirtualFile file, @Nullable final FileContent currentFC) + throws StorageException { + if (ourRebuildStatus.get(indexId).get() == REQUIRES_REBUILD) { + return; // the index is scheduled for rebuild, no need to update + } + myLocalModCount++; + + final StorageGuard.Holder lock = setDataBufferingEnabled(false); + + try { + final int inputId = Math.abs(getFileId(file)); + + final UpdatableIndex index = getIndex(indexId); + assert index != null; + + final Ref exRef = new Ref(null); + ProgressManager.getInstance().executeNonCancelableSection(new Runnable() { + @Override + public void run() { + try { + index.update(inputId, currentFC); + } + catch (StorageException e) { + exRef.set(e); + } + } + }); + final StorageException storageException = exRef.get(); + if (storageException != null) { + throw storageException; + } + ApplicationManager.getApplication().runReadAction(new Runnable() { + @Override + public void run() { + if (file.isValid()) { + if (currentFC != null) { + IndexingStamp.update(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId)); + } + else { + // mark the file as unindexed + IndexingStamp.update(file, indexId, -1L); + } + } + } + }); + } + finally { + lock.leave(); + } + } + + private boolean needsFileContentLoading(ID indexId) { + return !myNotRequiringContentIndices.contains(indexId); + } + + private abstract static class InvalidationTask implements Runnable { + private final VirtualFile mySubj; + + protected InvalidationTask(final VirtualFile subj) { + mySubj = subj; + } + + public VirtualFile getSubj() { + return mySubj; + } + } + + private final class ChangedFilesCollector extends VirtualFileAdapter { + private final Set myFilesToUpdate = new ConcurrentHashSet(); + private final Queue myFutureInvalidations = new ConcurrentLinkedQueue(); + + private final ManagingFS myManagingFS = ManagingFS.getInstance(); + // No need to react on movement events since files stay valid, their ids don't change and all associated attributes remain intact. + + @Override + public void fileCreated(@NotNull final VirtualFileEvent event) { + markDirty(event, false); + } + + @Override + public void fileDeleted(@NotNull final VirtualFileEvent event) { + myFilesToUpdate.remove(event.getFile()); // no need to update it anymore + } + + @Override + public void fileCopied(@NotNull final VirtualFileCopyEvent event) { + markDirty(event, false); + } + + @Override + public void beforeFileDeletion(@NotNull final VirtualFileEvent event) { + invalidateIndices(event.getFile(), false); + } + + @Override + public void beforeContentsChange(@NotNull final VirtualFileEvent event) { + invalidateIndices(event.getFile(), true); + } + + @Override + public void contentsChanged(@NotNull final VirtualFileEvent event) { + markDirty(event, true); + } + + @Override + public void beforePropertyChange(@NotNull final VirtualFilePropertyEvent event) { + if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) { + // indexes may depend on file name + final VirtualFile file = event.getFile(); + if (!file.isDirectory()) { + // name change may lead to filetype change so the file might become not indexable + // in general case have to 'unindex' the file and index it again if needed after the name has been changed + invalidateIndices(file, false); + } + } + } + + @Override + public void propertyChanged(@NotNull final VirtualFilePropertyEvent event) { + if (event.getPropertyName().equals(VirtualFile.PROP_NAME)) { + // indexes may depend on file name + if (!event.getFile().isDirectory()) { + markDirty(event, false); + } + } + } + + private void markDirty(@NotNull final VirtualFileEvent event, final boolean contentChange) { + final VirtualFile eventFile = event.getFile(); + cleanProcessedFlag(eventFile); + iterateIndexableFiles(eventFile, new Processor() { + @Override + public boolean process(@NotNull final VirtualFile file) { + if (!contentChange) ++myFilesModCount; + FileContent fileContent = null; + // handle 'content-less' indices separately + for (ID indexId : myNotRequiringContentIndices) { + if (getInputFilter(indexId).acceptInput(file)) { + try { + if (fileContent == null) { + fileContent = new FileContentImpl(file); + } + updateSingleIndex(indexId, file, fileContent); + } + catch (StorageException e) { + LOG.info(e); + requestRebuild(indexId); + } + } + } + // For 'normal indices' schedule the file for update and stop iteration if at least one index accepts it + if (!isTooLarge(file)) { + for (ID indexId : myIndices.keySet()) { + if (needsFileContentLoading(indexId) && getInputFilter(indexId).acceptInput(file)) { + scheduleForUpdate(file); + break; // no need to iterate further, as the file is already marked + } + } + } + + return true; + } + }); + IndexingStamp.flushCache(); + } + + public void scheduleForUpdate(VirtualFile file) { + myFilesToUpdate.add(file); + } + + void invalidateIndices(@NotNull final VirtualFile file, final boolean markForReindex) { + if (isUnderConfigOrSystem(file)) { + return; + } + if (file.isDirectory()) { + if (isMock(file) || myManagingFS.wereChildrenAccessed(file)) { + final Iterable children = file instanceof NewVirtualFile + ? ((NewVirtualFile)file).iterInDbChildren() : Arrays.asList(file.getChildren()); + for (VirtualFile child : children) { + invalidateIndices(child, markForReindex); + } + } + } + else { + cleanProcessedFlag(file); + IndexingStamp.flushCache(); + final List> affectedIndices = new ArrayList>(myIndices.size()); + + for (final ID indexId : myIndices.keySet()) { + try { + if (!needsFileContentLoading(indexId)) { + if (shouldUpdateIndex(file, indexId)) { + updateSingleIndex(indexId, file, null); + } + } + else { // the index requires file content + if (shouldUpdateIndex(file, indexId)) { + affectedIndices.add(indexId); + } + } + } + catch (StorageException e) { + LOG.info(e); + requestRebuild(indexId); + } + } + + if (!affectedIndices.isEmpty()) { + if (markForReindex && !isTooLarge(file)) { + // only mark the file as unindexed, reindex will be done lazily + ApplicationManager.getApplication().runReadAction(new Runnable() { + @Override + public void run() { + for (ID indexId : affectedIndices) { + IndexingStamp.update(file, indexId, -2L); + } + } + }); + // the file is for sure not a dir and it was previously indexed by at least one index + scheduleForUpdate(file); + } + else { + myFutureInvalidations.offer(new InvalidationTask(file) { + @Override + public void run() { + removeFileDataFromIndices(affectedIndices, file); + } + }); + } + } + if (!markForReindex) { + final boolean removedFromUpdateQueue = myFilesToUpdate.remove(file);// no need to update it anymore + if (removedFromUpdateQueue && affectedIndices.isEmpty()) { + // Currently the file is about to be deleted and previously it was scheduled for update and not processed up to now. + // Because the file was scheduled for update, at the moment of scheduling it was marked as unindexed, + // so, to be on the safe side, we have to schedule data invalidation from all content-requiring indices for this file + myFutureInvalidations.offer(new InvalidationTask(file) { + @Override + public void run() { + removeFileDataFromIndices(myRequiringContentIndices, file); + } + }); + } + } + + IndexingStamp.flushCache(); + } + } + + private void removeFileDataFromIndices(@NotNull Collection> affectedIndices, @NotNull VirtualFile file) { + Throwable unexpectedError = null; + for (ID indexId : affectedIndices) { + try { + updateSingleIndex(indexId, file, null); + } + catch (StorageException e) { + LOG.info(e); + requestRebuild(indexId); + } + catch (ProcessCanceledException ignored) { + } + catch (Throwable e) { + LOG.info(e); + if (unexpectedError == null) { + unexpectedError = e; + } + } + } + IndexingStamp.flushCache(); + if (unexpectedError != null) { + LOG.error(unexpectedError); + } + } + + public int getNumberOfPendingInvalidations() { + return myFutureInvalidations.size(); + } + + public void ensureAllInvalidateTasksCompleted() { + final int size = getNumberOfPendingInvalidations(); + if (size == 0) { + return; + } + final ProgressIndicator current = ProgressManager.getInstance().getProgressIndicator(); + final ProgressIndicator indicator = current != null ? current : new EmptyProgressIndicator(); + indicator.setText(""); + int count = 0; + while (true) { + InvalidationTask task = myFutureInvalidations.poll(); + + if (task == null) { + break; + } + indicator.setFraction((double)count++ /size); + indicator.setText2(task.getSubj().getPresentableUrl()); + task.run(); + } + } + + private void iterateIndexableFiles(@NotNull final VirtualFile file, @NotNull final Processor processor) { + if (file.isDirectory()) { + final ContentIterator iterator = new ContentIterator() { + @Override + public boolean processFile(@NotNull final VirtualFile fileOrDir) { + if (!fileOrDir.isDirectory()) { + processor.process(fileOrDir); + } + return true; + } + }; + + for (IndexableFileSet set : myIndexableSets) { + if (set.isInSet(file)) { + set.iterateIndexableFilesIn(file, iterator); + } + } + } + else { + for (IndexableFileSet set : myIndexableSets) { + if (set.isInSet(file)) { + processor.process(file); + break; + } + } + } + } + + public Collection getAllFilesToUpdate() { + if (myFilesToUpdate.isEmpty()) { + return Collections.emptyList(); + } + return new ArrayList(myFilesToUpdate); + } + + private final Semaphore myForceUpdateSemaphore = new Semaphore(); + + private void forceUpdate(@Nullable Project project, @Nullable GlobalSearchScope filter, @Nullable VirtualFile restrictedTo, + boolean onlyRemoveOutdatedData) { + myChangedFilesCollector.ensureAllInvalidateTasksCompleted(); + for (VirtualFile file: getAllFilesToUpdate()) { + if (filter == null || filter.accept(file) || file == restrictedTo) { + try { + myForceUpdateSemaphore.down(); + // process only files that can affect result + processFileImpl(project, new com.intellij.ide.caches.FileContent(file), onlyRemoveOutdatedData); + } + finally { + myForceUpdateSemaphore.up(); + } + } + } + + // If several threads entered the method at the same time and there were files to update, + // all the threads should leave the method synchronously after all the files scheduled for update are reindexed, + // no matter which thread will do reindexing job. + // Thus we ensure that all the threads that entered the method will get the most recent data + + while (!myForceUpdateSemaphore.waitFor(500)) { // may need to wait until another thread is done with indexing + if (Thread.holdsLock(PsiLock.LOCK)) { + break; // hack. Most probably that other indexing threads is waiting for PsiLock, which we're are holding. + } + } + } + + private void processFileImpl(Project project, @NotNull final com.intellij.ide.caches.FileContent fileContent, boolean onlyRemoveOutdatedData) { + final VirtualFile file = fileContent.getVirtualFile(); + final boolean reallyRemoved = myFilesToUpdate.remove(file); + if (reallyRemoved && file.isValid()) { + if (onlyRemoveOutdatedData) { + // on shutdown there is no need to re-index the file, just remove outdated data from indices + final List> affected = new ArrayList>(); + for (final ID indexId : myIndices.keySet()) { + if (getInputFilter(indexId).acceptInput(file)) { + affected.add(indexId); + } + } + removeFileDataFromIndices(affected, file); + } + else { + indexFileContent(project, fileContent); + } + IndexingStamp.flushCache(); + } + } + } + + private class UnindexedFilesFinder implements CollectingContentIterator { + private final List myFiles = new ArrayList(); + private final ProgressIndicator myProgressIndicator; + + private UnindexedFilesFinder() { + myProgressIndicator = ProgressManager.getInstance().getProgressIndicator(); + } + + @NotNull + @Override + public List getFiles() { + return myFiles; + } + + @Override + public boolean processFile(@NotNull final VirtualFile file) { + if (!file.isValid()) { + return true; + } + if (!file.isDirectory()) { + if (file instanceof NewVirtualFile && ((NewVirtualFile)file).getFlag(ALREADY_PROCESSED)) { + return true; + } + + if (file instanceof VirtualFileWithId) { + try { + FileTypeManagerImpl.cacheFileType(file, file.getFileType()); + + boolean oldStuff = true; + if (!isTooLarge(file)) { + for (ID indexId : myIndices.keySet()) { + try { + if (needsFileContentLoading(indexId) && shouldIndexFile(file, indexId)) { + myFiles.add(file); + oldStuff = false; + break; + } + } + catch (RuntimeException e) { + final Throwable cause = e.getCause(); + if (cause instanceof IOException || cause instanceof StorageException) { + LOG.info(e); + requestRebuild(indexId); + } + else { + throw e; + } + } + } + } + FileContent fileContent = null; + for (ID indexId : myNotRequiringContentIndices) { + if (shouldIndexFile(file, indexId)) { + oldStuff = false; + try { + if (fileContent == null) { + fileContent = new FileContentImpl(file); + } + updateSingleIndex(indexId, file, fileContent); + } + catch (StorageException e) { + LOG.info(e); + requestRebuild(indexId); + } + } + } + IndexingStamp.flushCache(); + + if (oldStuff && file instanceof NewVirtualFile) { + ((NewVirtualFile)file).setFlag(ALREADY_PROCESSED, true); + } + } + finally { + FileTypeManagerImpl.cacheFileType(file, null); + } + } + } + else { + if (myProgressIndicator != null) { + myProgressIndicator.setText("Scanning files to index"); + myProgressIndicator.setText2(file.getPresentableUrl()); + } + } + return true; + } + } + + private boolean shouldUpdateIndex(final VirtualFile file, final ID indexId) { + return getInputFilter(indexId).acceptInput(file) && + (isMock(file) || IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId))); + } + + private boolean shouldIndexFile(final VirtualFile file, final ID indexId) { + return getInputFilter(indexId).acceptInput(file) && + (isMock(file) || !IndexingStamp.isFileIndexed(file, indexId, IndexInfrastructure.getIndexCreationStamp(indexId))); + } + + private boolean isUnderConfigOrSystem(@NotNull VirtualFile file) { + final String filePath = file.getPath(); + return myConfigPath != null && FileUtil.startsWith(filePath, myConfigPath) || + mySystemPath != null && FileUtil.startsWith(filePath, mySystemPath); + } + + private static boolean isMock(final VirtualFile file) { + return !(file instanceof NewVirtualFile); + } + + private boolean isTooLarge(@NotNull VirtualFile file) { + if (SingleRootFileViewProvider.isTooLarge(file)) { + final FileType type = file.getFileType(); + return !myNoLimitCheckTypes.contains(type); + } + return false; + } + + private boolean isTooLarge(@NotNull VirtualFile file, long contentSize) { + if (SingleRootFileViewProvider.isTooLarge(file, contentSize)) { + final FileType type = file.getFileType(); + return !myNoLimitCheckTypes.contains(type); + } + return false; + } + + @NotNull + public CollectingContentIterator createContentIterator() { + ++myFilesModCount; + return new UnindexedFilesFinder(); + } + + public void registerIndexableSet(@NotNull IndexableFileSet set, @Nullable Project project) { + myIndexableSets.add(set); + myIndexableSetToProjectMap.put(set, project); + } + + public void removeIndexableSet(@NotNull IndexableFileSet set) { + myChangedFilesCollector.forceUpdate(null, null, null, true); + myIndexableSets.remove(set); + myIndexableSetToProjectMap.remove(set); + } + + @Nullable + private static PsiFile findLatestKnownPsiForUncomittedDocument(@NotNull Document doc, @NotNull Project project) { + return PsiDocumentManager.getInstance(project).getCachedPsiFile(doc); + } + + private static class IndexableFilesFilter implements InputFilter { + private final InputFilter myDelegate; + + private IndexableFilesFilter(InputFilter delegate) { + myDelegate = delegate; + } + + @Override + public boolean acceptInput(final VirtualFile file) { + return file instanceof VirtualFileWithId && myDelegate.acceptInput(file); + } + } + + private static void cleanupProcessedFlag() { + final VirtualFile[] roots = ManagingFS.getInstance().getRoots(); + for (VirtualFile root : roots) { + cleanProcessedFlag(root); + } + } + + private static void cleanProcessedFlag(@NotNull final VirtualFile file) { + if (!(file instanceof NewVirtualFile)) return; + + final NewVirtualFile nvf = (NewVirtualFile)file; + if (file.isDirectory()) { + for (VirtualFile child : nvf.getCachedChildren()) { + cleanProcessedFlag(child); + } + } + else { + nvf.setFlag(ALREADY_PROCESSED, false); + } + } + + public static void iterateIndexableFiles(@NotNull final ContentIterator processor, @NotNull Project project, ProgressIndicator indicator) { + if (project.isDisposed()) { + return; + } + final ProjectFileIndex projectFileIndex = ProjectRootManager.getInstance(project).getFileIndex(); + // iterate project content + projectFileIndex.iterateContent(processor); + + if (project.isDisposed()) { + return; + } + + Set visitedRoots = new THashSet(); + for (IndexedRootsProvider provider : Extensions.getExtensions(IndexedRootsProvider.EP_NAME)) { + //important not to depend on project here, to support per-project background reindex + // each client gives a project to FileBasedIndex + if (project.isDisposed()) { + return; + } + for (VirtualFile root : IndexableSetContributor.getRootsToIndex(provider)) { + if (visitedRoots.add(root)) { + iterateRecursively(root, processor, indicator); + } + } + for (VirtualFile root : IndexableSetContributor.getProjectRootsToIndex(provider, project)) { + if (visitedRoots.add(root)) { + iterateRecursively(root, processor, indicator); + } + } + } + + if (project.isDisposed()) { + return; + } + // iterate associated libraries + for (Module module : ModuleManager.getInstance(project).getModules()) { + if (module.isDisposed()) { + return; + } + OrderEntry[] orderEntries = ModuleRootManager.getInstance(module).getOrderEntries(); + for (OrderEntry orderEntry : orderEntries) { + if (orderEntry instanceof LibraryOrderEntry || orderEntry instanceof JdkOrderEntry) { + if (orderEntry.isValid()) { + final VirtualFile[] libSources = orderEntry.getFiles(OrderRootType.SOURCES); + final VirtualFile[] libClasses = orderEntry.getFiles(OrderRootType.CLASSES); + for (VirtualFile[] roots : new VirtualFile[][]{libSources, libClasses}) { + for (VirtualFile root : roots) { + if (visitedRoots.add(root)) { + iterateRecursively(root, processor, indicator); + } + } + } + } + } + } + } + } + + private static void iterateRecursively(@Nullable final VirtualFile root, @NotNull final ContentIterator processor, @Nullable ProgressIndicator indicator) { + if (root != null) { + if (indicator != null) { + indicator.checkCanceled(); + indicator.setText2(root.getPresentableUrl()); + } + + if (root.isDirectory()) { + for (VirtualFile file : root.getChildren()) { + if (file.isDirectory()) { + iterateRecursively(file, processor, indicator); + } + else { + processor.processFile(file); + } + } + } + else { + processor.processFile(root); + } + } + } + + private static class StorageGuard { + private int myHolds = 0; + + public interface Holder { + void leave(); + } + + private final Holder myTrueHolder = new Holder() { + @Override + public void leave() { + StorageGuard.this.leave(true); + } + }; + private final Holder myFalseHolder = new Holder() { + @Override + public void leave() { + StorageGuard.this.leave(false); + } + }; + + @NotNull + public synchronized Holder enter(boolean mode) { + if (mode) { + while (myHolds < 0) { + try { + wait(); + } + catch (InterruptedException ignored) { + } + } + myHolds++; + return myTrueHolder; + } + else { + while (myHolds > 0) { + try { + wait(); + } + catch (InterruptedException ignored) { + } + } + myHolds--; + return myFalseHolder; + } + } + + private synchronized void leave(boolean mode) { + myHolds += mode? -1 : 1; + if (myHolds == 0) { + notifyAll(); + } + } + + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java index 7e1994218251..13fee5b720ab 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexProjectHandler.java @@ -42,12 +42,12 @@ import org.jetbrains.annotations.NotNull; import java.util.Collection; public class FileBasedIndexProjectHandler extends AbstractProjectComponent implements IndexableFileSet { - private final FileBasedIndex myIndex; + private final FileBasedIndexImpl myIndex; private final ProjectRootManagerEx myRootManager; private final FileTypeManager myFileTypeManager; private final ProjectFileExclusionManagerImpl myExclusionManager; - public FileBasedIndexProjectHandler(final FileBasedIndex index, final Project project, final ProjectRootManagerEx rootManager, FileTypeManager ftManager, final ProjectManager projectManager) { + public FileBasedIndexProjectHandler(final FileBasedIndexImpl index, final Project project, final ProjectRootManagerEx rootManager, FileTypeManager ftManager, final ProjectManager projectManager) { super(project); myIndex = index; myRootManager = rootManager; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesUpdater.java b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesUpdater.java index d8ea84dc00b0..efb1445da0be 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesUpdater.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/UnindexedFilesUpdater.java @@ -33,11 +33,11 @@ import java.util.List; */ public class UnindexedFilesUpdater implements CacheUpdater { private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.UnindexedFilesUpdater"); - private final FileBasedIndex myIndex; + private final FileBasedIndexImpl myIndex; private final Project myProject; private long myStarted; - public UnindexedFilesUpdater(final Project project, FileBasedIndex index) { + public UnindexedFilesUpdater(final Project project, FileBasedIndexImpl index) { myIndex = index; myProject = project; } @@ -51,7 +51,7 @@ public class UnindexedFilesUpdater implements CacheUpdater { public VirtualFile[] queryNeededFiles(ProgressIndicator indicator) { CollectingContentIterator finder = myIndex.createContentIterator(); long l = System.currentTimeMillis(); - FileBasedIndex.iterateIndexableFiles(finder, myProject, indicator); + FileBasedIndexImpl.iterateIndexableFiles(finder, myProject, indicator); LOG.info("Indexable files iterated in " + (System.currentTimeMillis() - l) + " ms"); List files = finder.getFiles(); LOG.info("Unindexed files update started: " + files.size() + " files to update"); diff --git a/platform/platform-resources/src/componentSets/Lang.xml b/platform/platform-resources/src/componentSets/Lang.xml index f4ce41d26018..54e0cc2a543c 100644 --- a/platform/platform-resources/src/componentSets/Lang.xml +++ b/platform/platform-resources/src/componentSets/Lang.xml @@ -12,7 +12,8 @@ - com.intellij.util.indexing.FileBasedIndex + com.intellij.util.indexing.FileBasedIndex + com.intellij.util.indexing.FileBasedIndexImpl com.intellij.psi.stubs.StubIndex diff --git a/platform/testFramework/src/com/intellij/testFramework/LightPlatformTestCase.java b/platform/testFramework/src/com/intellij/testFramework/LightPlatformTestCase.java index a8ba4077789d..a7e2d39a9746 100644 --- a/platform/testFramework/src/com/intellij/testFramework/LightPlatformTestCase.java +++ b/platform/testFramework/src/com/intellij/testFramework/LightPlatformTestCase.java @@ -93,6 +93,7 @@ import com.intellij.util.IncorrectOperationException; import com.intellij.util.LocalTimeCounter; import com.intellij.util.containers.CollectionFactory; import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.FileBasedIndexImpl; import com.intellij.util.indexing.IndexableFileSet; import com.intellij.util.messages.MessageBusConnection; import com.intellij.util.ui.UIUtil; @@ -234,7 +235,7 @@ public abstract class LightPlatformTestCase extends UsefulTestCase implements Da throw new RuntimeException(e); } - FileBasedIndex.getInstance().registerIndexableSet(new IndexableFileSet() { + ((FileBasedIndexImpl)FileBasedIndex.getInstance()).registerIndexableSet(new IndexableFileSet() { @Override public boolean isInSet(@NotNull final VirtualFile file) { return ourSourceRoot != null && file.getFileSystem() == ourSourceRoot.getFileSystem() && ourProject.isOpen();