diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 5f1b1fe0dbcb..47dba419f45b 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -99,11 +99,11 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } private AsyncState getAsyncState() { - if (!myInitialized) { // memory barrier - //throw new IndexNotReadyException(); - LOG.error("Unexpected initialization problem"); - } - AsyncState state = myState; + //if (!myInitialized) { // memory barrier + // //throw new IndexNotReadyException(); + // LOG.error("Unexpected initialization problem"); + //} + AsyncState state = myState; // memory barrier if (state == null) { try { myState = state = myStateFuture.get(); @@ -512,9 +512,9 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe } } - @NotNull @Override public StubIndexState getState() { + if (!myInitialized) return null; return new StubIndexState(getAsyncState().myIndices.keySet()); } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 59b72a981352..6d04d4cb96b5 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -99,7 +99,7 @@ import java.util.concurrent.locks.ReentrantLock; * @since Dec 20, 2007 */ public class FileBasedIndexImpl extends FileBasedIndex { - private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.FileBasedIndexImpl"); + static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.FileBasedIndexImpl"); private static final String CORRUPTION_MARKER_NAME = "corruption.marker"; private static final NotificationGroup NOTIFICATIONS = new NotificationGroup("Indexing", NotificationDisplayType.BALLOON, false); @@ -140,17 +140,17 @@ public class FileBasedIndexImpl extends FileBasedIndex { private final IndexAccessValidator myAccessValidator = new IndexAccessValidator(); @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"}) private volatile boolean myInitialized; - // need this variable for memory barrier - private Future myStateFuture; - private volatile State myState; + private Future myStateFuture; + private volatile IndexConfiguration myState; - private State getState() { - if (!myInitialized) { // memory barrier + private IndexConfiguration getState() { + if (!myInitialized) { //throw new IndexNotReadyException(); LOG.error("Unexpected initialization problem"); } - State state = myState; + + IndexConfiguration state = myState; // memory barrier if (state == null) { try { state = myState = myStateFuture.get(); @@ -161,13 +161,6 @@ public class FileBasedIndexImpl extends FileBasedIndex { return state; } - private static class State { - private final Map, Pair, InputFilter>> myIndices = new THashMap, Pair, InputFilter>>(); - private final TObjectIntHashMap> myIndexIdToVersionMap = new TObjectIntHashMap>(); - private final List> myIndicesWithoutFileTypeInfo = new ArrayList>(); - private final Map>> myFileType2IndicesWithFileTypeInfoMap = new THashMap>>(); - } - public FileBasedIndexImpl(@SuppressWarnings("UnusedParameters") VirtualFileManager vfManager, FileDocumentManager fdm, FileTypeManagerImpl fileTypeManager, @@ -241,7 +234,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { private void rebuildAllIndices() { IndexingStamp.flushCaches(); - for (ID indexId : getState().myIndices.keySet()) { + for (ID indexId : getState().getIndexIDs()) { try { clearIndex(indexId); } @@ -331,12 +324,9 @@ public class FileBasedIndexImpl extends FileBasedIndex { /** * @return true if registered index requires full rebuild for some reason, e.g. is just created or corrupted */ - private static boolean registerIndexer(@NotNull final FileBasedIndexExtension extension, State state) throws IOException { + private static boolean registerIndexer(@NotNull final FileBasedIndexExtension extension, IndexConfiguration state) throws IOException { final ID name = extension.getName(); final int version = extension.getVersion(); - synchronized (state) { - state.myIndexIdToVersionMap.put(name, version); - } final File versionFile = IndexInfrastructure.getVersionFile(name); final boolean versionFileExisted = versionFile.exists(); @@ -359,7 +349,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { return versionChanged; } - private static void initIndexStorage(@NotNull FileBasedIndexExtension extension, int version, @NotNull File versionFile, State state) + private static void initIndexStorage(@NotNull FileBasedIndexExtension extension, int version, @NotNull File versionFile, IndexConfiguration state) throws IOException { MapIndexStorage storage = null; final ID name = extension.getName(); @@ -380,38 +370,29 @@ public class FileBasedIndexImpl extends FileBasedIndex { extension.traceKeyHashToVirtualFileMapping() ); - InputFilter inputFilter = extension.getInputFilter(); - Pair, InputFilter> pair = new Pair<>( - createIndex(extension, new MemoryIndexStorage(storage)), - new IndexableFilesFilter(inputFilter) - ); + final InputFilter inputFilter = extension.getInputFilter(); + final Set addedTypes = new THashSet(); - if (inputFilter instanceof FileTypeSpecificInputFilter) { - ((FileTypeSpecificInputFilter)inputFilter).registerFileTypesUsedForIndexing(new Consumer() { - final Set addedTypes = new THashSet(); + if (inputFilter instanceof FileBasedIndex.FileTypeSpecificInputFilter) { + ((FileBasedIndex.FileTypeSpecificInputFilter)inputFilter).registerFileTypesUsedForIndexing(new Consumer() { @Override public void consume(FileType type) { - if (type == null || !addedTypes.add(type)) { - return; - } - - synchronized (state) { - List> ids = state.myFileType2IndicesWithFileTypeInfoMap.get(type); - if (ids == null) state.myFileType2IndicesWithFileTypeInfoMap.put(type, ids = new ArrayList>(5)); - ids.add(name); - } + if (type != null) addedTypes.add(type); } }); } - else { - synchronized (state) { - state.myIndicesWithoutFileTypeInfo.add(name); - } - } - synchronized (state) { - state.myIndices.put(name, pair); - } + state.registerIndex(name, + createIndex(extension, new MemoryIndexStorage(storage)), + new FileBasedIndex.InputFilter() { + + @Override + public boolean acceptInput(@NotNull VirtualFile file) { + return file instanceof VirtualFileWithId && inputFilter.acceptInput(file); + } + }, + version, + addedTypes); break; } catch (Exception e) { @@ -525,8 +506,9 @@ public class FileBasedIndexImpl extends FileBasedIndex { } IndexingStamp.flushCaches(); - for (ID indexId : getState().myIndices.keySet()) { - final UpdatableIndex index = getIndex(indexId); + IndexConfiguration state = getState(); + for (ID indexId : state.getIndexIDs()) { + final UpdatableIndex index = state.getIndex(indexId); assert index != null; if(ourRebuildStatus.get(indexId).get() != OK) { doClearIndex(indexId); // if the index was scheduled for rebuild, only clean it @@ -548,10 +530,11 @@ public class FileBasedIndexImpl extends FileBasedIndex { // All (indices) IDs should be valid in this running session (e.g. we can have ID instance existing but index is not registered) final List> currentFileIndexedStates = IndexingStamp.getNontrivialFileIndexedStates(fileId); Collection> states = currentFileIndexedStates; - Map, Pair, InputFilter>> indices = getState().myIndices; + IndexConfiguration state = getState(); + for(ID currentFileIndexedState: currentFileIndexedStates) { - if (!indices.containsKey(currentFileIndexedState)) { - states = ContainerUtil.intersection(currentFileIndexedStates, indices.keySet()); + if (!state.hasIndex(currentFileIndexedState)) { + states = ContainerUtil.intersection(currentFileIndexedStates, state.getIndexIDs()); break; } } @@ -598,12 +581,13 @@ public class FileBasedIndexImpl extends FileBasedIndex { return; } IndexingStamp.flushCaches(); - for (ID indexId : new ArrayList>(getState().myIndices.keySet())) { + IndexConfiguration state = getState(); + for (ID indexId : new ArrayList>(state.getIndexIDs())) { if (HeavyProcessLatch.INSTANCE.isRunning() || modCount != myLocalModCount) { return; // do not interfere with 'main' jobs } try { - final UpdatableIndex index = getIndex(indexId); + final UpdatableIndex index = state.getIndex(indexId); if (index != null) { index.flush(); } @@ -1169,14 +1153,14 @@ public class FileBasedIndexImpl extends FileBasedIndex { } private void doClearIndex(ID indexId) throws StorageException { - final UpdatableIndex index = getIndex(indexId); + final UpdatableIndex index = myState.getIndex(indexId); assert index != null : "Index with key " + indexId + " not found or not registered properly"; index.clear(); } private void advanceIndexVersion(ID indexId) { try { - IndexingStamp.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), getState().myIndexIdToVersionMap.get(indexId)); + IndexingStamp.rewriteVersion(IndexInfrastructure.getVersionFile(indexId), myState.getIndexVersion(indexId)); } catch (IOException e) { LOG.error(e); @@ -1376,8 +1360,9 @@ public class FileBasedIndexImpl extends FileBasedIndex { if (myPreviousDataBufferingState != enabled) { synchronized (myBufferingStateUpdateLock) { if (myPreviousDataBufferingState != enabled) { - for (ID indexId : getState().myIndices.keySet()) { - final MapReduceIndex index = (MapReduceIndex)getIndex(indexId); + IndexConfiguration state = getState(); + for (ID indexId : state.getIndexIDs()) { + final MapReduceIndex index = (MapReduceIndex)state.getIndex(indexId); assert index != null; ((MemoryIndexStorage)index.getStorage()).setBufferingEnabled(enabled); } @@ -1390,8 +1375,9 @@ public class FileBasedIndexImpl extends FileBasedIndex { private void cleanupMemoryStorage() { myLastIndexedDocStamps.clear(); - for (ID indexId : getState().myIndices.keySet()) { - final MapReduceIndex index = (MapReduceIndex)getIndex(indexId); + IndexConfiguration state = getState(); + for (ID indexId : state.getIndexIDs()) { + final MapReduceIndex index = (MapReduceIndex)state.getIndex(indexId); assert index != null; final MemoryIndexStorage memStorage = (MemoryIndexStorage)index.getStorage(); index.getWriteLock().lock(); @@ -1424,6 +1410,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { cleanupProcessedFlag(); + if (!myInitialized) return; advanceIndexVersion(indexId); final Runnable rebuildRunnable = new Runnable() { @@ -1463,20 +1450,11 @@ public class FileBasedIndexImpl extends FileBasedIndex { } private UpdatableIndex getIndex(ID indexId) { - final Pair, InputFilter> pair = getState().myIndices.get(indexId); - - assert pair != null : "Index data is absent for index " + indexId; - - //noinspection unchecked - return (UpdatableIndex)pair.getFirst(); + return getState().getIndex(indexId); } private InputFilter getInputFilter(@NotNull ID indexId) { - final Pair, InputFilter> pair = getState().myIndices.get(indexId); - - assert pair != null : "Index data is absent for index " + indexId; - - return pair.getSecond(); + return getState().getInputFilter(indexId); } public int getChangedFileCount() { @@ -1621,10 +1599,8 @@ public class FileBasedIndexImpl extends FileBasedIndex { } FileType fileType = file.getFileType(); if(isProjectOrWorkspaceFile(file, fileType)) return Collections.emptyList(); - State state = getState(); - List> ids = state.myFileType2IndicesWithFileTypeInfoMap.get(fileType); - if (ids == null) ids = state.myIndicesWithoutFileTypeInfo; - return ids; + + return getState().getFileTypesForIndex(fileType); } private static void cleanFileContent(@NotNull FileContentImpl fc, PsiFile psiFile) { @@ -2227,19 +2203,6 @@ public class FileBasedIndexImpl extends FileBasedIndex { return PsiDocumentManager.getInstance(project).getCachedPsiFile(doc); } - private static class IndexableFilesFilter implements InputFilter { - private final InputFilter myDelegate; - - private IndexableFilesFilter(InputFilter delegate) { - myDelegate = delegate; - } - - @Override - public boolean acceptInput(@NotNull final VirtualFile file) { - return file instanceof VirtualFileWithId && myDelegate.acceptInput(file); - } - } - private static void cleanupProcessedFlag() { final VirtualFile[] roots = ManagingFS.getInstance().getRoots(); for (VirtualFile root : roots) { @@ -2362,65 +2325,6 @@ public class FileBasedIndexImpl extends FileBasedIndex { return tasks; } - @SuppressWarnings({"WhileLoopSpinsOnField", "SynchronizeOnThis"}) - private static class StorageGuard { - private int myHolds; - private int myWaiters; - - public interface StorageModeExitHandler { - void leave(); - } - - private final StorageModeExitHandler myTrueStorageModeExitHandler = new StorageModeExitHandler() { - @Override - public void leave() { - StorageGuard.this.leave(true); - } - }; - private final StorageModeExitHandler myFalseStorageModeExitHandler = new StorageModeExitHandler() { - @Override - public void leave() { - StorageGuard.this.leave(false); - } - }; - - @NotNull - private synchronized StorageModeExitHandler enter(boolean mode) { - if (mode) { - while (myHolds < 0) { - doWait(); - } - myHolds++; - return myTrueStorageModeExitHandler; - } - else { - while (myHolds > 0) { - doWait(); - } - myHolds--; - return myFalseStorageModeExitHandler; - } - } - - private void doWait() { - try { - ++myWaiters; - wait(); - } - catch (InterruptedException ignored) { - } finally { - --myWaiters; - } - } - - private synchronized void leave(boolean mode) { - myHolds += mode ? -1 : 1; - if (myHolds == 0 && myWaiters > 0) { - notifyAll(); - } - } - } - private final class DocumentUpdateTask extends UpdateTask { private final ID myIndexId; @@ -2434,8 +2338,8 @@ public class FileBasedIndexImpl extends FileBasedIndex { } } - private class FileIndexDataInitialization extends IndexInfrastructure.DataInitialization { - private final State state = new State(); + private class FileIndexDataInitialization extends IndexInfrastructure.DataInitialization { + private final IndexConfiguration state = new IndexConfiguration(); private final AtomicBoolean versionChanged = new AtomicBoolean(); private boolean currentVersionCorrupted; private SerializationManagerEx mySerializationManagerEx; @@ -2494,11 +2398,9 @@ public class FileBasedIndexImpl extends FileBasedIndex { } @Override - protected State finish() { + protected IndexConfiguration finish() { try { - for (List> value : state.myFileType2IndicesWithFileTypeInfoMap.values()) { - value.addAll(state.myIndicesWithoutFileTypeInfo); - } + state.finalizeFileTypeMappingForIndices(); String rebuildNotification = null; if (currentVersionCorrupted) { @@ -2513,9 +2415,10 @@ public class FileBasedIndexImpl extends FileBasedIndex { NOTIFICATIONS.createNotification("Index Rebuild", rebuildNotification, NotificationType.INFORMATION, null).notify(null); } - myState = state; + state.freeze(); + myState = state; // memory barrier // check if rebuild was requested for any index during registration - for (ID indexId : state.myIndices.keySet()) { + for (ID indexId : state.getIndexIDs()) { if (ourRebuildStatus.get(indexId).compareAndSet(REQUIRES_REBUILD, OK)) { try { clearIndex(indexId); @@ -2539,7 +2442,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { performShutdown(); } }); - saveRegisteredIndicesAndDropUnregisteredOnes(state.myIndices.keySet()); + saveRegisteredIndicesAndDropUnregisteredOnes(state.getIndexIDs()); myFlushingFuture = FlushingDaemon.everyFiveSeconds(new Runnable() { private int lastModCount; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexConfiguration.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexConfiguration.java new file mode 100644 index 000000000000..3ed71e8659a9 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexConfiguration.java @@ -0,0 +1,116 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import com.intellij.openapi.fileTypes.FileType; +import com.intellij.openapi.util.Pair; +import gnu.trove.THashMap; +import gnu.trove.TObjectIntHashMap; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +class IndexConfiguration { + private final Map, Pair, FileBasedIndex.InputFilter>> myIndices = + new THashMap, Pair, FileBasedIndex.InputFilter>>(); + private final TObjectIntHashMap> myIndexIdToVersionMap = new TObjectIntHashMap>(); + private final List> myIndicesWithoutFileTypeInfo = new ArrayList>(); + private final Map>> myFileType2IndicesWithFileTypeInfoMap = new THashMap>>(); + private volatile boolean myFreezed; + + UpdatableIndex getIndex(ID indexId) { + assert myFreezed; + final Pair, FileBasedIndex.InputFilter> pair = myIndices.get(indexId); + + assert pair != null : "Index data is absent for index " + indexId; + + //noinspection unchecked + return (UpdatableIndex)pair.getFirst(); + } + + FileBasedIndex.InputFilter getInputFilter(@NotNull ID indexId) { + assert myFreezed; + final Pair, FileBasedIndex.InputFilter> pair = myIndices.get(indexId); + + assert pair != null : "Index data is absent for index " + indexId; + + return pair.getSecond(); + } + + void freeze() { + myFreezed = true; + } + + void registerIndex(ID name, + UpdatableIndex index, + FileBasedIndex.InputFilter inputFilter, + int version, + @Nullable Collection associatedFileTypes + ) { + assert !myFreezed; + + synchronized (myIndices) { + myIndexIdToVersionMap.put(name, version); + + if (associatedFileTypes != null && !associatedFileTypes.isEmpty()) { + for(FileType fileType:associatedFileTypes) { + List> ids = myFileType2IndicesWithFileTypeInfoMap.get(fileType); + if (ids == null) myFileType2IndicesWithFileTypeInfoMap.put(fileType, ids = new ArrayList>(5)); + ids.add(name); + } + } else { + myIndicesWithoutFileTypeInfo.add(name); + } + + myIndices.put(name, new Pair<>(index, inputFilter)); + } + } + + List> getFileTypesForIndex(FileType fileType) { + assert myFreezed; + List> ids = myFileType2IndicesWithFileTypeInfoMap.get(fileType); + if (ids == null) ids = myIndicesWithoutFileTypeInfo; + return ids; + } + + void finalizeFileTypeMappingForIndices() { + assert !myFreezed; + synchronized (myIndices) { + for (List> value : myFileType2IndicesWithFileTypeInfoMap.values()) { + value.addAll(myIndicesWithoutFileTypeInfo); + } + } + } + + Collection> getIndexIDs() { + assert myFreezed; + return myIndices.keySet(); + } + + boolean hasIndex(ID name) { + assert myFreezed; + return myIndices.containsKey(name); + } + + int getIndexVersion(ID id) { + assert myFreezed; + return myIndexIdToVersionMap.get(id); + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/StorageGuard.java b/platform/lang-impl/src/com/intellij/util/indexing/StorageGuard.java new file mode 100644 index 000000000000..39158940954b --- /dev/null +++ b/platform/lang-impl/src/com/intellij/util/indexing/StorageGuard.java @@ -0,0 +1,78 @@ +/* + * Copyright 2000-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +import org.jetbrains.annotations.NotNull; + +@SuppressWarnings({"WhileLoopSpinsOnField", "SynchronizeOnThis"}) +class StorageGuard { + private int myHolds; + private int myWaiters; + + public interface StorageModeExitHandler { + void leave(); + } + + private final StorageModeExitHandler myTrueStorageModeExitHandler = new StorageModeExitHandler() { + @Override + public void leave() { + StorageGuard.this.leave(true); + } + }; + private final StorageModeExitHandler myFalseStorageModeExitHandler = new StorageModeExitHandler() { + @Override + public void leave() { + StorageGuard.this.leave(false); + } + }; + + @NotNull + synchronized StorageModeExitHandler enter(boolean mode) { + if (mode) { + while (myHolds < 0) { + doWait(); + } + myHolds++; + return myTrueStorageModeExitHandler; + } + else { + while (myHolds > 0) { + doWait(); + } + myHolds--; + return myFalseStorageModeExitHandler; + } + } + + private void doWait() { + try { + ++myWaiters; + wait(); + } + catch (InterruptedException ignored) { + } + finally { + --myWaiters; + } + } + + private synchronized void leave(boolean mode) { + myHolds += mode ? -1 : 1; + if (myHolds == 0 && myWaiters > 0) { + notifyAll(); + } + } +}