diff --git a/platform/indexing-api/src/com/intellij/psi/stubs/StringStubIndexExtension.java b/platform/indexing-api/src/com/intellij/psi/stubs/StringStubIndexExtension.java index 8aacbce5f45d..0e4cd6b8fff4 100644 --- a/platform/indexing-api/src/com/intellij/psi/stubs/StringStubIndexExtension.java +++ b/platform/indexing-api/src/com/intellij/psi/stubs/StringStubIndexExtension.java @@ -33,4 +33,8 @@ public abstract class StringStubIndexExtension extends A public KeyDescriptor getKeyDescriptor() { return new EnumeratorStringDescriptor(); } + + public boolean traceKeyHashToVirtualFileMapping() { + return false; + } } \ No newline at end of file diff --git a/platform/indexing-api/src/com/intellij/psi/stubs/StubIndex.java b/platform/indexing-api/src/com/intellij/psi/stubs/StubIndex.java index eee5e6491827..bbe7324cf22e 100644 --- a/platform/indexing-api/src/com/intellij/psi/stubs/StubIndex.java +++ b/platform/indexing-api/src/com/intellij/psi/stubs/StubIndex.java @@ -28,7 +28,9 @@ import com.intellij.psi.search.GlobalSearchScope; import com.intellij.psi.util.PsiUtilCore; import com.intellij.util.Processor; import com.intellij.util.indexing.FileBasedIndex; +import com.intellij.util.indexing.IdFilter; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.Collection; import java.util.Iterator; @@ -59,6 +61,10 @@ public abstract class StubIndex { public abstract boolean processAllKeys(@NotNull StubIndexKey indexKey, @NotNull Project project, Processor processor); + public boolean processAllKeys(@NotNull StubIndexKey indexKey, Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) { + return processAllKeys(indexKey, scope.getProject(), processor); + } + public Collection safeGet(@NotNull StubIndexKey indexKey, @NotNull Key key, @NotNull final Project project, diff --git a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java index 6d07a272279c..926071480d87 100644 --- a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java +++ b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndex.java @@ -126,6 +126,10 @@ public abstract class FileBasedIndex implements BaseComponent { */ public abstract boolean processAllKeys(@NotNull ID indexId, Processor processor, @Nullable Project project); + public boolean processAllKeys(@NotNull ID indexId, @NotNull Processor processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) { + return processAllKeys(indexId, processor, scope.getProject()); + } + public interface ValueProcessor { /** * @param value a value to process @@ -145,4 +149,7 @@ public abstract class FileBasedIndex implements BaseComponent { public interface FileTypeSpecificInputFilter extends InputFilter { void registerFileTypesUsedForIndexing(@NotNull Consumer fileTypeSink); } + + // TODO: remove once changes becomes permamnent + public static final boolean ourEnableTracingOfKeyHashToVirtualFileMapping = ApplicationManager.getApplication().isInternal(); } diff --git a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java index 14fcf8081719..0dc808f4c672 100644 --- a/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java +++ b/platform/indexing-api/src/com/intellij/util/indexing/FileBasedIndexExtension.java @@ -80,6 +80,10 @@ public abstract class FileBasedIndexExtension { return false; } + public boolean traceKeyHashToVirtualFileMapping() { + return false; + } + /** Per-filetype index version support */ public Map getVersionMap() { return Collections.emptyMap(); diff --git a/platform/indexing-api/src/com/intellij/util/indexing/IdFilter.java b/platform/indexing-api/src/com/intellij/util/indexing/IdFilter.java new file mode 100644 index 000000000000..c4b7550b11d8 --- /dev/null +++ b/platform/indexing-api/src/com/intellij/util/indexing/IdFilter.java @@ -0,0 +1,23 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.util.indexing; + +/** +* Created by Maxim.Mossienko on 8/14/13. +*/ +public abstract class IdFilter { + public abstract boolean contains(int id); +} diff --git a/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ChooseByNameBase.java b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ChooseByNameBase.java index fe1a9f9fdd30..9903fbde32ba 100644 --- a/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ChooseByNameBase.java +++ b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ChooseByNameBase.java @@ -70,6 +70,7 @@ import com.intellij.util.Alarm; import com.intellij.util.Consumer; import com.intellij.util.Processor; import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.indexing.FileBasedIndex; import com.intellij.util.text.Matcher; import com.intellij.util.text.MatcherHolder; import com.intellij.util.ui.AsyncProcessIcon; @@ -153,6 +154,7 @@ public abstract class ChooseByNameBase { private String myFindUsagesTitle; private ShortcutSet myCheckBoxShortcut; protected boolean myInitIsDone; + static final boolean ourLoadNamesEachTime = FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping; public boolean checkDisposed() { if (myDisposedFlag && myPostponedOkAction != null && !myPostponedOkAction.isProcessed()) { @@ -795,7 +797,10 @@ public abstract class ChooseByNameBase { } } - if (index == 1 && myModel instanceof ContributorsBasedGotoByModel && myNames[0] != null) { + if (index == 1 && + myModel instanceof ContributorsBasedGotoByModel && + ((ContributorsBasedGotoByModel)myModel).sameNamesForProjectAndLibraries() && + myNames[0] != null) { // there is no way in indices to have different keys for project symbols vs libraries, we always have same ones myNames[1] = myNames[0]; return; @@ -814,6 +819,10 @@ public abstract class ChooseByNameBase { @NotNull public String[] getNames(boolean checkboxState) { + if (ourLoadNamesEachTime) { + myNames[checkboxState ? 1 : 0] = null; + ensureNamesLoaded(checkboxState); + } return checkboxState ? myNames[1] : myNames[0]; } @@ -1193,8 +1202,9 @@ public abstract class ChooseByNameBase { } final String text = myTextField.getText(); + if (text.length() == 0) return Collections.emptyList(); final boolean checkBoxState = myCheckBox.isSelected(); - //ensureNamesLoaded(checkBoxState); + if (ourLoadNamesEachTime) ensureNamesLoaded(checkBoxState); final String[] names = checkBoxState ? myNames[1] : myNames[0]; if (names == null) return Collections.emptyList(); @@ -1489,7 +1499,7 @@ public abstract class ChooseByNameBase { public void run() { try { boolean everywhere = myCheckboxState; - ensureNamesLoaded(everywhere); + if (!ourLoadNamesEachTime) ensureNamesLoaded(everywhere); addElementsByPattern(myPattern, elements, myCancelled, everywhere); } catch (ProcessCanceledException e) { diff --git a/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ContributorsBasedGotoByModel.java b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ContributorsBasedGotoByModel.java index 1fc318a88820..b98d2e4ec854 100644 --- a/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ContributorsBasedGotoByModel.java +++ b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/ContributorsBasedGotoByModel.java @@ -20,6 +20,7 @@ import com.intellij.diagnostic.PluginException; import com.intellij.ide.plugins.PluginManager; import com.intellij.ide.util.NavigationItemListCellRenderer; import com.intellij.navigation.ChooseByNameContributor; +import com.intellij.navigation.EfficientChooseByNameContributor; import com.intellij.navigation.NavigationItem; import com.intellij.openapi.application.ReadActionProcessor; import com.intellij.openapi.diagnostic.Logger; @@ -32,9 +33,12 @@ import com.intellij.openapi.project.DumbService; import com.intellij.openapi.project.IndexNotReadyException; import com.intellij.openapi.project.Project; import com.intellij.util.ArrayUtil; +import com.intellij.util.CommonProcessors; import com.intellij.util.Processor; +import com.intellij.util.containers.ConcurrentHashMap; import com.intellij.util.containers.ContainerUtil; import gnu.trove.THashSet; +import gnu.trove.TIntHashSet; import org.jetbrains.annotations.NotNull; import javax.swing.*; @@ -47,7 +51,7 @@ import java.util.List; /** * Contributor-based goto model */ -public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel { +public abstract class ContributorsBasedGotoByModel implements EfficientChooseByNameModel { public static final Logger LOG = Logger.getInstance("#com.intellij.ide.util.gotoByName.ContributorsBasedGotoByModel"); protected final Project myProject; @@ -74,11 +78,14 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel }; } - @NotNull - @Override - public String[] getNames(final boolean checkBoxState) { - final THashSet allNames = ContainerUtil.newTroveSet(); + public boolean sameNamesForProjectAndLibraries() { + return !ChooseByNameBase.ourLoadNamesEachTime; + } + private final ConcurrentHashMap myContributorToItsSymbolsMap = new ConcurrentHashMap(); + + @Override + public void processNames(final Processor nameProcessor, final boolean checkBoxState) { long start = System.currentTimeMillis(); List liveContribs = filterDumb(myContributors); ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator(); @@ -87,10 +94,30 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel public boolean processInReadAction(ChooseByNameContributor contributor) { try { if (!myProject.isDisposed()) { - String[] names = contributor.getNames(myProject, checkBoxState); - synchronized (allNames) { - allNames.ensureCapacity(names.length); - ContainerUtil.addAll(allNames, names); + long contributorStarted = System.currentTimeMillis(); + final TIntHashSet filter = new TIntHashSet(1000); + myContributorToItsSymbolsMap.put(contributor, filter); + if (contributor instanceof EfficientChooseByNameContributor) { + ((EfficientChooseByNameContributor)contributor).processNames(new Processor() { + @Override + public boolean process(String s) { + if (nameProcessor.process(s)) { + filter.add(s.hashCode()); + } + return true; + } + }, DefaultFileNavigationContributor.getScope(myProject, checkBoxState), DefaultFileNavigationContributor.getFilter(myProject, checkBoxState)); + } else { + String[] names = contributor.getNames(myProject, checkBoxState); + for (String element : names) { + if (nameProcessor.process(element)) { + filter.add(element.hashCode()); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug(contributor + " for " + (System.currentTimeMillis() - contributorStarted)); } } } @@ -114,7 +141,18 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel } long finish = System.currentTimeMillis(); if (LOG.isDebugEnabled()) { - LOG.debug("getNames(): "+(finish-start)+"ms; (got "+allNames.size()+" elements)"); + LOG.debug("processNames(): "+(finish-start)+"ms;"); + } + } + + @NotNull + @Override + public String[] getNames(final boolean checkBoxState) { + final THashSet allNames = ContainerUtil.newTroveSet(); + + processNames(new CommonProcessors.CollectProcessor(allNames), checkBoxState); + if (LOG.isDebugEnabled()) { + LOG.debug("getNames(): (got "+allNames.size()+" elements)"); } return ArrayUtil.toStringArray(allNames); } @@ -133,6 +171,7 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel @NotNull public Object[] getElementsByName(final String name, final boolean checkBoxState, final String pattern, @NotNull final ProgressIndicator canceled) { + long elementByNameStarted = System.currentTimeMillis(); final List items = Collections.synchronizedList(new ArrayList()); Processor processor = new Processor() { @@ -141,8 +180,12 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel if (myProject.isDisposed()) { return true; } + TIntHashSet filter = myContributorToItsSymbolsMap.get(contributor); + if (!filter.contains(name.hashCode())) return true; try { - for (NavigationItem item : contributor.getItemsByName(name, pattern, myProject, checkBoxState)) { + long contributorStarted = System.currentTimeMillis(); + NavigationItem[] itemsByName = contributor.getItemsByName(name, pattern, myProject, checkBoxState); + for (NavigationItem item : itemsByName) { canceled.checkCanceled(); if (item == null) { PluginId pluginId = PluginManager.getPluginByClassName(contributor.getClass().getName()); @@ -159,6 +202,10 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel items.add(item); } } + + if (LOG.isDebugEnabled()) { + LOG.debug(System.currentTimeMillis() - contributorStarted + "," + contributor + "," + itemsByName.length); + } } catch (ProcessCanceledException ex) { // index corruption detected, ignore @@ -173,6 +220,9 @@ public abstract class ContributorsBasedGotoByModel implements ChooseByNameModel canceled.cancel(); } canceled.checkCanceled(); // if parallel job execution was canceled because of PCE, rethrow it from here + if (LOG.isDebugEnabled()) { + LOG.debug("Retrieving " + name + ":" + items.size() + " for " + (System.currentTimeMillis() - elementByNameStarted)); + } return ArrayUtil.toObjectArray(items); } diff --git a/platform/lang-impl/src/com/intellij/ide/util/gotoByName/DefaultChooseByNameItemProvider.java b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/DefaultChooseByNameItemProvider.java index 00de48d0c723..31903e7da41c 100644 --- a/platform/lang-impl/src/com/intellij/ide/util/gotoByName/DefaultChooseByNameItemProvider.java +++ b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/DefaultChooseByNameItemProvider.java @@ -47,11 +47,11 @@ public class DefaultChooseByNameItemProvider implements ChooseByNameItemProvider } @Override - public boolean filterElements(@NotNull ChooseByNameBase base, - @NotNull String pattern, + public boolean filterElements(@NotNull final ChooseByNameBase base, + @NotNull final String pattern, boolean everywhere, @NotNull ProgressIndicator indicator, - @NotNull Processor consumer) { + @NotNull final Processor consumer) { String namePattern = getNamePattern(base, pattern); String qualifierPattern = getQualifierPattern(base, pattern); @@ -60,13 +60,40 @@ public class DefaultChooseByNameItemProvider implements ChooseByNameItemProvider final ChooseByNameModel model = base.getModel(); String matchingPattern = convertToMatchingPattern(base, namePattern); List namesList = new ArrayList(); - String[] names = base.getNames(everywhere); - CollectConsumer collect = new SynchronizedCollectConsumer(namesList); - processNamesByPattern(base, names, matchingPattern, indicator, collect); + + final CollectConsumer collect = new SynchronizedCollectConsumer(namesList); + long started; + + if (model instanceof EfficientChooseByNameModel) { + indicator.checkCanceled(); + started = System.currentTimeMillis(); + final MinusculeMatcher matcher = buildPatternMatcher(matchingPattern, NameUtil.MatchingCaseSensitivity.NONE); + ((EfficientChooseByNameModel)model).processNames(new Processor() { + @Override + public boolean process(String sequence) { + ProgressManager.checkCanceled(); + MatchResult result = matches(base, pattern, matcher, sequence); + if (result != null) { + collect.consume(result); + return true; + } + return false; + } + }, everywhere); + System.out.println("loaded + matched:"+ (System.currentTimeMillis() - started)+ "," + collect.getResult().size()); + } else { + String[] names = base.getNames(everywhere); + started = System.currentTimeMillis(); + processNamesByPattern(base, names, matchingPattern, indicator, collect); + System.out.println("matched:"+ (System.currentTimeMillis() - started)+ "," + names.length); + } indicator.checkCanceled(); - sortNamesList(matchingPattern, (List)collect.getResult()); + started = System.currentTimeMillis(); + List results = (List)collect.getResult(); + sortNamesList(matchingPattern, results); + System.out.println("sorted:"+ (System.currentTimeMillis() - started) + ",results:" + results.size()); indicator.checkCanceled(); List sameNameElements = new SmartList(); diff --git a/platform/lang-impl/src/com/intellij/ide/util/gotoByName/EfficientChooseByNameModel.java b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/EfficientChooseByNameModel.java new file mode 100644 index 000000000000..c50d3473c607 --- /dev/null +++ b/platform/lang-impl/src/com/intellij/ide/util/gotoByName/EfficientChooseByNameModel.java @@ -0,0 +1,22 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.ide.util.gotoByName; + +import com.intellij.util.Processor; + +public interface EfficientChooseByNameModel extends ChooseByNameModel { + void processNames(Processor processor, boolean inLibraries); +} diff --git a/platform/lang-impl/src/com/intellij/navigation/EfficientChooseByNameContributor.java b/platform/lang-impl/src/com/intellij/navigation/EfficientChooseByNameContributor.java new file mode 100644 index 000000000000..d70bd7a10d2f --- /dev/null +++ b/platform/lang-impl/src/com/intellij/navigation/EfficientChooseByNameContributor.java @@ -0,0 +1,25 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.navigation; + +import com.intellij.openapi.project.Project; +import com.intellij.psi.search.GlobalSearchScope; +import com.intellij.util.Processor; +import com.intellij.util.indexing.IdFilter; + +public interface EfficientChooseByNameContributor extends ChooseByNameContributor { + void processNames(Processor processor, GlobalSearchScope scope, IdFilter filter); +} diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java index 744f9104146f..7b5f9ff9039b 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/psi/stubs/StubIndexImpl.java @@ -142,7 +142,9 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe IndexInfrastructure.getStorageFile(indexKey), extension.getKeyDescriptor(), new StubIdExternalizer(), - extension.getCacheSize() + extension.getCacheSize(), + false, + extension instanceof StringStubIndexExtension && ((StringStubIndexExtension)extension).traceKeyHashToVirtualFileMapping() ); final MemoryIndexStorage memStorage = new MemoryIndexStorage(storage); @@ -285,11 +287,16 @@ public class StubIndexImpl extends StubIndex implements ApplicationComponent, Pe @Override public boolean processAllKeys(@NotNull StubIndexKey indexKey, @NotNull Project project, Processor processor) { - FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, project, GlobalSearchScope.allScope(project)); + return processAllKeys(indexKey, processor, GlobalSearchScope.allScope(project), null); + } + + public boolean processAllKeys(@NotNull StubIndexKey indexKey, Processor processor, GlobalSearchScope scope, @Nullable IdFilter idFilter) { + + FileBasedIndex.getInstance().ensureUpToDate(StubUpdatingIndex.INDEX_ID, scope.getProject(), scope); final MyIndex index = (MyIndex)myIndices.get(indexKey); try { - return index.processAllKeys(processor); + return index.processAllKeys(processor, idFilter); } catch (StorageException e) { forceRebuild(e); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java index 60949b83fcbe..10adf97d59b1 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/AbstractIndex.java @@ -18,6 +18,7 @@ package com.intellij.util.indexing; import com.intellij.util.Processor; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; /** * @author Eugene Zhuravlev @@ -27,5 +28,5 @@ public interface AbstractIndex { @NotNull ValueContainer getData(Key key) throws StorageException; - boolean processAllKeys(Processor processor) throws StorageException; + boolean processAllKeys(Processor processor, @Nullable IdFilter idFilter) throws StorageException; } diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 642c455c3ac2..c381c796bcb3 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -413,7 +413,8 @@ public class FileBasedIndexImpl extends FileBasedIndex { extension.getKeyDescriptor(), extension.getValueExternalizer(), extension.getCacheSize(), - extension.isKeyHighlySelective() + extension.isKeyHighlySelective(), + extension.traceKeyHashToVirtualFileMapping() ); final MemoryIndexStorage memStorage = new MemoryIndexStorage(storage); @@ -722,13 +723,17 @@ public class FileBasedIndexImpl extends FileBasedIndex { @Override public boolean processAllKeys(@NotNull final ID indexId, Processor processor, @Nullable Project project) { + return processAllKeys(indexId, processor, project != null ? GlobalSearchScope.allScope(project) : new EverythingGlobalScope(), null); + } + + public boolean processAllKeys(@NotNull ID indexId, Processor processor, @NotNull GlobalSearchScope scope, @Nullable IdFilter idFilter) { try { final UpdatableIndex index = getIndex(indexId); if (index == null) { return true; } - ensureUpToDate(indexId, project, project != null ? GlobalSearchScope.allScope(project) : new EverythingGlobalScope()); - return index.processAllKeys(processor); + ensureUpToDate(indexId, scope.getProject(), scope); + return index.processAllKeys(processor, idFilter); } catch (StorageException e) { scheduleRebuild(indexId, e); @@ -1014,7 +1019,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { myContentlessIndicesUpdateQueue.signalUpdateEnd(); } - public static final class ProjectIndexableFilesFilter { + public static final class ProjectIndexableFilesFilter extends IdFilter { private static final int SHIFT = 6; private static final int MASK = (1 << SHIFT) - 1; private final long[] myBitMask; @@ -1735,7 +1740,9 @@ public class FileBasedIndexImpl extends FileBasedIndex { } FileType fileType = file.getFileType(); if(isProjectOrWorkspaceFile(file, fileType)) return Collections.emptyList(); - + if (fileType == StdFileTypes.HTML || fileType == StdFileTypes.XML) { + int a = 1; + } List> ids = myFileType2IndicesWithFileTypeInfoMap.get(fileType); if (ids == null) ids = myIndicesWithoutFileTypeInfo; return ids; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java index a268c0c12f27..5e0296b335dd 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/IndexStorage.java @@ -18,6 +18,7 @@ package com.intellij.util.indexing; import com.intellij.util.Processor; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.Flushable; import java.io.IOException; @@ -38,7 +39,7 @@ public interface IndexStorage extends Flushable { @NotNull ValueContainer read(Key key) throws StorageException; - boolean processKeys(Processor processor) throws StorageException; + boolean processKeys(Processor processor, @Nullable IdFilter idFilter) throws StorageException; Collection getKeys() throws StorageException; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java index ffdc0ab5ae69..0448c65d006b 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapIndexStorage.java @@ -22,12 +22,13 @@ import com.intellij.openapi.util.io.FileUtil; import com.intellij.util.CommonProcessors; import com.intellij.util.Processor; import com.intellij.util.containers.SLRUCache; -import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.KeyDescriptor; -import com.intellij.util.io.PersistentMap; +import com.intellij.util.io.*; +import gnu.trove.TIntHashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.DataInput; +import java.io.DataOutput; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -42,7 +43,9 @@ import java.util.concurrent.locks.ReentrantLock; */ public final class MapIndexStorage implements IndexStorage{ private static final Logger LOG = Logger.getInstance("#com.intellij.util.indexing.MapIndexStorage"); + private final boolean myBuildKeyHashToVirtualFileMapping; private PersistentMap> myMap; + private PersistentBTreeEnumerator myKeyHashToVirtualFileMapping; private SLRUCache> myCache; private final File myStorageFile; private final KeyDescriptor myKeyDescriptor; @@ -71,20 +74,23 @@ public final class MapIndexStorage implements IndexStorage valueExternalizer, final int cacheSize ) throws IOException { - this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false); + this(storageFile, keyDescriptor, valueExternalizer, cacheSize, false, false); } public MapIndexStorage(@NotNull File storageFile, @NotNull KeyDescriptor keyDescriptor, @NotNull DataExternalizer valueExternalizer, final int cacheSize, - boolean highKeySelectivity) throws IOException { + boolean highKeySelectivity, + boolean buildKeyHashToVirtualFileMapping + ) throws IOException { myStorageFile = storageFile; myKeyDescriptor = keyDescriptor; myCacheSize = cacheSize; myDataExternalizer = valueExternalizer; myHighKeySelectivity = highKeySelectivity; + myBuildKeyHashToVirtualFileMapping = buildKeyHashToVirtualFileMapping && FileBasedIndex.ourEnableTracingOfKeyHashToVirtualFileMapping; initMapAndCache(); } @@ -133,6 +139,37 @@ public final class MapIndexStorage implements IndexStorage(getProjectFile(), new KeyDescriptor() { + @Override + public void save(DataOutput out, int[] value) throws IOException { + DataInputOutputUtil.writeINT(out, value[0]); + DataInputOutputUtil.writeINT(out, value[1]); + } + + @Override + public int[] read(DataInput in) throws IOException { + return new int[] {DataInputOutputUtil.readINT(in), DataInputOutputUtil.readINT(in)}; + } + + @Override + public int getHashCode(int[] value) { + return value[0] * 31 + value[1]; + } + + @Override + public boolean isEqual(int[] val1, int[] val2) { + return val1[0] == val2[0] && val1[1] == val2[1]; + } + }, 4096) { + protected boolean serializationEquivalenceIsExhausting() { + return true; + } + }: null; + } + + private File getProjectFile() { + return new File(myStorageFile.getPath() + ".project"); } @Override @@ -143,6 +180,7 @@ public final class MapIndexStorage implements IndexStorage implements IndexStorage implements IndexStorage implements IndexStorage processor) throws StorageException { + public boolean processKeys(final Processor processor, final IdFilter idFilter) throws StorageException { l.lock(); try { myCache.clear(); // this will ensure that all new keys are made into the map + if (myBuildKeyHashToVirtualFileMapping && idFilter != null) { + final TIntHashSet hashMaskSet = new TIntHashSet(1000); + long l = System.currentTimeMillis(); + myKeyHashToVirtualFileMapping.iterateData(new Processor() { + @Override + public boolean process(int[] key) { + if (!idFilter.contains(key[1])) return true; + hashMaskSet.add(key[0]); + return true; + } + }); + if (LOG.isDebugEnabled()) { + LOG.debug("Scanned keyHashToVirtualFileMapping of " + myStorageFile + " for " + (System.currentTimeMillis() - l)); + } + return myMap.processKeys(new Processor() { + @Override + public boolean process(Key key) { + if (!hashMaskSet.contains(myKeyDescriptor.getHashCode(key))) return true; + return processor.process(key); + } + }); + } return myMap.processKeys(processor); } catch (IOException e) { @@ -227,7 +290,7 @@ public final class MapIndexStorage implements IndexStorage getKeys() throws StorageException { List keys = new ArrayList(); - processKeys(new CommonProcessors.CollectProcessor(keys)); + processKeys(new CommonProcessors.CollectProcessor(keys), null); return keys; } @@ -256,6 +319,10 @@ public final class MapIndexStorage implements IndexStorage implements UpdatableIndex processor) throws StorageException { + public boolean processAllKeys(Processor processor, IdFilter idFilter) throws StorageException { final Lock lock = getReadLock(); try { lock.lock(); - return myStorage.processKeys(processor); + return myStorage.processKeys(processor, idFilter); } finally { lock.unlock(); diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java index 7a87af782497..f07fc85b85cb 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MemoryIndexStorage.java @@ -101,12 +101,12 @@ public class MemoryIndexStorage implements IndexStorage @Override public Collection getKeys() throws StorageException { final Set keys = new HashSet(); - processKeys(new CommonProcessors.CollectProcessor(keys)); + processKeys(new CommonProcessors.CollectProcessor(keys), null); return keys; } @Override - public boolean processKeys(final Processor processor) throws StorageException { + public boolean processKeys(final Processor processor, IdFilter idFilter) throws StorageException { final Set stopList = new HashSet(); Processor decoratingProcessor = new Processor() { @@ -128,7 +128,7 @@ public class MemoryIndexStorage implements IndexStorage } stopList.add(key); } - return myBackendStorage.processKeys(decoratingProcessor); + return myBackendStorage.processKeys(stopList.size() == 0 && myMap.size() == 0 ? processor : decoratingProcessor, idFilter); } @Override diff --git a/platform/util/src/com/intellij/util/io/PersistentEnumeratorBase.java b/platform/util/src/com/intellij/util/io/PersistentEnumeratorBase.java index 172e74a7b81c..543483c5230b 100644 --- a/platform/util/src/com/intellij/util/io/PersistentEnumeratorBase.java +++ b/platform/util/src/com/intellij/util/io/PersistentEnumeratorBase.java @@ -39,7 +39,7 @@ import java.util.List; * @author jeka */ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") -abstract class PersistentEnumeratorBase implements Forceable, Closeable { +abstract public class PersistentEnumeratorBase implements Forceable, Closeable { protected static final Logger LOG = Logger.getInstance("#com.intellij.util.io.PersistentEnumerator"); protected static final int NULL_ID = 0; @@ -427,10 +427,14 @@ abstract class PersistentEnumeratorBase implements Forceable, Closeable { comparer.close(); if (sameValue[0]) return true; - + if (serializationEquivalenceIsExhausting()) return false; return myDataDescriptor.isEqual(valueOf(idx), value); } + protected boolean serializationEquivalenceIsExhausting() { + return false; + } + protected int writeData(final Data value, int hashCode) { try { markDirty(true); @@ -498,7 +502,7 @@ abstract class PersistentEnumeratorBase implements Forceable, Closeable { return pos; } - protected boolean iterateData(final Processor processor) throws IOException { + public boolean iterateData(final Processor processor) throws IOException { lockStorage(); try { if (myKeyStorage == null) {